Skip to content

Commit

Permalink
omit chunks on client HTTP error (like 404, 403,...) (#134)
Browse files Browse the repository at this point in the history
* omit chunk on client HTTP error

* Allow non-404 errors only if user sets it

* Add a doc string

* add a test case

---------

Co-authored-by: Fabian Gans <[email protected]>
  • Loading branch information
Alexander-Barth and meggart authored Feb 8, 2024
1 parent cbcaead commit d932969
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 5 deletions.
36 changes: 31 additions & 5 deletions src/Storage/http.jl
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,24 @@ python package. In case you experience performance options, one can try to use
"""
struct HTTPStore <: AbstractStore
url::String
allowed_codes::Set{Int}
end
HTTPStore(url) = HTTPStore(url,Set((404,)))

function Base.getindex(s::HTTPStore, k::String)
r = HTTP.request("GET",string(s.url,"/",k),status_exception = false,socket_type_tls=OpenSSL.SSLStream)
if r.status >= 300
if r.status == 404
if r.status in s.allowed_codes
nothing
else
error("Error connecting to $(s.url) :", String(r.body))
err_msg =
"""Received error code $(r.status) when connecting to $(s.url) with message $(String(r.body)).
This might be an actual error or an indication that the server returns a different error code
than 404 for missing chunks. In the later case you can run
`Zarr.missing_chunk_return_code!(a.storage,$(r.status))` where a is your Zarr array or group to
fix the issue.
"""
throw(ErrorException(err_msg))
end
else
r.body
Expand All @@ -32,11 +41,28 @@ push!(storageregexlist,r"^https://"=>HTTPStore)
push!(storageregexlist,r"^http://"=>HTTPStore)
storefromstring(::Type{<:HTTPStore}, s,_) = ConsolidatedStore(HTTPStore(s),""),""

"""
missing_chunk_return_code!(s::HTTPStore, code::Union{Int,AbstractVector{Int}})
Extends the list of HTTP return codes that signals that a certain key in a HTTPStore is not available. Most data providers
return code 404 for missing elements, but some may use different return codes like 403. This function can be used
to add return codes that signal missing chunks.
### Example
````julia
a = zopen("https://path/to/remote/array")
missing_chunk_return_code!(a.storage, 403)
````
"""
missing_chunk_return_code!(s::ConsolidatedStore,code) = missing_chunk_return_code!(s.parent,code)
missing_chunk_return_code!(s::HTTPStore, code::Integer) = push!(s.allowed_codes,code)
missing_chunk_return_code!(s::HTTPStore, codes::AbstractVector{<:Integer}) = foreach(c->push!(s.allowed_codes,c),codes)
store_read_strategy(::HTTPStore) = ConcurrentRead(concurrent_io_tasks[])


## This is a server implementation for Zarr datasets
function zarr_req_handler(s::AbstractStore, p)
function zarr_req_handler(s::AbstractStore, p, notfound = 404)
if s[p,".zmetadata"] === nothing
consolidate_metadata(s)
end
Expand All @@ -47,12 +73,12 @@ function zarr_req_handler(s::AbstractStore, p)
r = s[p,k]
try
if r === nothing
return HTTP.Response(404, "Error: Key $k not found")
return HTTP.Response(notfound, "Error: Key $k not found")
else
return HTTP.Response(200, r)
end
catch e
return HTTP.Response(404, "Error: $e")
return HTTP.Response(notfound, "Error: $e")
end
end
end
Expand Down
12 changes: 12 additions & 0 deletions test/storage.jl
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,16 @@ end
@test g2["a1"].attrs == Dict("arratt"=>2.5)
@test g2["a1"][:,:] == reshape(1:200,10,20)
close(server)
#Test server that returns 403 instead of 404 for missing chunks
server = Sockets.listen(0)
ip,port = getsockname(server)
s = Zarr.DictStore()
g = zgroup(s, attrs = Dict("groupatt"=>5))
a = zcreate(Int,g,"a",10,20,chunks=(5,5),attrs=Dict("arratt"=>2.5),fill_value = -1)
@async HTTP.serve(Zarr.zarr_req_handler(s,g.path,403),ip,port,server=server)
g3 = zopen("http://$ip:$port")
@test_throws "Received error code 403" g3["a"][:,:]
Zarr.missing_chunk_return_code!(g3.storage,403)
@test all(==(-1),g3["a"][:,:])
close(server)
end

0 comments on commit d932969

Please sign in to comment.