Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Artifacts: Add download size to Artifacts.toml #4171

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Artifacts: Add download size to Artifacts.toml #4171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 56 additions & 10 deletions src/Artifacts.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import ..Types: write_env_usage, parse_toml
export create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact,
artifact_meta, artifact_hash, bind_artifact!, unbind_artifact!, download_artifact,
find_artifacts_toml, ensure_artifact_installed, @artifact_str, archive_artifact,
select_downloadable_artifacts
select_downloadable_artifacts, ArtifactDownloadInfo

"""
create_artifact(f::Function)
Expand Down Expand Up @@ -138,6 +138,56 @@ function archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Boo
end
end

"""
ArtifactDownloadInfo

Auxilliary information about an artifact to be used with `bind_artifact!()` to give
a download location for that artifact, as well as the hash and size of that artifact.
"""
struct ArtifactDownloadInfo
# URL the artifact is available at as a gzip-compressed tarball
url::String

# SHA256 hash of the tarball
hash::Vector{UInt8}

# Size in bytes of the tarball. `size <= 0` means unknown.
size::Int64

function ArtifactDownloadInfo(url, hash::AbstractVector, size = 0)
valid_hash_len = SHA.digestlen(SHA256_CTX)
hash_len = length(hash)
if hash_len != valid_hash_len
throw(ArgumentError("Invalid hash length '$(hash_len)', must be $(valid_hash_len)"))
end
return new(
String(url),
Vector{UInt8}(hash),
Int64(size),
)
end
end

# Convenience constructor for string hashes
ArtifactDownloadInfo(url, hash::AbstractString, args...) = ArtifactDownloadInfo(url, hex2bytes(hash), args...)

# Convenience constructor for legacy Tuple representation
ArtifactDownloadInfo(args::Tuple) = ArtifactDownloadInfo(args...)

ArtifactDownloadInfo(adi::ArtifactDownloadInfo) = adi

# Make the dict that will be embedded in the TOML
function make_dict(adi::ArtifactDownloadInfo)
ret = Dict{String,Any}(
"url" => adi.url,
"sha256" => bytes2hex(adi.hash),
)
if adi.size > 0
ret["size"] = adi.size
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
ret["size"] = adi.size
ret["size"] = Int64(adi.size)

Converting UInt64 to Int64 here prevents TOML.jl from writing the number in hex.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Darn, I was hoping it would just call string(x), but I guess not. I'll just make everything an Int64 then.

end
return ret
end

"""
bind_artifact!(artifacts_toml::String, name::String, hash::SHA1;
platform::Union{AbstractPlatform,Nothing} = nothing,
Expand All @@ -159,7 +209,7 @@ downloaded until it is accessed via the `artifact"name"` syntax, or
"""
function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1;
platform::Union{AbstractPlatform,Nothing} = nothing,
download_info::Union{Vector{<:Tuple},Nothing} = nothing,
download_info::Union{Vector{<:Tuple},Vector{<:ArtifactDownloadInfo},Nothing} = nothing,
lazy::Bool = false,
force::Bool = false)
# First, check to see if this artifact is already bound:
Expand Down Expand Up @@ -188,15 +238,11 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1;
meta["lazy"] = true
end

# Integrate download info, if it is given. We represent the download info as a
# vector of dicts, each with its own `url` and `sha256`, since different tarballs can
# expand to the same tree hash.
# Integrate download info, if it is given. Note that there can be multiple
# download locations, each with its own tarball with its own hash, but which
# expands to the same content/treehash.
if download_info !== nothing
meta["download"] = [
Dict("url" => dl[1],
"sha256" => dl[2],
) for dl in download_info
]
meta["download"] = make_dict.(ArtifactDownloadInfo.(download_info))
end

if platform === nothing
Expand Down
11 changes: 7 additions & 4 deletions test/artifacts.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,8 @@ end

# Test platform-specific binding and providing download_info
download_info = [
("http://google.com/hello_world", "0"^64),
("http://microsoft.com/hello_world", "a"^64),
ArtifactDownloadInfo("http://google.com/hello_world", "0"^64),
ArtifactDownloadInfo("http://microsoft.com/hello_world", "a"^64, 1),
]

# First, test the binding of things with various platforms and overwriting and such works properly
Expand All @@ -249,8 +249,8 @@ end
@test artifact_hash("foo_txt", artifacts_toml; platform=linux64) == hash
@test artifact_hash("foo_txt", artifacts_toml; platform=Platform("x86_64", "macos")) == nothing
@test_throws ErrorException bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64)
bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64, force=true)
bind_artifact!(artifacts_toml, "foo_txt", hash; download_info=download_info, platform=win32)
bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64, force=true)
@test artifact_hash("foo_txt", artifacts_toml; platform=linux64) == hash2
@test artifact_hash("foo_txt", artifacts_toml; platform=win32) == hash
@test ensure_artifact_installed("foo_txt", artifacts_toml; platform=linux64) == artifact_path(hash2)
Expand All @@ -259,7 +259,9 @@ end
# Next, check that we can get the download_info properly:
meta = artifact_meta("foo_txt", artifacts_toml; platform=win32)
@test meta["download"][1]["url"] == "http://google.com/hello_world"
@test !haskey(meta["download"][1], "size")
@test meta["download"][2]["sha256"] == "a"^64
@test meta["download"][2]["size"] == 1

rm(artifacts_toml)

Expand Down Expand Up @@ -419,11 +421,12 @@ end
)
disengaged_platform = HostPlatform()
disengaged_platform["flooblecrank"] = "disengaged"
disengaged_adi = ArtifactDownloadInfo(disengaged_url, disengaged_sha256)
Pkg.Artifacts.bind_artifact!(
artifacts_toml,
"gooblebox",
disengaged_hash;
download_info = [(disengaged_url, disengaged_sha256)],
download_info = [disengaged_adi],
platform = disengaged_platform,
)
end
Expand Down