From 355ddef631f26efd23d1847a5e7de0d1b436b04b Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 24 Feb 2025 06:27:18 +0000 Subject: [PATCH] Artifacts: Add download size to `Artifacts.toml` This shows one possible way of adding an optional download size value to an artifact binding. While this value is not used anywhere yet, it could be used to create warnings when downloading large artifacts, or to feed progress meters when the server does not provide content size headers. --- src/Artifacts.jl | 66 ++++++++++++++++++++++++++++++++++++++++------- test/artifacts.jl | 11 +++++--- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 0702e3670c..b3dfcde867 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -18,7 +18,7 @@ import ..Types: write_env_usage, parse_toml export create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact, artifact_meta, artifact_hash, bind_artifact!, unbind_artifact!, download_artifact, find_artifacts_toml, ensure_artifact_installed, @artifact_str, archive_artifact, - select_downloadable_artifacts + select_downloadable_artifacts, ArtifactDownloadInfo """ create_artifact(f::Function) @@ -138,6 +138,56 @@ function archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Boo end end +""" + ArtifactDownloadInfo + +Auxilliary information about an artifact to be used with `bind_artifact!()` to give +a download location for that artifact, as well as the hash and size of that artifact. +""" +struct ArtifactDownloadInfo + # URL the artifact is available at as a gzip-compressed tarball + url::String + + # SHA256 hash of the tarball + hash::Vector{UInt8} + + # Size in bytes of the tarball. `size <= 0` means unknown. + size::Int64 + + function ArtifactDownloadInfo(url, hash::AbstractVector, size = 0) + valid_hash_len = SHA.digestlen(SHA256_CTX) + hash_len = length(hash) + if hash_len != valid_hash_len + throw(ArgumentError("Invalid hash length '$(hash_len)', must be $(valid_hash_len)")) + end + return new( + String(url), + Vector{UInt8}(hash), + Int64(size), + ) + end +end + +# Convenience constructor for string hashes +ArtifactDownloadInfo(url, hash::AbstractString, args...) = ArtifactDownloadInfo(url, hex2bytes(hash), args...) + +# Convenience constructor for legacy Tuple representation +ArtifactDownloadInfo(args::Tuple) = ArtifactDownloadInfo(args...) + +ArtifactDownloadInfo(adi::ArtifactDownloadInfo) = adi + +# Make the dict that will be embedded in the TOML +function make_dict(adi::ArtifactDownloadInfo) + ret = Dict{String,Any}( + "url" => adi.url, + "sha256" => bytes2hex(adi.hash), + ) + if adi.size > 0 + ret["size"] = adi.size + end + return ret +end + """ bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; platform::Union{AbstractPlatform,Nothing} = nothing, @@ -159,7 +209,7 @@ downloaded until it is accessed via the `artifact"name"` syntax, or """ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; platform::Union{AbstractPlatform,Nothing} = nothing, - download_info::Union{Vector{<:Tuple},Nothing} = nothing, + download_info::Union{Vector{<:Tuple},Vector{<:ArtifactDownloadInfo},Nothing} = nothing, lazy::Bool = false, force::Bool = false) # First, check to see if this artifact is already bound: @@ -188,15 +238,11 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; meta["lazy"] = true end - # Integrate download info, if it is given. We represent the download info as a - # vector of dicts, each with its own `url` and `sha256`, since different tarballs can - # expand to the same tree hash. + # Integrate download info, if it is given. Note that there can be multiple + # download locations, each with its own tarball with its own hash, but which + # expands to the same content/treehash. if download_info !== nothing - meta["download"] = [ - Dict("url" => dl[1], - "sha256" => dl[2], - ) for dl in download_info - ] + meta["download"] = make_dict.(ArtifactDownloadInfo.(download_info)) end if platform === nothing diff --git a/test/artifacts.jl b/test/artifacts.jl index fb9c6e1a66..4f180e5f51 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -238,8 +238,8 @@ end # Test platform-specific binding and providing download_info download_info = [ - ("http://google.com/hello_world", "0"^64), - ("http://microsoft.com/hello_world", "a"^64), + ArtifactDownloadInfo("http://google.com/hello_world", "0"^64), + ArtifactDownloadInfo("http://microsoft.com/hello_world", "a"^64, 1), ] # First, test the binding of things with various platforms and overwriting and such works properly @@ -249,8 +249,8 @@ end @test artifact_hash("foo_txt", artifacts_toml; platform=linux64) == hash @test artifact_hash("foo_txt", artifacts_toml; platform=Platform("x86_64", "macos")) == nothing @test_throws ErrorException bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64) - bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64, force=true) bind_artifact!(artifacts_toml, "foo_txt", hash; download_info=download_info, platform=win32) + bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64, force=true) @test artifact_hash("foo_txt", artifacts_toml; platform=linux64) == hash2 @test artifact_hash("foo_txt", artifacts_toml; platform=win32) == hash @test ensure_artifact_installed("foo_txt", artifacts_toml; platform=linux64) == artifact_path(hash2) @@ -259,7 +259,9 @@ end # Next, check that we can get the download_info properly: meta = artifact_meta("foo_txt", artifacts_toml; platform=win32) @test meta["download"][1]["url"] == "http://google.com/hello_world" + @test !haskey(meta["download"][1], "size") @test meta["download"][2]["sha256"] == "a"^64 + @test meta["download"][2]["size"] == 1 rm(artifacts_toml) @@ -419,11 +421,12 @@ end ) disengaged_platform = HostPlatform() disengaged_platform["flooblecrank"] = "disengaged" + disengaged_adi = ArtifactDownloadInfo(disengaged_url, disengaged_sha256) Pkg.Artifacts.bind_artifact!( artifacts_toml, "gooblebox", disengaged_hash; - download_info = [(disengaged_url, disengaged_sha256)], + download_info = [disengaged_adi], platform = disengaged_platform, ) end