From 547c7bfa4193fb696a981f3376cf4308787133be Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 11:20:57 +0200 Subject: [PATCH 01/19] Add latest versions. --- .github/workflows/ci.yml | 2 +- deps/Releases.toml | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f0c8e06b..ecc800156 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: arch: - x64 build_spec: - - v1.5.0 # entry straight in Versions.toml + - v1.7.0 # entry straight in Versions.toml - nightly # entry from Builds.toml - 8cb458c6dcd8e067a3bd430b006efb0dfde56cf9 # directly from Git, never built - master # directly from Git, likely built diff --git a/deps/Releases.toml b/deps/Releases.toml index 1a15545d9..6b6202b73 100644 --- a/deps/Releases.toml +++ b/deps/Releases.toml @@ -16,11 +16,17 @@ url = "https://julialang-s3.julialang.org/bin/linux/x64/1.4/julia-1.4-latest-lin ["1.5"] url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5-latest-linux-x86_64.tar.gz" +["1.6"] +url = "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6-latest-linux-x86_64.tar.gz" + +["1.7"] +url = "https://julialang-s3.julialang.org/bin/linux/x64/1.7/julia-1.7-latest-linux-x86_64.tar.gz" + ["lts"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0-latest-linux-x86_64.tar.gz" +url = "https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.6-latest-linux-x86_64.tar.gz" ["stable"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2-latest-linux-x86_64.tar.gz" +url = "https://julialang-s3.julialang.org/bin/linux/x64/1.7/julia-1.7-latest-linux-x86_64.tar.gz" ["nightly"] url = "https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz" From 9740b5eea7f49b330874c1f6a0c9649802933575 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 11:21:11 +0200 Subject: [PATCH 02/19] Reorganize skipped packages. --- deps/Registries.toml | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/deps/Registries.toml b/deps/Registries.toml index bd5b31fed..efda1287e 100644 --- a/deps/Registries.toml +++ b/deps/Registries.toml @@ -3,18 +3,26 @@ url = "https://github.com/JuliaRegistries/General.git" uuid = "23338594-aafe-5451-b93e-139f81909106" skip = [ "julia", - "MATLAB", # Requires MATLAB - "MATLABDiffEq", # Requires MATLAB - "deSolveDiffEq", # Requires R - "FEniCS", # Requires Docker Installation - "NeuralNetDiffEq", # Deprecated - "DiffEqBiological", # Deprecated - "DiffEqMonteCarlo", # Deprecated - "DiffEqPDEBase", # Deprecated - "ClimateMachine", # Requires MPI - "MPI", # Requires MPI - "CUDAnative", # Deprecated - "BitemporalPostgres" # requires POSTGRES + "BugReporting", # nested rr is not supported + + # deprecated + "NeuralNetDiffEq", + "DiffEqBiological", + "DiffEqMonteCarlo", + "DiffEqPDEBase", + "CUDAnative", + "CuArrays", + "CUDAdrv", + "CUDAapi", + + # requires binaries + "MATLAB", # MATLAB + "MATLABDiffEq", # MATLAB + "deSolveDiffEq", # R + "FEniCS", # Docker + "ClimateMachine", # MPI + "MPI", # MPI + "BitemporalPostgres", # POSTGRES ] retry = [ "EcologicalNetworks", From 5324fe9c3bf0a1b43b700eb5ca701e4c88f80d1d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 11:21:29 +0200 Subject: [PATCH 03/19] Detect network errors before test failures. --- src/run.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/run.jl b/src/run.jl index 05bf81b84..8526a7086 100644 --- a/src/run.jl +++ b/src/run.jl @@ -399,10 +399,6 @@ function run_sandboxed_test(install::String, pkg; do_depwarns=false, kwargs...) :missing_dependency elseif occursin(r"Package .+ not found in current path", log) :missing_package - elseif occursin("Some tests did not pass", log) || occursin("Test Failed", log) - :test_failures - elseif occursin("ERROR: LoadError: syntax", log) - :syntax elseif occursin("GC error (probable corruption)", log) :gc_corruption elseif occursin("signal (11): Segmentation fault", log) @@ -418,6 +414,10 @@ function run_sandboxed_test(install::String, pkg; do_depwarns=false, kwargs...) occursin("Could not download", log) || occursin(r"Error: HTTP/\d \d+", log) :network + elseif occursin("ERROR: LoadError: syntax", log) + :syntax + elseif occursin("Some tests did not pass", log) || occursin("Test Failed", log) + :test_failures else :unknown end From 18338c4904e01ce8e8f29007be26eee55ad04fe8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 11:21:44 +0200 Subject: [PATCH 04/19] Don't needlessly update the registry during testing. --- test/runtests.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c29a9d261..ab5d48731 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,22 +28,21 @@ end const pkgnames = ["TimerOutputs", "Crayons", "Example", "Gtk"] -@testset "low-level interface" begin - PkgEval.prepare_registry() - - pkgs = PkgEval.read_pkgs(pkgnames) - +@testset "time and output limits" begin # timeouts - results = PkgEval.run([Configuration(julia=julia_version)], pkgs; time_limit = 0.1) + results = PkgEval.run([Configuration(julia=julia_version)], pkgnames; + time_limit=0.1, update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :time_limit) # log limit - results = PkgEval.run([Configuration(julia=julia_version)], pkgs; log_limit = 1) + results = PkgEval.run([Configuration(julia=julia_version)], pkgnames; + log_limit=1, update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :log_limit) end @testset "main entrypoint" begin - results = PkgEval.run([Configuration(julia=julia_version)], pkgnames) + results = PkgEval.run([Configuration(julia=julia_version)], pkgnames; + update_registry=false) if !(julia_spec == "master" || julia_spec == "nightly") @test all(results.status .== :ok) for result in eachrow(results) @@ -53,7 +52,8 @@ end end @testset "PackageCompiler" begin - results = PkgEval.run([Configuration(julia=julia_version, compiled=true)], ["Example"]) + results = PkgEval.run([Configuration(julia=julia_version, compiled=true)], ["Example"]; + update_registry=false) if !(julia_spec == "master" || julia_spec == "nightly") @test all(results.status .== :ok) for result in eachrow(results) @@ -65,7 +65,7 @@ end @testset "reporting" begin lts = Configuration(julia=v"1.0.5") stable = Configuration(julia=v"1.2.0") - results = PkgEval.run([lts, stable], ["Example"]) + results = PkgEval.run([lts, stable], ["Example"]; update_registry=false) PkgEval.compare(results) end From b72b6b03624772523e060c089be8e211108709ca Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 17:10:52 +0200 Subject: [PATCH 05/19] Simplify Julia building, and refactor Configuration object. --- .github/workflows/ci.yml | 2 +- Manifest.toml | 551 ------------------------------------ Project.toml | 5 +- deps/Releases.toml | 32 --- deps/Versions.toml | 51 ---- src/PkgEval.jl | 38 +-- src/{run.jl => evaluate.jl} | 174 ++++++------ src/julia.jl | 537 ++++++++++++----------------------- src/registry.jl | 3 +- src/report.jl | 85 ------ test/runtests.jl | 47 ++- 11 files changed, 307 insertions(+), 1218 deletions(-) delete mode 100644 Manifest.toml delete mode 100644 deps/Releases.toml delete mode 100644 deps/Versions.toml rename src/{run.jl => evaluate.jl} (79%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecc800156..3daf37be5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - master # directly from Git, likely built env: JULIA_DEBUG: PkgEval - JULIA_SPEC: ${{ matrix.build_spec }} + JULIA: ${{ matrix.build_spec }} steps: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index 0c64ddf6d..000000000 --- a/Manifest.toml +++ /dev/null @@ -1,551 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -[[ArgParse]] -deps = ["Logging", "TextWrap"] -git-tree-sha1 = "3102bce13da501c9104df33549f511cd25264d7d" -uuid = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" -version = "1.1.4" - -[[ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" - -[[Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[AssetRegistry]] -deps = ["Distributed", "JSON", "Pidfile", "SHA", "Test"] -git-tree-sha1 = "b25e88db7944f98789130d7b503276bc34bc098e" -uuid = "bf4720bc-e11a-5d0c-854e-bdca1663c893" -version = "0.1.0" - -[[AutoHashEquals]] -git-tree-sha1 = "45bb6705d93be619b81451bb2006b7ee5d4e4453" -uuid = "15f4f7f2-30c1-5605-9d31-71845cf9641f" -version = "0.2.0" - -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[BinaryBuilder]] -deps = ["ArgParse", "BinaryBuilderBase", "Dates", "Downloads", "GitHub", "HTTP", "JLD2", "JSON", "LibGit2", "Libdl", "Logging", "LoggingExtras", "ObjectFile", "OutputCollectors", "Pkg", "PkgLicenses", "ProgressMeter", "REPL", "Random", "Registrator", "RegistryTools", "SHA", "Scratch", "Sockets", "UUIDs", "ghr_jll"] -git-tree-sha1 = "3055a2f80b7c5603d8d0da364c61eab915a7cfe7" -uuid = "12aac903-9f7c-5d81-afc2-d9565ea332ae" -version = "0.4.8" - -[[BinaryBuilderBase]] -deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "LibGit2_jll", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll", "pigz_jll"] -git-tree-sha1 = "cf7521d94ab88367140f7157e83aa2269884e8b4" -uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e" -version = "1.4.0" - -[[CodecZlib]] -deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" -uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.0" - -[[Compat]] -deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "44c37b4636bc54afac5c574d2d02b625349d6582" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.41.0" - -[[CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" - -[[Crayons]] -git-tree-sha1 = "b618084b49e78985ffa8422f32b9838e397b9fc2" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.1.0" - -[[DataAPI]] -git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.9.0" - -[[DataFrames]] -deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "cfdfef912b7f93e4b848e80b9befdf9e331bc05a" -uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.3.1" - -[[DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "3daef5523dd2e769dad2365274f760ff5f282c7d" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.11" - -[[DataValueInterfaces]] -git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" -uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" -version = "1.0.0" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[DelimitedFiles]] -deps = ["Mmap"] -uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" - -[[Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" - -[[ExprTools]] -git-tree-sha1 = "24565044e60bc48a7562e75bcf14f084901dc0b6" -uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.7" - -[[FileIO]] -deps = ["Pkg", "Requires", "UUIDs"] -git-tree-sha1 = "67551df041955cc6ee2ed098718c8fcd7fc7aebe" -uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.12.0" - -[[FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" - -[[Formatting]] -deps = ["Printf"] -git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" -uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" -version = "0.4.2" - -[[Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - -[[GitForge]] -deps = ["Dates", "HTTP", "JSON2"] -git-tree-sha1 = "0e6b8635059d6ac9e421825e75f8fca170f557d2" -uuid = "8f6bce27-0656-5410-875b-07a5572985df" -version = "0.1.7" - -[[GitHub]] -deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"] -git-tree-sha1 = "056781ae7b953289778408b136f8708a46837979" -uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" -version = "5.7.2" - -[[HTTP]] -deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] -git-tree-sha1 = "0fa77022fe4b511826b39c894c90daf5fce3334a" -uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "0.9.17" - -[[Hiccup]] -deps = ["MacroTools", "Test"] -git-tree-sha1 = "6187bb2d5fcbb2007c39e7ac53308b0d371124bd" -uuid = "9fb69e20-1954-56bb-a84f-559cc56a8ff7" -version = "0.2.2" - -[[IniFile]] -deps = ["Test"] -git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" -uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" -version = "0.5.0" - -[[InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[InvertedIndices]] -git-tree-sha1 = "bee5f1ef5bf65df56bdd2e40447590b272a5471f" -uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.1.0" - -[[IteratorInterfaceExtensions]] -git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" -uuid = "82899510-4779-5014-852e-03e436cf321d" -version = "1.0.0" - -[[JLD2]] -deps = ["DataStructures", "FileIO", "MacroTools", "Mmap", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"] -git-tree-sha1 = "09ef0c32a26f80b465d808a1ba1e85775a282c97" -uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.4.17" - -[[JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "22df5b96feef82434b07327e2d3c770a9b21e023" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.4.0" - -[[JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.2" - -[[JSON2]] -deps = ["Dates", "Parsers", "Test"] -git-tree-sha1 = "dcc3c2d9bdc036677a031ea97b76925983ca1f18" -uuid = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3" -version = "0.3.4" - -[[LazyArtifacts]] -deps = ["Artifacts", "Pkg"] -uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" - -[[LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" - -[[LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" - -[[LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" - -[[LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[LinearAlgebra]] -deps = ["Libdl", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[LoggingExtras]] -deps = ["Dates", "Logging"] -git-tree-sha1 = "dfeda1c1130990428720de0024d4516b1902ce98" -uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "0.4.7" - -[[MacroTools]] -deps = ["Markdown", "Random"] -git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.9" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[MbedTLS]] -deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"] -git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe" -uuid = "739be429-bea8-5141-9913-cc70e7f3736d" -version = "1.0.3" - -[[MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" - -[[Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.0.2" - -[[Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[Mocking]] -deps = ["Compat", "ExprTools"] -git-tree-sha1 = "29714d0a7a8083bba8427a4fbfb00a540c681ce7" -uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" -version = "0.7.3" - -[[MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" - -[[Mustache]] -deps = ["Printf", "Tables"] -git-tree-sha1 = "21d7a05c3b94bcf45af67beccab4f2a1f4a3c30a" -uuid = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" -version = "1.0.12" - -[[Mux]] -deps = ["AssetRegistry", "Base64", "HTTP", "Hiccup", "Pkg", "Sockets", "WebSockets"] -git-tree-sha1 = "82dfb2cead9895e10ee1b0ca37a01088456c4364" -uuid = "a975b10e-0019-58db-a62f-e48ff68538c9" -version = "0.7.6" - -[[NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" - -[[ObjectFile]] -deps = ["Reexport", "StructIO"] -git-tree-sha1 = "55ce61d43409b1fb0279d1781bf3b0f22c83ab3b" -uuid = "d8793406-e978-5875-9003-1fc021f44a92" -version = "0.3.7" - -[[OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" - -[[OrderedCollections]] -git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.4.1" - -[[OutputCollectors]] -git-tree-sha1 = "5d3f2b3b2e2a9d7d6f1774c78e94530ac7f360cc" -uuid = "6c11c7d4-943b-4e2b-80de-f2cfc2930a8c" -version = "0.1.1" - -[[Parsers]] -deps = ["Dates"] -git-tree-sha1 = "92f91ba9e5941fc781fecf5494ac1da87bdac775" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.2.0" - -[[Pidfile]] -deps = ["FileWatching", "Test"] -git-tree-sha1 = "1be8660b2064893cd2dae4bd004b589278e4440d" -uuid = "fa939f87-e72e-5be4-a000-7fc836dbe307" -version = "1.2.0" - -[[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[PkgLicenses]] -deps = ["Test"] -git-tree-sha1 = "0af826be249c6751a3e783c07b8cd3034f508943" -uuid = "fc669557-7ec9-5e45-bca9-462afbc28879" -version = "0.2.0" - -[[PooledArrays]] -deps = ["DataAPI", "Future"] -git-tree-sha1 = "db3a23166af8aebf4db5ef87ac5b00d36eb771e2" -uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "1.4.0" - -[[Preferences]] -deps = ["TOML"] -git-tree-sha1 = "2cf929d64681236a2e074ffafb8d568733d2e6af" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.3" - -[[PrettyTables]] -deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"] -git-tree-sha1 = "dfb54c4e414caa595a1f2ed759b160f5a3ddcba5" -uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "1.3.1" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[ProgressMeter]] -deps = ["Distributed", "Printf"] -git-tree-sha1 = "afadeba63d90ff223a6a48d2009434ecee2ec9e8" -uuid = "92933f4c-e287-5a05-a399-4b506db050ca" -version = "1.7.1" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["SHA", "Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[Registrator]] -deps = ["AutoHashEquals", "Base64", "Dates", "Distributed", "FileWatching", "GitForge", "GitHub", "HTTP", "JSON", "LibGit2", "Logging", "MbedTLS", "Mocking", "Mustache", "Mux", "Pkg", "RegistryTools", "Serialization", "Sockets", "TimeToLive", "UUIDs", "ZMQ"] -git-tree-sha1 = "8e1a5ac2695627143951512d700c7e3c445102ec" -uuid = "4418983a-e44d-11e8-3aec-9789530b3b3e" -version = "1.2.9" - -[[RegistryTools]] -deps = ["AutoHashEquals", "LibGit2", "Pkg", "SHA", "UUIDs"] -git-tree-sha1 = "7ba3c65d4f40a399e5a2f0ec5abc89de871604e5" -uuid = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4" -version = "1.7.0" - -[[Requires]] -deps = ["UUIDs"] -git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.0" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Sandbox]] -deps = ["LazyArtifacts", "Libdl", "Preferences", "Random", "SHA", "Scratch", "TOML", "Tar", "UserNSSandbox_jll"] -git-tree-sha1 = "980e4dbd6832606400cf2f1352940f07208af488" -uuid = "9307e30f-c43e-9ca7-d17c-c2dc59df670d" -version = "1.1.0" - -[[Scratch]] -deps = ["Dates"] -git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" -uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.1.0" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[SharedArrays]] -deps = ["Distributed", "Mmap", "Random", "Serialization"] -uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" - -[[SimpleBufferStream]] -git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" -uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" -version = "1.1.0" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[SodiumSeal]] -deps = ["Base64", "Libdl", "libsodium_jll"] -git-tree-sha1 = "80cef67d2953e33935b41c6ab0a178b9987b1c99" -uuid = "2133526b-2bfb-4018-ac12-889fb3908a75" -version = "0.1.1" - -[[SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.0.1" - -[[SparseArrays]] -deps = ["LinearAlgebra", "Random"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[StructIO]] -deps = ["Test"] -git-tree-sha1 = "010dc73c7146869c042b49adcdb6bf528c12e859" -uuid = "53d494c1-5632-5724-8f4c-31dff12d585f" -version = "0.3.0" - -[[TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" - -[[TableTraits]] -deps = ["IteratorInterfaceExtensions"] -git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" -uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" -version = "1.0.1" - -[[Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"] -git-tree-sha1 = "bb1064c9a84c52e277f1096cf41434b675cd368b" -uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.6.1" - -[[Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" - -[[Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[TextWrap]] -git-tree-sha1 = "9250ef9b01b66667380cf3275b3f7488d0e25faf" -uuid = "b718987f-49a8-5099-9789-dcd902bef87d" -version = "1.0.1" - -[[TimeToLive]] -deps = ["Dates"] -git-tree-sha1 = "1f1389007d16385ec02e497bef6c2caffba99b65" -uuid = "37f0c46e-897f-50ef-b453-b26c3eed3d6c" -version = "0.3.0" - -[[TranscodingStreams]] -deps = ["Random", "Test"] -git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.6" - -[[URIs]] -git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" -uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.3.0" - -[[UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[UserNSSandbox_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "3ed015214caa3c75c8fb4753d56c1efec5a71225" -uuid = "b88861f7-1d72-59dd-91e7-a8cc876a4984" -version = "2021.8.18+0" - -[[WebSockets]] -deps = ["Base64", "Dates", "HTTP", "Logging", "Sockets"] -git-tree-sha1 = "f91a602e25fe6b89afc93cf02a4ae18ee9384ce3" -uuid = "104b5d7c-a370-577a-8038-80a2059c5097" -version = "1.5.9" - -[[ZMQ]] -deps = ["FileWatching", "Sockets", "ZeroMQ_jll"] -git-tree-sha1 = "fc68e8a3719166950a0f3e390a14c7302c48f8de" -uuid = "c2297ded-f4af-51ae-bb23-16f91089e4e1" -version = "1.2.1" - -[[ZeroMQ_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "libsodium_jll"] -git-tree-sha1 = "fe5c65a526f066fb3000da137d5785d9649a8a47" -uuid = "8f1865be-045e-5c20-9c9f-bfbfb0764568" -version = "4.3.4+0" - -[[Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" - -[[ghr_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f5c8cb306d4fe2d1fff90443a088fc5ba536c134" -uuid = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" -version = "0.13.0+1" - -[[libblastrampoline_jll]] -deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" - -[[libsodium_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "848ab3d00fe39d6fbc2a8641048f8f272af1c51e" -uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" -version = "1.0.20+0" - -[[nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" - -[[p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" - -[[pigz_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] -git-tree-sha1 = "8c379a72c82099ceb4be53f4f427690376279052" -uuid = "1bc43ea1-30af-5bc8-a9d4-c018457e6e3e" -version = "2.5.0+0" diff --git a/Project.toml b/Project.toml index 5f72f7a25..7cbfab838 100644 --- a/Project.toml +++ b/Project.toml @@ -8,13 +8,14 @@ BinaryBuilder = "12aac903-9f7c-5d81-afc2-d9565ea332ae" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" -LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" Sandbox = "9307e30f-c43e-9ca7-d17c-c2dc59df670d" +Scratch = "6c6a2e73-6563-6170-7368-637461726353" [compat] BinaryBuilder = "0.4" diff --git a/deps/Releases.toml b/deps/Releases.toml deleted file mode 100644 index 6b6202b73..000000000 --- a/deps/Releases.toml +++ /dev/null @@ -1,32 +0,0 @@ -["1.0"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0-latest-linux-x86_64.tar.gz" - -["1.1"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.1/julia-1.1-latest-linux-x86_64.tar.gz" - -["1.2"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2-latest-linux-x86_64.tar.gz" - -["1.3"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.3/julia-1.3-latest-linux-x86_64.tar.gz" - -["1.4"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.4/julia-1.4-latest-linux-x86_64.tar.gz" - -["1.5"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5-latest-linux-x86_64.tar.gz" - -["1.6"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6-latest-linux-x86_64.tar.gz" - -["1.7"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.7/julia-1.7-latest-linux-x86_64.tar.gz" - -["lts"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.6-latest-linux-x86_64.tar.gz" - -["stable"] -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.7/julia-1.7-latest-linux-x86_64.tar.gz" - -["nightly"] -url = "https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz" diff --git a/deps/Versions.toml b/deps/Versions.toml deleted file mode 100644 index 4a4cad16c..000000000 --- a/deps/Versions.toml +++ /dev/null @@ -1,51 +0,0 @@ -["1.0.5"] -sha = "9dedd613777ba6ebd8aee5796915ff50aa6188ea03ed143cb687fc2aefd76b03" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.5-linux-x86_64.tar.gz" - -["1.2.0"] -sha = "926ced5dec5d726ed0d2919e849ff084a320882fb67ab048385849f9483afc47" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2.0-linux-x86_64.tar.gz" - -["1.3.0"] -sha = "9ec9e8076f65bef9ba1fb3c58037743c5abb3b53d845b827e44a37e7bcacffe8" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.3/julia-1.3.0-linux-x86_64.tar.gz" - -["1.3.1"] -sha = "faa707c8343780a6fe5eaf13490355e8190acf8e2c189b9e7ecbddb0fa2643ad" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.3/julia-1.3.1-linux-x86_64.tar.gz" - -["1.4.0"] -sha = "30d126dc3598f3cd0942de21cc38493658037ccc40eb0882b3b4c418770ca751" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.4/julia-1.4.0-linux-x86_64.tar.gz" - -["1.4.1"] -sha = "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.4/julia-1.4.1-linux-x86_64.tar.gz" - -["1.4.2"] -sha = "d77311be23260710e89700d0b1113eecf421d6cf31a9cebad3f6bdd606165c28" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.4/julia-1.4.2-linux-x86_64.tar.gz" - -["1.5.0"] -sha = "be7af676f8474afce098861275d28a0eb8a4ece3f83a11027e3554dcdecddb91" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-linux-x86_64.tar.gz" - -["1.5.1"] -sha = "f5d37cb7fe40e3a730f721da8f7be40310f133220220949939d8f892ce2e86e3" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.1-linux-x86_64.tar.gz" - -["1.5.2"] -sha = "6da704fadcefa39725503e4c7a9cfa1a570ba8a647c4bd8de69a118f43584630" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.2-linux-x86_64.tar.gz" - -["1.5.3"] -sha = "f190c938dd6fed97021953240523c9db448ec0a6760b574afd4e9924ab5615f1" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.3-linux-x86_64.tar.gz" - -["1.5.4"] -sha = "80dec351d1a593e8ad152636971a48d0c81bfcfab92c87f3604663616f1e8bc5" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.4-linux-x86_64.tar.gz" - -["1.6.0"] -sha = "463b71dc70ca7094c0e0fd6d55d130051a7901e8dec5eb44d6002c57d1bd8585" -url = "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6.0-linux-x86_64.tar.gz" diff --git a/src/PkgEval.jl b/src/PkgEval.jl index 91cb96d11..b559ff666 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -9,33 +9,39 @@ using DataFrames using Random using Sandbox using LazyArtifacts +using JSON -# immutable: in package directory -versions_file() = joinpath(dirname(@__DIR__), "deps", "Versions.toml") -releases_file() = joinpath(dirname(@__DIR__), "deps", "Releases.toml") -registries_file() = joinpath(dirname(@__DIR__), "deps", "Registries.toml") +import Scratch: @get_scratch! +download_dir = "" +storage_dir = "" -# mutable: in .cache directory -cache_dir() = joinpath(get(ENV, "XDG_CACHE_HOME", joinpath(homedir(), ".cache")), "PkgEval") -download_dir(name) = joinpath(cache_dir(), "downloads", name) -storage_dir() = joinpath(cache_dir(), "storage") -extra_versions_file() = joinpath(cache_dir(), "Versions.toml") +export Configuration -# fixed locations -registry_dir() = joinpath(first(DEPOT_PATH), "registries") -registry_dir(name) = joinpath(registry_dir(), name) +Base.@kwdef mutable struct Configuration + julia::String = "nightly" + compiled::Bool = false + buildflags::Vector{String} = String[] + depwarn::Bool = false + # TODO: put even more here (rootfs, install_dir, limits, etc) +end + +# behave as a scalar in broadcast expressions +Base.broadcastable(x::Configuration) = Ref(x) # utils -isdebug(group) = Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing +isdebug(group) = + Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing include("registry.jl") include("julia.jl") -include("run.jl") +include("evaluate.jl") include("report.jl") function __init__() - mkpath(cache_dir()) - touch(extra_versions_file()) + global download_dir = @get_scratch!("downloads") + mkpath(joinpath(download_dir, "srccache")) + + global storage_dir = @get_scratch!("storage") end end # module diff --git a/src/run.jl b/src/evaluate.jl similarity index 79% rename from src/run.jl rename to src/evaluate.jl index 8526a7086..5092fd493 100644 --- a/src/run.jl +++ b/src/evaluate.jl @@ -37,9 +37,9 @@ function prepare_rootfs(distro="debian"; uid=1000, user="pkgeval", gid=1000, gro end """ - run_sandboxed_julia(install::String, args=``; env=Dict(), mounts=Dict(), - wait=true, stdin=stdin, stdout=stdout, stderr=stderr, - install_dir="/opt/julia", kwargs...) + sandboxed_julia(config::Configuration, install::String, args=``; env=Dict(), mounts=Dict(), + wait=true, stdin=stdin, stdout=stdout, stderr=stderr, + install_dir="/opt/julia", kwargs...) Run Julia inside of a sandbox, passing the given arguments `args` to it. The argument `wait` determines if the process will be waited on. Streams can be connected using the `stdin`, @@ -49,14 +49,14 @@ Further customization is possible using the `env` arg, to set environment variab `mounts` argument to mount additional directories. With `install_dir`, the directory where Julia is installed can be chosen. """ -function run_sandboxed_julia(install::String, args=``; wait=true, kwargs...) - config, cmd = runner_sandboxed_julia(install, args; kwargs...) +function sandboxed_julia(config::Configuration, install::String, args=``; wait=true, kwargs...) + sandbox_config, cmd = sandboxed_julia_cmd(install, args; kwargs...) # XXX: even when preferred_executor() returns UnprivilegedUserNamespacesExecutor, # sometimes a stray sudo happens at run time? no idea how. exe_typ = UnprivilegedUserNamespacesExecutor exe = exe_typ() - proc = Base.run(exe, config, cmd; wait) + proc = run(exe, sandbox_config, cmd; wait) # TODO: introduce a --stats flag that has the sandbox trace and report on CPU, network, ... usage @@ -80,20 +80,19 @@ end const xvfb_lock = ReentrantLock() const xvfb_proc = Ref{Union{Base.Process,Nothing}}(nothing) -function runner_sandboxed_julia(install::String, args=``; install_dir="/opt/julia", - stdin=stdin, stdout=stdout, stderr=stderr, - env::Dict{String,String}=Dict{String,String}(), - mounts::Dict{String,String}=Dict{String,String}(), - xvfb::Bool=true, cpus::Vector{Int}=Int[], - sysimage=nothing, rootfs=prepare_rootfs()) - julia_path = installed_julia_dir(install) +function sandboxed_julia_cmd(install::String, args=``; install_dir="/opt/julia", + stdin=stdin, stdout=stdout, stderr=stderr, + env::Dict{String,String}=Dict{String,String}(), + mounts::Dict{String,String}=Dict{String,String}(), + xvfb::Bool=true, cpus::Vector{Int}=Int[], + sysimage=nothing, rootfs=prepare_rootfs()) read_only_maps = Dict( "/" => rootfs.path, - install_dir => julia_path, - "/usr/local/share/julia/registries" => registry_dir(), + install_dir => install, + "/usr/local/share/julia/registries" => joinpath(first(DEPOT_PATH), "registries"), ) - artifacts_path = joinpath(storage_dir(), "artifacts") + artifacts_path = joinpath(storage_dir, "artifacts") mkpath(artifacts_path) read_write_maps = merge(mounts, Dict( joinpath(rootfs.home, ".julia/artifacts") => artifacts_path @@ -121,7 +120,7 @@ function runner_sandboxed_julia(install::String, args=``; install_dir="/opt/juli if xvfb lock(xvfb_lock) do if xvfb_proc[] === nothing || !process_running(xvfb_proc[]) - proc = Base.run(`Xvfb :1 -screen 0 1024x768x16`; wait=false) + proc = run(`Xvfb :1 -screen 0 1024x768x16`; wait=false) sleep(1) process_running(proc) || error("Could not start Xvfb") @@ -149,15 +148,15 @@ function runner_sandboxed_julia(install::String, args=``; install_dir="/opt/juli # actual storage on the host, and not just the (1G hard-coded) tmpfs, # because some packages like to generate a lot of data during testing. - config = SandboxConfig(read_only_maps, read_write_maps, env; - rootfs.uid, rootfs.gid, pwd=rootfs.home, persist=true, - stdin, stdout, stderr, verbose=isdebug(:sandbox)) + sandbox_config = SandboxConfig(read_only_maps, read_write_maps, env; + rootfs.uid, rootfs.gid, pwd=rootfs.home, persist=true, + stdin, stdout, stderr, verbose=isdebug(:sandbox)) if sysimage !== nothing args = `--sysimage=$sysimage $args` end - return config, `$cmd $args` + return sandbox_config, `$cmd $args` end function process_children(pid) @@ -188,20 +187,20 @@ function cpu_time(pid) end """ - run_sandboxed_script(install::String, script::String, args=``; - log_limit=2^20, time_limit=60*60) + sandboxed_script(config::Configuration, install::String, script::String, args=``; + log_limit=2^20, time_limit=60*60) Run a Julia script `script` in non-interactive mode, returning the process status and a failure reason if any (both represented by a symbol), and the full log. Execution of the script will be forcibly interrupted after `time_limit` seconds (defaults to 1h) or if the log becomes larger than `log_limit` (defaults to 1MB). -Refer to `run_sandboxed_julia`[@ref] for more possible `keyword arguments. +Refer to `sandboxed_julia`[@ref] for more possible `keyword arguments. """ -function run_sandboxed_script(install::String, script::String, args=``; - log_limit = 2^20 #= 1 MB =#, - time_limit = 60*60, - kwargs...) +function sandboxed_script(config::Configuration, install::String, script::String, args=``; + log_limit = 2^20 #= 1 MB =#, + time_limit = 60*60, + kwargs...) @assert log_limit > 0 cmd = `--eval 'eval(Meta.parse(read(stdin,String)))' $args` @@ -218,9 +217,9 @@ function run_sandboxed_script(install::String, script::String, args=``; input = Pipe() output = Pipe() - proc = run_sandboxed_julia(install, cmd; env, wait=false, - stdout=output, stderr=output, stdin=input, - kwargs...) + proc = sandboxed_julia(config, install, cmd; env, wait=false, + stdout=output, stderr=output, stdin=input, + kwargs...) close(output.in) # pass the script over standard input to avoid exceeding max command line size, @@ -316,20 +315,18 @@ function run_sandboxed_script(install::String, script::String, args=``; end """ - run_sandboxed_test(install::String, pkg; do_depwarns=false) + sandboxed_test(config::Configuration, install::String, pkg; kwargs...) Run the unit tests for a single package `pkg` inside of a sandbox using a Julia installation -at `install`. If `do_depwarns` is `true`, deprecation warnings emitted while running the -package's tests will cause the tests to fail. Test will be forcibly interrupted after -`time_limit` seconds (defaults to 1h) or if the log becomes larger than `log_limit` -(defaults to 1MB). - -A log for the tests is written to a version-specific directory in the PkgEval root -directory. +at `install`. -Refer to `run_sandboxed_script`[@ref] for more possible `keyword arguments. +Refer to `sandboxed_script`[@ref] for more possible `keyword arguments. """ -function run_sandboxed_test(install::String, pkg; do_depwarns=false, kwargs...) +function sandboxed_test(config::Configuration, install::String, pkg; kwargs...) + if config.compiled + return compiled_test(config, install, pkg; kwargs...) + end + script = raw""" try using Dates @@ -361,11 +358,11 @@ function run_sandboxed_test(install::String, pkg; do_depwarns=false, kwargs...) end""" args = `$(pkg.name)` - if do_depwarns + if config.depwarn args = `--depwarn=error $args` end - status, reason, log = run_sandboxed_script(install, script, args; kwargs...) + status, reason, log = sandboxed_script(config, install, script, args; kwargs...) # pick up the installed package version from the log version_match = match(Regex("Installed $(pkg.name) .+ v(.+)"), log) @@ -428,17 +425,17 @@ function run_sandboxed_test(install::String, pkg; do_depwarns=false, kwargs...) end """ - run_compiled_test(install::String, pkg; compile_time_limit=30*60) + compiled_test(install::String, pkg; compile_time_limit=30*60) -Run the unit tests for a single package `pkg` (see `run_compiled_test`[@ref] for details and +Run the unit tests for a single package `pkg` (see `compiled_test`[@ref] for details and a list of supported keyword arguments), after first having compiled a system image that contains this package and its dependencies. To find incompatibilities, the compilation happens on an Ubuntu-based runner, while testing is performed in an Arch Linux container. """ -function run_compiled_test(install::String, pkg; log_limit = 2^20 #= 1 MB =#, - compile_time_limit=30*60, do_depwarns=false, kwargs...) +function compiled_test(config::Configuration, install::String, pkg; log_limit = 2^20 #= 1 MB =#, + compile_time_limit=30*60, do_depwarns=false, kwargs...) script = raw""" try using Dates @@ -480,8 +477,8 @@ function run_compiled_test(install::String, pkg; log_limit = 2^20 #= 1 MB =#, sysimage_dir = mktempdir() mounts = Dict(dirname(sysimage_path) => sysimage_dir) - status, reason, log = run_sandboxed_script(install, script, args; mounts, - time_limit=compile_time_limit, kwargs...) + status, reason, log = sandboxed_script(config, install, script, args; mounts, + time_limit=compile_time_limit, kwargs...) # try to figure out the failure reason if status === nothing @@ -500,36 +497,34 @@ function run_compiled_test(install::String, pkg; log_limit = 2^20 #= 1 MB =#, # run the tests in an alternate environment (different OS, depot and Julia binaries # in another path, etc) + test_config = Configuration( + julia = config.julia, + depwarn = config.depwarn, + compiled = false, + ) rootfs = prepare_rootfs("arch"; user="user", group="group") version, status, reason, test_log = - run_sandboxed_test(install, pkg; mounts, rootfs, do_depwarns, log_limit, - sysimage=sysimage_path, install_dir="/usr/local/julia", - kwargs...) + sandboxed_test(test_config, install, pkg; mounts, rootfs, log_limit, + sysimage=sysimage_path, install_dir="/usr/local/julia", + kwargs...) rm(sysimage_dir; recursive=true) return version, status, reason, log * "\n" * test_log end -Base.@kwdef struct Configuration - julia::VersionNumber = Base.VERSION - compiled::Bool = false - # TODO: depwarn, checkbounds, etc - # TODO: also move buildflags here? -end - -# behave as a scalar in broadcast expressions -Base.broadcastable(x::Configuration) = Ref(x) - -function run(configs::Vector{Configuration}, pkgs::Vector; - ninstances::Integer=Sys.CPU_THREADS, retries::Integer=2, - registry::String=DEFAULT_REGISTRY, kwargs...) +function evaluate(configs::Vector{Configuration}, pkgs::Vector; + ninstances::Integer=Sys.CPU_THREADS, retries::Integer=2, + registry::String=DEFAULT_REGISTRY, kwargs...) # here we deal with managing execution: spawning workers, output, result I/O, etc # Julia installation - instantiated_configs = Dict{Configuration,String}() + instantiated_configs = Dict() for config in configs - install = prepare_julia(config.julia) - instantiated_configs[config] = install + install = prepare_julia(config) + # XXX: better get the version from a file in the tree + version_str = chomp(read(`$install/bin/julia --startup-file=no --eval "println(VERSION)"`, String)) + version = parse(VersionNumber, version_str) + instantiated_configs[config] = (install, version) end jobs = vec(collect(Iterators.product(instantiated_configs, pkgs))) @@ -619,7 +614,8 @@ function run(configs::Vector{Configuration}, pkgs::Vector; end # NOTE: we expand the Configuration into separate columns - result = DataFrame(julia = VersionNumber[], + result = DataFrame(julia_spec = String[], + julia_version = VersionNumber[], compiled = Bool[], name = String[], uuid = UUID[], @@ -649,7 +645,7 @@ function run(configs::Vector{Configuration}, pkgs::Vector; push!(all_workers, @async begin try while !isempty(jobs) && !done - (config, install), pkg = pop!(jobs) + (config, (julia_install, julia_version)), pkg = pop!(jobs) times[i] = now() running[i] = (config, pkg) @@ -658,33 +654,31 @@ function run(configs::Vector{Configuration}, pkgs::Vector; pkg_compat = Registry.compat_info(pkg_info) supported = any(pkg_compat) do (version, compat) !haskey(compat, Registry.JULIA_UUID) || - config.julia ∈ compat[Registry.JULIA_UUID] + julia_version ∈ compat[Registry.JULIA_UUID] end if !supported - push!(result, [config.julia, config.compiled, + push!(result, [config.julia, julia_version, config.compiled, pkg.name, pkg.uuid, missing, :skip, :unsupported, 0, missing]) running[i] = nothing continue elseif pkg.name in skip_lists[registry] - push!(result, [config.julia, config.compiled, + push!(result, [config.julia, julia_version, config.compiled, pkg.name, pkg.uuid, missing, :skip, :explicit, 0, missing]) running[i] = nothing continue elseif endswith(pkg.name, "_jll") - push!(result, [config.julia, config.compiled, + push!(result, [config.julia, julia_version, config.compiled, pkg.name, pkg.uuid, missing, :skip, :jll, 0, missing]) running[i] = nothing continue end - runner = config.compiled ? run_compiled_test : run_sandboxed_test - # perform an initial run pkg_version, status, reason, log = - runner(install, pkg; cpus=[i-1], kwargs...) + sandboxed_test(config, julia_install, pkg; cpus=[i-1], kwargs...) # certain packages are known to have flaky tests; retry them for j in 1:retries @@ -692,12 +686,12 @@ function run(configs::Vector{Configuration}, pkgs::Vector; pkg.name in retry_lists[registry] times[i] = now() pkg_version, status, reason, log = - runner(install, pkg; cpus=[i-1], kwargs...) + sandboxed_test(julia_install, pkg; cpus=[i-1], kwargs...) end end duration = (now()-times[i]) / Millisecond(1000) - push!(result, [config.julia, config.compiled, + push!(result, [config.julia, julia_version, config.compiled, pkg.name, pkg.uuid, pkg_version, status, reason, duration, log]) running[i] = nothing @@ -716,7 +710,7 @@ function run(configs::Vector{Configuration}, pkgs::Vector; println() # clean-up - for (config, install) in instantiated_configs + for (config, (install,version)) in instantiated_configs rm(install; recursive=true) end end @@ -725,27 +719,21 @@ function run(configs::Vector{Configuration}, pkgs::Vector; end """ - run(configs::Vector{Configuration}=[Configuration()], - pkg_names::Vector{String}=[]]; registry=General, update_registry=true, kwargs...) + evaluate(configs::Vector{Configuration}=[Configuration()], + pkg_names::Vector{String}=[]]; registry=General, update_registry=true, kwargs...) Run all tests for all packages in the registry `registry`, or only for the packages as identified by their name in `pkgnames`, using the configurations from `configs`. The registry is first updated if `update_registry` is set to true. -Refer to `run_sandboxed_test`[@ref] and `run_sandboxed_julia`[@ref] for more possible +Refer to `sandboxed_test`[@ref] and `sandboxed_julia`[@ref] for more possible keyword arguments. """ -function run(configs::Vector{Configuration}=[Configuration()], - pkg_names::Vector{String}=String[]; - registry::String=DEFAULT_REGISTRY, update_registry::Bool=true, kwargs...) +function evaluate(configs::Vector{Configuration}=[Configuration()], + pkg_names::Vector{String}=String[]; + registry::String=DEFAULT_REGISTRY, update_registry::Bool=true, kwargs...) prepare_registry(registry; update=update_registry) pkgs = read_pkgs(pkg_names) - run(configs, pkgs; registry, kwargs...) + evaluate(configs, pkgs; registry, kwargs...) end - -# for backwards compatibility -run(julia_versions::Vector{VersionNumber}, args...; kwargs...) = - run([Configuration(julia=julia_version) for julia_version in julia_versions], args...; - kwargs...) -prepare_runner() = return diff --git a/src/julia.jl b/src/julia.jl index 8edb917ee..56eee790f 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -1,373 +1,186 @@ -using BinaryBuilder -using Downloads -using LibGit2 -import SHA: sha256 - - -# -# Utilities -# - -function purge() - # remove old builds - vers = TOML.parse(read(extra_versions_file(), String)) - rm(extra_versions_file()) - open(extra_versions_file(); append=true) do io - for (ver, data) in vers - @assert haskey(data, "file") - @assert !haskey(data, "url") - - path = download_dir(data["file"]) - if round(now() - unix2datetime(mtime(path)), Dates.Week) < Dates.Week(1) - println(io, "[\"$ver\"]") - for (key, value) in data - println(io, "$key = $(repr(value))") - end - println(io) - else - rm(path) - rm(path * ".sha256"; force=true) - end - end - end -end - -function hash_file(path) - open(path, "r") do f - bytes2hex(sha256(f)) - end -end - -function installed_julia_dir(jp) - jp_contents = readdir(jp) - # Allow the unpacked directory to either be insider another directory (as produced by - # the buildbots) or directly inside the mapped directory (as produced by the BB script) - if length(jp_contents) == 1 - jp = joinpath(jp, first(jp_contents)) - end - jp -end - - -# -# Versions -# - -# a version is identified by a specific unique version, includes a hash to verify, and -# points to a local file or a remove resource. - -read_versions() = merge(TOML.parse(read(versions_file(), String)), - TOML.parse(read(extra_versions_file(), String))) - -""" - prepare_julia(the_ver, dir=mktempdir()) - -Download and extract the specified version of Julia using the information provided in -`Versions.toml` to the directory `dir`. -""" -function prepare_julia(the_ver::VersionNumber, dir::String=mktempdir()) - vers = read_versions() - for (ver, data) in vers - ver == string(the_ver) || continue - if haskey(data, "url") - url = data["url"] - - file = get(data, "file", "julia-$ver.tar.gz") - @assert !isabspath(file) - file = download_dir(file) - mkpath(dirname(file)) - - Pkg.PlatformEngines.download_verify_unpack(url, data["sha"], dir; - tarball_path=file, force=true, - ignore_existence=true) +import Downloads +import JSON +import Pkg +import Git: git +import BinaryBuilder: DirectorySource, ExecutableProduct, build_tarballs + +const VERSIONS_URL = "https://julialang-s3.julialang.org/bin/versions.json" + +function get_julia_release(spec::String) + dirpath = mktempdir() + if spec == "nightly" + @debug "Downloading nightly build..." + + url = if Sys.islinux() && Sys.ARCH == :x86_64 + "https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz" + elseif Sys.islinux() && Sys.ARCH == :i686 + "https://julialangnightlies-s3.julialang.org/bin/linux/x86/julia-latest-linux32.tar.gz" + elseif Sys.islinux() && Sys.ARCH == :aarch64 + "https://julialangnightlies-s3.julialang.org/bin/linux/aarch64/julia-latest-linuxaarch64.tar.gz" else - file = data["file"] - !isabspath(file) && (file = download_dir(file)) - Pkg.PlatformEngines.verify(file, data["sha"]) - Pkg.PlatformEngines.unpack(file, dir) + error("Don't know how to get nightly build for $(Sys.MACHINE)") end - # make sure the Julia installation is usable by the container user - Base.run(`chmod -R o=u $dir`) - - return dir - end - error("Requested Julia version $the_ver not found") -end + # download and extract to a temporary directory, but don't keep the tarball + Pkg.PlatformEngines.download_verify_unpack(url, nothing, dirpath; + ignore_existence=true) + else + @debug "Checking if '$spec' is an official release..." -function obtain_julia(spec::String)::VersionNumber - try - # maybe it's a named release - # NOTE: check this first, because "1.3" could otherwise be interpreted as v"1.3.0" - obtain_julia_release(spec) - catch - try - # maybe it already refers to a version in Versions.toml - version = VersionNumber(spec) - prepare_julia(version) - version - catch - # assume it points to something in our Git repository - obtain_julia_build(spec) + # the spec needs to be a valid version number + version_spec = tryparse(VersionNumber, spec) + if isnothing(version_spec) + @warn "Not a valid release name" + return nothing end - end -end - -# -# Releases -# - -# a release is identified by name, points to an online resource, but does not have a version -# number. it will be downloaded, probed, and added to Versions.toml. - -read_releases() = TOML.parsefile(releases_file()) + # download the versions.json and look up the version + versions = JSON.parse(sprint(io->Downloads.download(VERSIONS_URL, io))) + if !haskey(versions, string(version_spec)) + @warn "Unknown release name '$spec'" + return nothing + end + files = versions[string(version_spec)]["files"] -# get the Julia version and short hash from a Julia installation -function get_julia_version(base) - version = VersionNumber(read(`$(installed_julia_dir(base))/bin/julia -e 'print(Base.VERSION_STRING)'`, String)) - if version.prerelease != () - shorthash = read(`$(installed_julia_dir(base))/bin/julia -e 'print(Base.GIT_VERSION_INFO.commit_short)'`, String) - if endswith(shorthash, '*') - # strip the dirty marker - shorthash = shorthash[1:end-1] + # find a file entry for our machine + i = findfirst(files) do file + file["triplet"] == Sys.MACHINE end - version = VersionNumber(string(version) * string("-", shorthash)) + if isnothing(i) + @error "Release unavailable for $(Sys.MACHINE)" + return nothing + end + file = files[i] + + # download and extract to a temporary directory + filename = basename(file["url"]) + filepath = joinpath(download_dir, filename) + Pkg.PlatformEngines.download_verify_unpack(file["url"], file["sha256"], dirpath; + tarball_path=filepath, force=true, + ignore_existence=true) end - return version + return dirpath end -""" - version = obtain_julia_release(name::String) - -Download Julia from an on-line source listed in Releases.toml as identified by `name`. -Returns the `version` (what other functions use to identify this build). -This version will be added to Versions.toml. -""" -function obtain_julia_release(name::String) - releases = read_releases() - @assert haskey(releases, name) "Julia release $name is not registered in Releases.toml" - data = releases[name] - - # get the filename and extension from the url - url = data["url"] - filename = basename(url) - if endswith(filename, ".tar.gz") - ext = ".tar.gz" - base = filename[1:end-7] +function get_julia_repo(spec) + # split the spec into the repository and commit (e.g. `maleadt/julia#master`) + parts = split(spec, '#') + repo_spec, commit_spec = if length(parts) == 2 + parts + elseif length(parts) == 1 + "JuliaLang/julia", spec else - base, ext = splitext(filename) - end - - # download - filepath = download_dir(filename) - duplicates = 0 - while ispath(filepath) - duplicates += 1 - filepath = download_dir("$(base).$(duplicates)$(ext)") + error("Invalid repository/commit specification $spec") + return nothing end - mkpath(dirname(filepath)) - Downloads.download(url, filepath) - # get version - version = mktempdir() do install - Pkg.PlatformEngines.unpack(filepath, install) - get_julia_version(install) - end - - versions = read_versions() - if haskey(versions, string(version)) - @info "Julia $name (version $version) already available" - rm(filepath) + # perform a bare check-out of the repo + # TODO: check-out different repositories into a single bare check-out under different + # remotes? most objects are going to be shared, so this would save time and space. + @debug "Cloning/updating $repo_spec..." + bare_checkout = joinpath(download_dir, repo_spec) + if ispath(joinpath(bare_checkout, "config")) + run(`$(git()) -C $bare_checkout fetch --quiet`) else - # always use the hash of the downloaded file to force a check during `prepare_julia` - filehash = hash_file(filepath) - - # rename to include the version - filename = "julia-$version$ext" - if ispath(download_dir(filename)) - @warn "Destination file $filename already exists, assuming it matches" - rm(filepath) - else - mv(filepath, download_dir(filename)) - end - - # Update Versions.toml - version_stanza = """ - ["$version"] - file = "$filename" - sha = "$filehash" - """ - open(extra_versions_file(); append=true) do f - println(f, version_stanza) - end + run(`$(git()) clone --quiet --bare https://github.com/$(repo_spec).git $bare_checkout`) end - return version + # check-out the actual source code into a temporary directory + julia_checkout = mktempdir() + run(`$(git()) clone --quiet $bare_checkout $julia_checkout`) + run(`$(git()) -C $julia_checkout checkout --quiet $commit_spec`) + return julia_checkout end +function get_repo_details(repo) + version = VersionNumber(read(joinpath(repo, "VERSION"), String)) + hash = chomp(read(`$(git()) -C $repo rev-parse --verify HEAD`, String)) + shorthash = hash[1:10] + return (; version, hash, shorthash) +end -# -# Builds -# - -# get a repository handle, cloning or updating the repository along the way -function get_repo(name) - repo_path = download_dir(name) - if !isdir(repo_path) - @debug "Cloning $name to $repo_path..." - repo = LibGit2.clone("https://github.com/$name", repo_path) +function get_julia_build(repo) + @debug "Trying to download a Julia build..." + repo_details = get_repo_details(repo) + if Sys.MACHINE == "x86_64-linux-gnu" + url = "https://julialangnightlies.s3.amazonaws.com/bin/linux/x64/$(repo_details.version.major).$(repo_details.version.minor)/julia-$(repo_details.shorthash)-linux64.tar.gz" else - repo = LibGit2.GitRepo(repo_path) - LibGit2.fetch(repo) - - # prune to get rid of nonexisting branches (like the pull/PR/merge ones) - remote = LibGit2.get(LibGit2.GitRemote, repo, "origin") - fo = LibGit2.FetchOptions(prune=true) - LibGit2.fetch(remote, String[]; options=fo) + @debug "Don't know how to get build for $(Sys.MACHINE)" + return nothing end - return repo -end - -# look up a refspec in a git repository -function get_repo_commit(repo::LibGit2.GitRepo, spec) + # download and extract to a temporary directory + filename = basename(url) + filepath = joinpath(download_dir, filename) + dirpath = mktempdir() try - # maybe it's a remote branch - remote = LibGit2.get(LibGit2.GitRemote, repo, "origin") - LibGit2.GitCommit(repo, "refs/remotes/origin/$spec") + Pkg.PlatformEngines.download_verify_unpack(url, nothing, dirpath; + tarball_path=filepath, force=true, + ignore_existence=true) + return dirpath catch err - isa(err, LibGit2.GitError) || rethrow() - try - # maybe it's a version tag - LibGit2.GitCommit(repo, "refs/tags/v$spec") - catch err - isa(err, LibGit2.GitError) || rethrow() - try - # maybe it's a remote ref, and we need to fetch it first - remote = LibGit2.get(LibGit2.GitRemote, repo, "origin") - LibGit2.fetch(remote, ["+refs/$spec:refs/remotes/origin/$spec"]) - LibGit2.GitCommit(repo, "refs/remotes/origin/$spec") - catch err - isa(err, LibGit2.GitError) || rethrow() - - # give up and assume it's a commit or something - LibGit2.GitCommit(repo, spec) - end - end - end -end - -# get the Julia version and short hash corresponding with a refspec in a Julia repo -function get_julia_repoversion(spec, repo_name) - # get the Julia repo - repo = get_repo(repo_name) - commit = get_repo_commit(repo, spec) - - # lookup the version number and commit hash - tree = LibGit2.peel(LibGit2.GitTree, commit) - version = VersionNumber(chomp(LibGit2.content(tree["VERSION"]))) - hash = LibGit2.GitHash(commit) - # FIXME: no way to get the short hash with LibGit2? It just uses the length argument. - #shorthash = LibGit2.GitShortHash(hash, 7) - shorthash = LibGit2.GitShortHash(chomp(read(`git -C $(download_dir(repo_name)) rev-parse --short $hash`, String))) - # XXX: Julia's buildbot just takes the first 10 characters of the hash. - # we need to match this behavior exactly, because it's been observed that - # on different systems/checkouts/git versions the short hash can differ. - shorthash = LibGit2.GitShortHash(chomp(read(`git -C $(download_dir(repo_name)) rev-parse $hash`, String))[1:10]) - version = VersionNumber(string(version) * "-" * string(shorthash)) - # NOTE: we append the hash to differentiate commits with identical VERSION files - # (we can't only do this for -DEV versions because of backport branches) - - return version, string(hash), string(shorthash) -end - -function obtain_julia_build(spec::String="master", repo_name::String="JuliaLang/julia") - version, hash, shorthash = get_julia_repoversion(spec, repo_name) - versions = read_versions() - if haskey(versions, string(version)) - return version - end - - # try downloading it from the build bots - url = "https://julialangnightlies.s3.amazonaws.com/bin/linux/x64/$(version.major).$(version.minor)/julia-$(shorthash)-linux64.tar.gz" - try - filename = basename(url) - filepath = download_dir(filename) - if ispath(filepath) - @warn "Destination file $filename already exists, assuming it matches" - else - Downloads.download(url, filepath) - end - filehash = hash_file(filepath) - - # Update Versions.toml - version_stanza = """ - ["$version"] - file = "$filename" - sha = "$filehash" - """ - open(extra_versions_file(); append=true) do f - println(f, version_stanza) - end - - return version - catch ex - # if this was a download failure, proceed to build Julia ourselves - isa(ex, Downloads.RequestError) || rethrow() - bt = catch_backtrace() - @error "Could not download Julia $spec (version $version), performing a build" exception=(ex,bt) - perform_julia_build(spec, repo_name) + @debug "Could not download build" exception=(err, catch_backtrace()) + return nothing end end """ version = perform_julia_build(spec::String="master"; - binarybuilder_args::Vector{String}=String[] - buildflags::Vector{String}=String[]) + flags::Vector{String}=String[]) Check-out and build Julia at git reference `spec` using BinaryBuilder. Returns the `version` (what other functions use to identify this build). This version will be added to Versions.toml. """ -function perform_julia_build(spec::String="master", repo_name::String="JuliaLang/julia"; - binarybuilder_args::Vector{String}=String[], - buildflags::Vector{String}=String[]) - version, hash, shorthash = get_julia_repoversion(spec, repo_name) - if !isempty(buildflags) - version = VersionNumber(version.major, version.minor, version.patch, - (version.prerelease..., - "build-$(string(Base.hash(buildflags), base=16))")) - end - versions = read_versions() - if haskey(versions, string(version)) - return version - end +function build_julia(repo_path::String; + flags::Vector{String}=String[]) + repo_details = get_repo_details(repo_path) + println("Building Julia...") + + # NOTE: for simplicity, we don't cache the build (we'd need to include the flags, etc) # Define a Make.user - bundled = mktempdir() - open("$bundled/Make.user", "w") do io - println(io, "JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)") - for buildflag in buildflags - println(io, buildflag) + open("$repo_path/Make.user", "w") do io + cpu_target = if Sys.ARCH == :x86_64 + "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" + elseif Sys.ARCH == :i686 + "pentium4;sandybridge,-xsaveopt,clone_all" + elseif Sys.ARCH == :armv7l + "armv7-a;armv7-a,neon;armv7-a,neon,vfp4" + elseif Sys.ARCH == :aarch64 + "generic;cortex-a57;thunderx2t99;carmel" + elseif Sys.ARCH == :powerpc64le + "pwr8" + else + @warn "Cannot determine JULIA_CPU_TARGET for unknown architecture $(Sys.ARCH)" + nothing + end + if cpu_target !== nothing + println(io, "JULIA_CPU_TARGET=$cpu_target") + end + + for flag in flags + println(io, flag) end end # Collection of sources required to build julia - # NOTE: we need to check out Julia ourselves, because BinaryBuilder does not know how to - # fetch and checkout remote refs. - repo = get_repo(repo_name) - LibGit2.checkout!(repo, hash) - repo_path = download_dir(repo_name) sources = [ DirectorySource(repo_path), - DirectorySource(bundled), ] - # Pre-populate the srccache + # Pre-populate the srccache and save the downloaded files + srccache = joinpath(download_dir, "srccache") + repo_srccache = joinpath(repo_path, "deps", "srccache") + cp(srccache, repo_srccache) cd(repo_path) do Base.run(ignorestatus(`make -C deps getall NO_GIT=1`)) end + for file in readdir(repo_srccache) + if !ispath(joinpath(srccache, file)) + cp(joinpath(repo_srccache, file), joinpath(srccache, file)) + end + end # Bash recipe for building across all platforms script = raw""" @@ -391,54 +204,62 @@ function perform_julia_build(spec::String="master", repo_name::String="JuliaLang # These are the platforms we will build for by default, unless further # platforms are passed in on the command line platforms = [ - Platform("x86_64", "linux") + Base.BinaryPlatforms.HostPlatform() ] # The products that we will ensure are always built - products = Product[ + products = [ ExecutableProduct("julia", :julia) ] # Dependencies that must be installed before this package can be built dependencies = [] - # Build the tarballs - product_hashes = cd(joinpath(@__DIR__, "..", "deps")) do - build_tarballs(binarybuilder_args, "julia", version, sources, script, platforms, - products, dependencies, preferred_gcc_version=v"7", skip_audit=true, - verbose=isdebug(:binarybuilder)) - end - filepath, filehash = product_hashes[platforms[1]] - filename = basename(filepath) - if endswith(filename, ".tar.gz") - ext = ".tar.gz" - base = filename[1:end-7] - else - base, ext = splitext(filename) + # Build the tarballs and extract to a temporary directory + install_dir = mktempdir() + mktempdir() do dir + product_hashes = cd(dir) do + build_tarballs(binarybuilder_args, "julia", repo_details.version, sources, + script, platforms, products, dependencies, + preferred_gcc_version=v"7", skip_audit=true, + verbose=isdebug(:binarybuilder)) + end + filepath, _ = product_hashes[platforms[1]] + Pkg.unpack(filepath, install_dir) end - # Copy the generated tarball to the downloads folder - new_filepath = download_dir(filename) - duplicates = 0 - while ispath(new_filepath) - duplicates += 1 - new_filepath = download_dir("$(base).$(duplicates)$(ext)") - end - mv(filepath, download_dir(filename)) - - # Update Versions.toml - version_stanza = """ - ["$version"] - file = "$filename" - sha = "$filehash" - """ - open(extra_versions_file(); append=true) do f - println(f, version_stanza) + return install_dir +end + +function prepare_julia(config::Configuration) + if isempty(config.buildflags) + # check if it's an official release + dir = get_julia_release(config.julia) + if dir !== nothing + return joinpath(dir, only(readdir(dir))) + end end - # clean-up - rm(joinpath(dirname(@__DIR__), "deps", "build"); recursive=true, force=true) - rm(joinpath(dirname(@__DIR__), "deps", "products"); recursive=true, force=true) + # try to resolve to a Julia repository and hash + repo = get_julia_repo(config.julia) + if repo === nothing + error("Could not check-out Julia repository for $(config.julia)") + end + try + repo_details = get_repo_details(repo) + @debug "Julia $config.julia resolved to v$(repo_details.version), Git SHA $(repo_details.hash)" + + if isempty(config.buildflags) + # see if we can download a build + dir = get_julia_build(repo) + if dir !== nothing + return joinpath(dir, only(readdir(dir))) + end + end - return version + # perform a build + build_julia(repo; flags=config.buildflags) + finally + rm(repo; recursive=true) + end end diff --git a/src/registry.jl b/src/registry.jl index 967e30209..1d389d2f7 100644 --- a/src/registry.jl +++ b/src/registry.jl @@ -3,6 +3,7 @@ const DEFAULT_REGISTRY = "General" const skip_lists = Dict{String,Vector{String}}() const retry_lists = Dict{String,Vector{String}}() +registries_file() = joinpath(dirname(@__DIR__), "deps", "Registries.toml") read_registries() = TOML.parsefile(registries_file()) """ @@ -42,7 +43,7 @@ function read_pkgs(pkg_names::Vector{String}=String[]; registry=DEFAULT_REGISTRY want_all = isempty(pkg_names) pkg_data = [] - registry_instance = only(filter(ri->ri.name == "General", + registry_instance = only(filter(ri->ri.name == registry, Pkg.Registry.reachable_registries())) for (uuid, pkg) in registry_instance if !want_all diff --git a/src/report.jl b/src/report.jl index 8f6b63979..3333b202f 100644 --- a/src/report.jl +++ b/src/report.jl @@ -32,88 +32,3 @@ const reasons = Dict( :log_limit => "test log exceeded the size limit", :inactivity => "tests became inactive", ) - - -# simple comparison of two versions - -print_status(status, val=status) = print_status(stdout, status, val) - -function print_status(io::IO, status, val=status) - if status == :ok - printstyled(io, val; color = :green) - elseif status == :fail || status == :kill - printstyled(io, val; color = Base.error_color()) - elseif status == :skip - printstyled(io, val; color = Base.warn_color()) - else - error("Unknown status $status") - end -end - -function compare(result; rev::Bool=false) - pkg_names = unique(result.name) - - primary, against = groupby(result, [:julia]) - if rev # hack - primary, against = against, primary - end - - # overview - for df in (primary, against) - o = count(==(:ok), df[!, :status]) - s = count(==(:skip), df[!, :status]) - f = count(==(:fail), df[!, :status]) - k = count(==(:kill), df[!, :status]) - x = o + s + k + f - @assert x == nrow(df) - - print("On v$(first(df.julia)), out of $x packages ") - print_status(:ok, o) - print(" passed, ") - print_status(:fail, f) - print(" failed, ") - print_status(:kill, k) - print(" got killed and ") - print_status(:skip, s) - println(" were skipped.") - end - - println() - - # list of differences - println("Comparing $(first(primary.julia)) against $(first(against.julia)):") - new_failures = 0 - new_successes = 0 - for current in eachrow(primary) - pkg_name = current[:name] - - previous = against[against[!, :name] .== pkg_name, :] - nrow(previous) == 0 && continue - previous = first(previous) - - if current[:status] != previous[:status] - print("- $pkg_name status was $(previous[:status])") - ismissing(previous[:reason]) || print(" (reason: $(previous[:reason]))") - print(", now ") - print_status(current[:status]) - ismissing(current[:reason]) || print(" (reason: $(current[:reason]))") - println() - if current.status == :fail || current.status == :kill - new_failures += 1 - elseif current.status == :ok - new_successes += 1 - end - end - end - - println() - - # summary of differences - print("In summary, ") - print_status(:ok, new_successes) - print(" packages now succeed, while ") - print_status(:fail, new_failures) - println(" have started to fail.") - - return -end diff --git a/test/runtests.jl b/test/runtests.jl index ab5d48731..ca529f666 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,17 +1,18 @@ using PkgEval using Test -# determine which Julia to use -const julia_spec = get(ENV, "JULIA_SPEC", string(VERSION)) -@info "Testing with Julia $julia_spec" -const julia_version = PkgEval.obtain_julia(julia_spec)::VersionNumber -@info "Resolved to Julia v$julia_version" -const julia_install = PkgEval.prepare_julia(julia_version) +pkgnames = ["TimerOutputs", "Crayons", "Example", "Gtk"] + +julia = get(ENV, "JULIA", string(VERSION)) +@testset "PkgEval using Julia $julia" begin @testset "sandbox" begin + config = Configuration(; julia) + install = PkgEval.prepare_julia(config) + mktemp() do path, io try - PkgEval.run_sandboxed_julia(julia_install, `-e 'print(1337)'`; stdout=io) + PkgEval.sandboxed_julia(config, install, `-e 'print(1337)'`; stdout=io) close(io) @test read(path, String) == "1337" catch @@ -23,27 +24,25 @@ const julia_install = PkgEval.prepare_julia(julia_version) end # print versioninfo so we can verify in CI logs that the correct version is used - PkgEval.run_sandboxed_julia(julia_install, `-e 'using InteractiveUtils; versioninfo()'`) + PkgEval.sandboxed_julia(config, install, `-e 'using InteractiveUtils; versioninfo()'`) end -const pkgnames = ["TimerOutputs", "Crayons", "Example", "Gtk"] - @testset "time and output limits" begin # timeouts - results = PkgEval.run([Configuration(julia=julia_version)], pkgnames; - time_limit=0.1, update_registry=false) + results = PkgEval.evaluate([Configuration(; julia)], pkgnames; + time_limit=0.1, update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :time_limit) # log limit - results = PkgEval.run([Configuration(julia=julia_version)], pkgnames; - log_limit=1, update_registry=false) + results = PkgEval.evaluate([Configuration(; julia)], pkgnames; + log_limit=1, update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :log_limit) end @testset "main entrypoint" begin - results = PkgEval.run([Configuration(julia=julia_version)], pkgnames; - update_registry=false) - if !(julia_spec == "master" || julia_spec == "nightly") + results = PkgEval.evaluate([Configuration(; julia)], pkgnames; + update_registry=false) + if !(julia == "master" || julia == "nightly") @test all(results.status .== :ok) for result in eachrow(results) @test occursin("Testing $(result.name) tests passed", result.log) @@ -52,9 +51,9 @@ end end @testset "PackageCompiler" begin - results = PkgEval.run([Configuration(julia=julia_version, compiled=true)], ["Example"]; - update_registry=false) - if !(julia_spec == "master" || julia_spec == "nightly") + results = PkgEval.evaluate([Configuration(; julia, compiled=true)], ["Example"]; + update_registry=false) + if !(julia == "master" || julia == "nightly") @test all(results.status .== :ok) for result in eachrow(results) @test occursin("Testing $(result.name) tests passed", result.log) @@ -62,12 +61,4 @@ end end end -@testset "reporting" begin - lts = Configuration(julia=v"1.0.5") - stable = Configuration(julia=v"1.2.0") - results = PkgEval.run([lts, stable], ["Example"]; update_registry=false) - PkgEval.compare(results) end - -PkgEval.purge() -rm(julia_install; recursive=true) From 08791950f7549b8c48c62135ab7d14d0f930422e Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 18:11:10 +0200 Subject: [PATCH 06/19] Put more in the Configuration object, and compatibility with latest Sandbox.jl. --- src/PkgEval.jl | 23 +++++++++++++++++++++ src/evaluate.jl | 54 +++++++++++++++++++++---------------------------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/PkgEval.jl b/src/PkgEval.jl index b559ff666..e00b6bfd4 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -23,6 +23,29 @@ Base.@kwdef mutable struct Configuration buildflags::Vector{String} = String[] depwarn::Bool = false # TODO: put even more here (rootfs, install_dir, limits, etc) + + # the directory where Julia is installed in the run-time environment + julia_install_dir::String = "/opt/julia" + + # whether to launch Xvfb before starting Julia + xvfb::Bool = true + + # a list of CPUs to restrict the Julia process to (or empty if unconstrained). + # if set, JULIA_CPU_THREADS will also be set to a number equaling the number of CPUs. + cpus::Vector{Int} = Int[] + + # additional Julia arguments to pass to the process + julia_args::Cmd = `` +end + +# copy constructor that allows overriding specific fields +function Configuration(cfg::Configuration; kwargs...) + kwargs = Dict(kwargs...) + merged_kwargs = Dict() + for field in fieldnames(Configuration) + merged_kwargs[field] = get(kwargs, field, getfield(cfg, field)) + end + Configuration(; merged_kwargs...) end # behave as a scalar in broadcast expressions diff --git a/src/evaluate.jl b/src/evaluate.jl index 5092fd493..71a86788e 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -38,25 +38,24 @@ end """ sandboxed_julia(config::Configuration, install::String, args=``; env=Dict(), mounts=Dict(), - wait=true, stdin=stdin, stdout=stdout, stderr=stderr, - install_dir="/opt/julia", kwargs...) + wait=true, stdin=stdin, stdout=stdout, stderr=stderr, kwargs...) Run Julia inside of a sandbox, passing the given arguments `args` to it. The argument `wait` determines if the process will be waited on. Streams can be connected using the `stdin`, `stdout` and `sterr` arguments. Returns a `Process` object. Further customization is possible using the `env` arg, to set environment variables, and the -`mounts` argument to mount additional directories. With `install_dir`, the directory where -Julia is installed can be chosen. +`mounts` argument to mount additional directories. """ -function sandboxed_julia(config::Configuration, install::String, args=``; wait=true, kwargs...) - sandbox_config, cmd = sandboxed_julia_cmd(install, args; kwargs...) - +function sandboxed_julia(config::Configuration, install::String, args=``; wait=true, + stdin=stdin, stdout=stdout, stderr=stderr, kwargs...) # XXX: even when preferred_executor() returns UnprivilegedUserNamespacesExecutor, # sometimes a stray sudo happens at run time? no idea how. exe_typ = UnprivilegedUserNamespacesExecutor exe = exe_typ() - proc = run(exe, sandbox_config, cmd; wait) + + cmd = sandboxed_julia_cmd(config, install, exe, args; kwargs...) + proc = run(pipeline(cmd; stdin, stderr, stdout); wait) # TODO: introduce a --stats flag that has the sandbox trace and report on CPU, network, ... usage @@ -80,15 +79,13 @@ end const xvfb_lock = ReentrantLock() const xvfb_proc = Ref{Union{Base.Process,Nothing}}(nothing) -function sandboxed_julia_cmd(install::String, args=``; install_dir="/opt/julia", - stdin=stdin, stdout=stdout, stderr=stderr, +function sandboxed_julia_cmd(config::Configuration, install::String, executor, args=``; env::Dict{String,String}=Dict{String,String}(), mounts::Dict{String,String}=Dict{String,String}(), - xvfb::Bool=true, cpus::Vector{Int}=Int[], - sysimage=nothing, rootfs=prepare_rootfs()) + rootfs=prepare_rootfs()) read_only_maps = Dict( "/" => rootfs.path, - install_dir => install, + config.julia_install_dir => install, "/usr/local/share/julia/registries" => joinpath(first(DEPOT_PATH), "registries"), ) @@ -117,7 +114,7 @@ function sandboxed_julia_cmd(install::String, args=``; install_dir="/opt/julia", env["TERM"] = ENV["TERM"] end - if xvfb + if config.xvfb lock(xvfb_lock) do if xvfb_proc[] === nothing || !process_running(xvfb_proc[]) proc = run(`Xvfb :1 -screen 0 1024x768x16`; wait=false) @@ -136,12 +133,12 @@ function sandboxed_julia_cmd(install::String, args=``; install_dir="/opt/julia", read_write_maps["/tmp/.X11-unix"] = "/tmp/.X11-unix" end - cmd = `$install_dir/bin/julia` + cmd = `$(config.julia_install_dir)/bin/julia` # restrict resource usage - if !isempty(cpus) - cmd = `/usr/bin/taskset --cpu-list $(join(cpus, ',')) $cmd` - env["JULIA_CPU_THREADS"] = string(length(cpus)) # JuliaLang/julia#35787 + if !isempty(config.cpus) + cmd = `/usr/bin/taskset --cpu-list $(join(config.cpus, ',')) $cmd` + env["JULIA_CPU_THREADS"] = string(length(config.cpus)) # JuliaLang/julia#35787 end # NOTE: we use persist=true so that modifications to the rootfs are backed by @@ -150,13 +147,8 @@ function sandboxed_julia_cmd(install::String, args=``; install_dir="/opt/julia", sandbox_config = SandboxConfig(read_only_maps, read_write_maps, env; rootfs.uid, rootfs.gid, pwd=rootfs.home, persist=true, - stdin, stdout, stderr, verbose=isdebug(:sandbox)) - - if sysimage !== nothing - args = `--sysimage=$sysimage $args` - end - - return sandbox_config, `$cmd $args` + verbose=isdebug(:sandbox)) + Sandbox.build_executor_command(executor, sandbox_config, `$cmd $(config.julia_args) $args`) end function process_children(pid) @@ -497,15 +489,14 @@ function compiled_test(config::Configuration, install::String, pkg; log_limit = # run the tests in an alternate environment (different OS, depot and Julia binaries # in another path, etc) - test_config = Configuration( - julia = config.julia, - depwarn = config.depwarn, + test_config = Configuration(config; + julia_install_dir="/usr/local/julia", + julia_args = `$(config.julia_args) --sysimage $sysimage_path`, compiled = false, ) rootfs = prepare_rootfs("arch"; user="user", group="group") version, status, reason, test_log = sandboxed_test(test_config, install, pkg; mounts, rootfs, log_limit, - sysimage=sysimage_path, install_dir="/usr/local/julia", kwargs...) rm(sysimage_dir; recursive=true) @@ -677,8 +668,9 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; end # perform an initial run + config′ = Configuration(config; cpus=[i-1]) pkg_version, status, reason, log = - sandboxed_test(config, julia_install, pkg; cpus=[i-1], kwargs...) + sandboxed_test(config′, julia_install, pkg; kwargs...) # certain packages are known to have flaky tests; retry them for j in 1:retries @@ -686,7 +678,7 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; pkg.name in retry_lists[registry] times[i] = now() pkg_version, status, reason, log = - sandboxed_test(julia_install, pkg; cpus=[i-1], kwargs...) + sandboxed_test(julia_install, pkg; kwargs...) end end From cac9ae8d0cc7dd272b4efba298d9632a04050512 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 18:23:58 +0200 Subject: [PATCH 07/19] Put rootfs stuff in the Configuration. --- src/PkgEval.jl | 8 ++++++++ src/evaluate.jl | 41 +++++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/PkgEval.jl b/src/PkgEval.jl index e00b6bfd4..d5fa3a4fc 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -36,6 +36,14 @@ Base.@kwdef mutable struct Configuration # additional Julia arguments to pass to the process julia_args::Cmd = `` + + # rootfs properties + distro::String = "debian" + uid::Int = 1000 + user::String = "pkgeval" + gid::Int = 1000 + group::String = "pkgeval" + home::String = "/home/pkgeval" end # copy constructor that allows overriding specific fields diff --git a/src/evaluate.jl b/src/evaluate.jl index 71a86788e..28efa9843 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -4,10 +4,10 @@ lazy_artifact(x) = @artifact_str(x) const rootfs_lock = ReentrantLock() const rootfs_cache = Dict() -function prepare_rootfs(distro="debian"; uid=1000, user="pkgeval", gid=1000, group="pkgeval", home="/home/$user") +function prepare_rootfs(config::Configuration) lock(rootfs_lock) do - get!(rootfs_cache, (distro, uid, user, gid, group, home)) do - base = lazy_artifact(distro) + get!(rootfs_cache, (config.distro, config.uid, config.user, config.gid, config.group, config.home)) do + base = lazy_artifact(config.distro) # a bare rootfs isn't usable out-of-the-box derived = mktempdir() @@ -16,22 +16,22 @@ function prepare_rootfs(distro="debian"; uid=1000, user="pkgeval", gid=1000, gro # add a user and group chmod(joinpath(derived, "etc/passwd"), 0o644) open(joinpath(derived, "etc/passwd"), "a") do io - println(io, "$user:x:$uid:$gid::$home:/bin/bash") + println(io, "$(config.user):x:$(config.uid):$(config.gid)::$(config.home):/bin/bash") end chmod(joinpath(derived, "etc/group"), 0o644) open(joinpath(derived, "etc/group"), "a") do io - println(io, "$group:x:$gid:") + println(io, "$(config.group):x:$(config.gid):") end chmod(joinpath(derived, "etc/shadow"), 0o640) open(joinpath(derived, "etc/shadow"), "a") do io - println(io, "$user:*:::::::") + println(io, "$(config.user):*:::::::") end # replace resolv.conf rm(joinpath(derived, "etc/resolv.conf"); force=true) write(joinpath(derived, "etc/resolv.conf"), read("/etc/resolv.conf")) - return (path=derived, uid, user, gid, group, home) + return derived end end end @@ -81,10 +81,10 @@ const xvfb_proc = Ref{Union{Base.Process,Nothing}}(nothing) function sandboxed_julia_cmd(config::Configuration, install::String, executor, args=``; env::Dict{String,String}=Dict{String,String}(), - mounts::Dict{String,String}=Dict{String,String}(), - rootfs=prepare_rootfs()) + mounts::Dict{String,String}=Dict{String,String}()) + rootfs = prepare_rootfs(config) read_only_maps = Dict( - "/" => rootfs.path, + "/" => rootfs, config.julia_install_dir => install, "/usr/local/share/julia/registries" => joinpath(first(DEPOT_PATH), "registries"), ) @@ -92,7 +92,7 @@ function sandboxed_julia_cmd(config::Configuration, install::String, executor, a artifacts_path = joinpath(storage_dir, "artifacts") mkpath(artifacts_path) read_write_maps = merge(mounts, Dict( - joinpath(rootfs.home, ".julia/artifacts") => artifacts_path + joinpath(config.home, ".julia/artifacts") => artifacts_path )) env = merge(env, Dict( @@ -108,7 +108,7 @@ function sandboxed_julia_cmd(config::Configuration, install::String, executor, a # some essential env vars (since we don't run from a shell) "PATH" => "/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin", - "HOME" => rootfs.home, + "HOME" => config.home, )) if haskey(ENV, "TERM") env["TERM"] = ENV["TERM"] @@ -146,7 +146,7 @@ function sandboxed_julia_cmd(config::Configuration, install::String, executor, a # because some packages like to generate a lot of data during testing. sandbox_config = SandboxConfig(read_only_maps, read_write_maps, env; - rootfs.uid, rootfs.gid, pwd=rootfs.home, persist=true, + config.uid, config.gid, pwd=config.home, persist=true, verbose=isdebug(:sandbox)) Sandbox.build_executor_command(executor, sandbox_config, `$cmd $(config.julia_args) $args`) end @@ -490,14 +490,19 @@ function compiled_test(config::Configuration, install::String, pkg; log_limit = # run the tests in an alternate environment (different OS, depot and Julia binaries # in another path, etc) test_config = Configuration(config; - julia_install_dir="/usr/local/julia", - julia_args = `$(config.julia_args) --sysimage $sysimage_path`, compiled = false, + julia_args = `$(config.julia_args) --sysimage $sysimage_path`, + # install Julia at a different path + julia_install_dir="/usr/local/julia", + # use a different Linux distro + distro="arch", + # run as a different user + user="user", + group="group", + home="/home/user", ) - rootfs = prepare_rootfs("arch"; user="user", group="group") version, status, reason, test_log = - sandboxed_test(test_config, install, pkg; mounts, rootfs, log_limit, - kwargs...) + sandboxed_test(test_config, install, pkg; mounts, log_limit, kwargs...) rm(sysimage_dir; recursive=true) return version, status, reason, log * "\n" * test_log From 40dbe82ca56a4b89b938155674fe93c6dd69d825 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 18:42:46 +0200 Subject: [PATCH 08/19] Fix retry mode. --- src/evaluate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evaluate.jl b/src/evaluate.jl index 28efa9843..3a9af2a21 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -683,7 +683,7 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; pkg.name in retry_lists[registry] times[i] = now() pkg_version, status, reason, log = - sandboxed_test(julia_install, pkg; kwargs...) + sandboxed_test(config′, julia_install, pkg; kwargs...) end end From 8113482098eb648578d4c94c07c8f667878246ef Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 18:52:51 +0200 Subject: [PATCH 09/19] Put limits in Configuration object. --- src/PkgEval.jl | 6 ++++-- src/evaluate.jl | 33 +++++++++++++++------------------ test/runtests.jl | 8 ++++---- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/PkgEval.jl b/src/PkgEval.jl index d5fa3a4fc..94e992612 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -19,10 +19,12 @@ export Configuration Base.@kwdef mutable struct Configuration julia::String = "nightly" - compiled::Bool = false buildflags::Vector{String} = String[] depwarn::Bool = false - # TODO: put even more here (rootfs, install_dir, limits, etc) + log_limit::Int = 2^20 # 1 MB + time_limit = 60*60 # 1 hour + compiled::Bool = false + compile_time_limit::Int = 30*60 # 30 mins # the directory where Julia is installed in the run-time environment julia_install_dir::String = "/opt/julia" diff --git a/src/evaluate.jl b/src/evaluate.jl index 3a9af2a21..e80a58a13 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -179,21 +179,16 @@ function cpu_time(pid) end """ - sandboxed_script(config::Configuration, install::String, script::String, args=``; - log_limit=2^20, time_limit=60*60) + sandboxed_script(config::Configuration, install::String, script::String, args=``) Run a Julia script `script` in non-interactive mode, returning the process status and a -failure reason if any (both represented by a symbol), and the full log. Execution of the -script will be forcibly interrupted after `time_limit` seconds (defaults to 1h) or if the -log becomes larger than `log_limit` (defaults to 1MB). +failure reason if any (both represented by a symbol), and the full log. Refer to `sandboxed_julia`[@ref] for more possible `keyword arguments. """ function sandboxed_script(config::Configuration, install::String, script::String, args=``; - log_limit = 2^20 #= 1 MB =#, - time_limit = 60*60, kwargs...) - @assert log_limit > 0 + @assert config.log_limit > 0 cmd = `--eval 'eval(Meta.parse(read(stdin,String)))' $args` @@ -249,7 +244,7 @@ function sandboxed_script(config::Configuration, install::String, script::String reason = missing # kill on timeout - timeout_monitor = Timer(time_limit) do timer + timeout_monitor = Timer(config.time_limit) do timer process_running(proc) || return status = :kill reason = :time_limit @@ -280,7 +275,7 @@ function sandboxed_script(config::Configuration, install::String, script::String print(io, readline(output; keep=true)) # kill on too-large logs - if io.size > log_limit + if io.size > config.log_limit process_running(proc) || break status = :kill reason = :log_limit @@ -296,10 +291,10 @@ function sandboxed_script(config::Configuration, install::String, script::String close(inactivity_monitor) log = fetch(log_monitor) - if sizeof(log) > log_limit + if sizeof(log) > config.log_limit # even though the monitor above should have limited the log size, # a single line may still have exceeded the limit, so make sure we truncate. - ind = prevind(log, log_limit) + ind = prevind(log, config.log_limit) log = log[1:ind] end @@ -417,7 +412,7 @@ function sandboxed_test(config::Configuration, install::String, pkg; kwargs...) end """ - compiled_test(install::String, pkg; compile_time_limit=30*60) + compiled_test(install::String, pkg) Run the unit tests for a single package `pkg` (see `compiled_test`[@ref] for details and a list of supported keyword arguments), after first having compiled a system image that @@ -426,8 +421,7 @@ contains this package and its dependencies. To find incompatibilities, the compilation happens on an Ubuntu-based runner, while testing is performed in an Arch Linux container. """ -function compiled_test(config::Configuration, install::String, pkg; log_limit = 2^20 #= 1 MB =#, - compile_time_limit=30*60, do_depwarns=false, kwargs...) +function compiled_test(config::Configuration, install::String, pkg; kwargs...) script = raw""" try using Dates @@ -469,8 +463,11 @@ function compiled_test(config::Configuration, install::String, pkg; log_limit = sysimage_dir = mktempdir() mounts = Dict(dirname(sysimage_path) => sysimage_dir) - status, reason, log = sandboxed_script(config, install, script, args; mounts, - time_limit=compile_time_limit, kwargs...) + compile_config = Configuration(config; + time_limit = config.compile_time_limit + ) + status, reason, log = sandboxed_script(compile_config, install, script, args; mounts, + kwargs...) # try to figure out the failure reason if status === nothing @@ -502,7 +499,7 @@ function compiled_test(config::Configuration, install::String, pkg; log_limit = home="/home/user", ) version, status, reason, test_log = - sandboxed_test(test_config, install, pkg; mounts, log_limit, kwargs...) + sandboxed_test(test_config, install, pkg; mounts, kwargs...) rm(sysimage_dir; recursive=true) return version, status, reason, log * "\n" * test_log diff --git a/test/runtests.jl b/test/runtests.jl index ca529f666..9e7344b56 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,13 +29,13 @@ end @testset "time and output limits" begin # timeouts - results = PkgEval.evaluate([Configuration(; julia)], pkgnames; - time_limit=0.1, update_registry=false) + results = PkgEval.evaluate([Configuration(; julia, time_limit=0.1)], pkgnames; + update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :time_limit) # log limit - results = PkgEval.evaluate([Configuration(; julia)], pkgnames; - log_limit=1, update_registry=false) + results = PkgEval.evaluate([Configuration(; julia, log_limit=1)], pkgnames; + update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :log_limit) end From d6d6bf96dce353df95c7c78ff644330813bc6cc4 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 18:53:02 +0200 Subject: [PATCH 10/19] Bump version. --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 7cbfab838..549a32b77 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PkgEval" uuid = "9f2e2246-6dce-11e8-3d98-4b291446da6e" authors = ["Keno Fischer ", "Tim Besard "] -version = "0.1.0" +version = "0.2.0" [deps] BinaryBuilder = "12aac903-9f7c-5d81-afc2-d9565ea332ae" From 1fdab7a143a714a06a717113e2f4fef22607f139 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 19:25:35 +0200 Subject: [PATCH 11/19] Remove needless kwargs forwarding. --- src/evaluate.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/evaluate.jl b/src/evaluate.jl index e80a58a13..d1f1ed6e1 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -507,7 +507,7 @@ end function evaluate(configs::Vector{Configuration}, pkgs::Vector; ninstances::Integer=Sys.CPU_THREADS, retries::Integer=2, - registry::String=DEFAULT_REGISTRY, kwargs...) + registry::String=DEFAULT_REGISTRY) # here we deal with managing execution: spawning workers, output, result I/O, etc # Julia installation @@ -672,7 +672,7 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; # perform an initial run config′ = Configuration(config; cpus=[i-1]) pkg_version, status, reason, log = - sandboxed_test(config′, julia_install, pkg; kwargs...) + sandboxed_test(config′, julia_install, pkg) # certain packages are known to have flaky tests; retry them for j in 1:retries @@ -680,7 +680,7 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; pkg.name in retry_lists[registry] times[i] = now() pkg_version, status, reason, log = - sandboxed_test(config′, julia_install, pkg; kwargs...) + sandboxed_test(config′, julia_install, pkg) end end From 70075c8c48c6bce582c514ed45a5c1e324ea6da2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 19:25:44 +0200 Subject: [PATCH 12/19] Simplify test I/O handling. --- test/runtests.jl | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 9e7344b56..f6a0c847b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,21 +10,23 @@ julia = get(ENV, "JULIA", string(VERSION)) config = Configuration(; julia) install = PkgEval.prepare_julia(config) - mktemp() do path, io - try - PkgEval.sandboxed_julia(config, install, `-e 'print(1337)'`; stdout=io) - close(io) - @test read(path, String) == "1337" - catch - # if we failed to spawn a container, make sure to print the reason - flush(io) - @error read(path, String) - rethrow() - end + let + p = Pipe() + close(p.in) + PkgEval.sandboxed_julia(config, install, `-e 'print(1337)'`; stdout=p.out) + @test read(p.out, String) == "1337" end - # print versioninfo so we can verify in CI logs that the correct version is used - PkgEval.sandboxed_julia(config, install, `-e 'using InteractiveUtils; versioninfo()'`) + # try to compare the version info + let + p = Pipe() + PkgEval.sandboxed_julia(config, install, `-e 'println(VERSION)'`; stdout=p.out) + version_str = read(p.out, String) + requested_version = tryparse(VersionNumber, julia) + if requested_version !== nothing + @test parse(VersionNumber, version_str) == requested_version + end + end end @testset "time and output limits" begin From e0f85d90985c4703e4f856b8be2fb5f115cf2196 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 27 Jul 2022 19:28:06 +0200 Subject: [PATCH 13/19] Export evaluate function. --- src/PkgEval.jl | 2 +- test/runtests.jl | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/PkgEval.jl b/src/PkgEval.jl index 94e992612..244b20d46 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -15,7 +15,7 @@ import Scratch: @get_scratch! download_dir = "" storage_dir = "" -export Configuration +export Configuration, evaluate Base.@kwdef mutable struct Configuration julia::String = "nightly" diff --git a/test/runtests.jl b/test/runtests.jl index f6a0c847b..3ed2ca9b6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -31,19 +31,18 @@ end @testset "time and output limits" begin # timeouts - results = PkgEval.evaluate([Configuration(; julia, time_limit=0.1)], pkgnames; - update_registry=false) + results = evaluate([Configuration(; julia, time_limit=0.1)], pkgnames; + update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :time_limit) # log limit - results = PkgEval.evaluate([Configuration(; julia, log_limit=1)], pkgnames; - update_registry=false) + results = evaluate([Configuration(; julia, log_limit=1)], pkgnames; + update_registry=false) @test all(results.status .== :kill) && all(results.reason .== :log_limit) end @testset "main entrypoint" begin - results = PkgEval.evaluate([Configuration(; julia)], pkgnames; - update_registry=false) + results = evaluate([Configuration(; julia)], pkgnames; update_registry=false) if !(julia == "master" || julia == "nightly") @test all(results.status .== :ok) for result in eachrow(results) @@ -53,8 +52,8 @@ end end @testset "PackageCompiler" begin - results = PkgEval.evaluate([Configuration(; julia, compiled=true)], ["Example"]; - update_registry=false) + results = evaluate([Configuration(; julia, compiled=true)], ["Example"]; + update_registry=false) if !(julia == "master" || julia == "nightly") @test all(results.status .== :ok) for result in eachrow(results) From 9897851565b8cb64cb97d99a020ab6f010d215e5 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 11:49:02 +0200 Subject: [PATCH 14/19] Use a Package struct so that we can add specific versions. --- deps/Registries.toml => Packages.toml | 3 - src/PkgEval.jl | 37 ++++++++++-- src/evaluate.jl | 81 +++++++++---------------- src/julia.jl | 2 +- src/registry.jl | 59 +++--------------- test/runtests.jl | 87 ++++++++++++++++++++------- 6 files changed, 137 insertions(+), 132 deletions(-) rename deps/Registries.toml => Packages.toml (86%) diff --git a/deps/Registries.toml b/Packages.toml similarity index 86% rename from deps/Registries.toml rename to Packages.toml index efda1287e..55541208d 100644 --- a/deps/Registries.toml +++ b/Packages.toml @@ -1,6 +1,3 @@ -["General"] -url = "https://github.com/JuliaRegistries/General.git" -uuid = "23338594-aafe-5451-b93e-139f81909106" skip = [ "julia", "BugReporting", # nested rr is not supported diff --git a/src/PkgEval.jl b/src/PkgEval.jl index 244b20d46..de393a1cd 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -15,9 +15,12 @@ import Scratch: @get_scratch! download_dir = "" storage_dir = "" -export Configuration, evaluate +skip_list = String[] +retry_list = String[] -Base.@kwdef mutable struct Configuration +export Configuration, Package, evaluate + +Base.@kwdef struct Configuration julia::String = "nightly" buildflags::Vector{String} = String[] depwarn::Bool = false @@ -48,6 +51,28 @@ Base.@kwdef mutable struct Configuration home::String = "/home/pkgeval" end +Base.@kwdef struct Package + # source of the package; forwarded to PackageSpec + name::String + version::Union{Nothing,VersionNumber} = nothing + url::Union{Nothing,String} = nothing + rev::Union{Nothing,String} = nothing + + retries::Int = 0 +end + +# convert a Package to a tuple that's Pkg.add'able +function package_spec_tuple(pkg::Package) + spec = (;) + for field in (:name, :version, :url, :rev) + val = getfield(pkg, field) + if val !== nothing + spec = merge(spec, NamedTuple{(field,)}((val,))) + end + end + spec +end + # copy constructor that allows overriding specific fields function Configuration(cfg::Configuration; kwargs...) kwargs = Dict(kwargs...) @@ -58,9 +83,6 @@ function Configuration(cfg::Configuration; kwargs...) Configuration(; merged_kwargs...) end -# behave as a scalar in broadcast expressions -Base.broadcastable(x::Configuration) = Ref(x) - # utils isdebug(group) = Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing @@ -75,6 +97,11 @@ function __init__() mkpath(joinpath(download_dir, "srccache")) global storage_dir = @get_scratch!("storage") + + # read Packages.toml + packages = TOML.parsefile(joinpath(dirname(@__DIR__), "Packages.toml")) + global skip_list = get(packages, "skip", String[]) + global retry_list = get(packages, "retry", String[]) end end # module diff --git a/src/evaluate.jl b/src/evaluate.jl index d1f1ed6e1..6853056ab 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -4,7 +4,7 @@ lazy_artifact(x) = @artifact_str(x) const rootfs_lock = ReentrantLock() const rootfs_cache = Dict() -function prepare_rootfs(config::Configuration) +function create_rootfs(config::Configuration) lock(rootfs_lock) do get!(rootfs_cache, (config.distro, config.uid, config.user, config.gid, config.group, config.home)) do base = lazy_artifact(config.distro) @@ -82,7 +82,7 @@ const xvfb_proc = Ref{Union{Base.Process,Nothing}}(nothing) function sandboxed_julia_cmd(config::Configuration, install::String, executor, args=``; env::Dict{String,String}=Dict{String,String}(), mounts::Dict{String,String}=Dict{String,String}()) - rootfs = prepare_rootfs(config) + rootfs = create_rootfs(config) read_only_maps = Dict( "/" => rootfs, config.julia_install_dir => install, @@ -309,7 +309,7 @@ at `install`. Refer to `sandboxed_script`[@ref] for more possible `keyword arguments. """ -function sandboxed_test(config::Configuration, install::String, pkg; kwargs...) +function sandboxed_test(config::Configuration, install::String, pkg::Package; kwargs...) if config.compiled return compiled_test(config, install, pkg; kwargs...) end @@ -327,12 +327,13 @@ function sandboxed_test(config::Configuration, install::String, pkg; kwargs...) print("\n\n", '#'^80, "\n# Installation: $(now())\n#\n\n") using Pkg - Pkg.add(ARGS[1]) + package_spec = eval(Meta.parse(ARGS[1])) + Pkg.add(; package_spec...) print("\n\n", '#'^80, "\n# Testing: $(now())\n#\n\n") - Pkg.test(ARGS[1]) + Pkg.test(package_spec.name) println("\nPkgEval succeeded") catch err @@ -344,7 +345,8 @@ function sandboxed_test(config::Configuration, install::String, pkg; kwargs...) print("\n\n", '#'^80, "\n# PkgEval teardown: $(now())\n#\n\n") end""" - args = `$(pkg.name)` + # generate a PackageSpec we'll use to install the package + args = `$(repr(package_spec_tuple(pkg)))` if config.depwarn args = `--depwarn=error $args` end @@ -505,22 +507,31 @@ function compiled_test(config::Configuration, install::String, pkg; kwargs...) return version, status, reason, log * "\n" * test_log end -function evaluate(configs::Vector{Configuration}, pkgs::Vector; - ninstances::Integer=Sys.CPU_THREADS, retries::Integer=2, - registry::String=DEFAULT_REGISTRY) +""" + evaluate(configs::Vector{Configuration}, packages::Vector{String}=[]]; kwargs...) + +Run all tests for all packages in the registry `registry`, or only for the packages as +identified by their name in `pkgnames`, using the configurations from `configs`. +The registry is first updated if `update_registry` is set to true. + +Refer to `sandboxed_test`[@ref] and `sandboxed_julia`[@ref] for more possible +keyword arguments. +""" +function evaluate(configs::Vector{Configuration}, packages::Vector{Package}; + ninstances::Integer=Sys.CPU_THREADS) # here we deal with managing execution: spawning workers, output, result I/O, etc # Julia installation instantiated_configs = Dict() for config in configs - install = prepare_julia(config) + install = install_julia(config) # XXX: better get the version from a file in the tree version_str = chomp(read(`$install/bin/julia --startup-file=no --eval "println(VERSION)"`, String)) version = parse(VersionNumber, version_str) instantiated_configs[config] = (install, version) end - jobs = vec(collect(Iterators.product(instantiated_configs, pkgs))) + jobs = vec(collect(Iterators.product(instantiated_configs, packages))) # use a random test order to (hopefully) get a more reasonable ETA shuffle!(jobs) @@ -611,7 +622,6 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; julia_version = VersionNumber[], compiled = Bool[], name = String[], - uuid = UUID[], version = Union{Missing,VersionNumber}[], status = Symbol[], reason = Union{Missing,Symbol}[], @@ -642,28 +652,16 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; times[i] = now() running[i] = (config, pkg) - # can we even test this package? - pkg_info = Registry.registry_info(pkg) - pkg_compat = Registry.compat_info(pkg_info) - supported = any(pkg_compat) do (version, compat) - !haskey(compat, Registry.JULIA_UUID) || - julia_version ∈ compat[Registry.JULIA_UUID] - end - if !supported - push!(result, [config.julia, julia_version, config.compiled, - pkg.name, pkg.uuid, missing, - :skip, :unsupported, 0, missing]) - running[i] = nothing - continue - elseif pkg.name in skip_lists[registry] + # should we even test this package? + if pkg.name in skip_list push!(result, [config.julia, julia_version, config.compiled, - pkg.name, pkg.uuid, missing, + pkg.name, missing, :skip, :explicit, 0, missing]) running[i] = nothing continue elseif endswith(pkg.name, "_jll") push!(result, [config.julia, julia_version, config.compiled, - pkg.name, pkg.uuid, missing, + pkg.name, missing, :skip, :jll, 0, missing]) running[i] = nothing continue @@ -675,9 +673,8 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; sandboxed_test(config′, julia_install, pkg) # certain packages are known to have flaky tests; retry them - for j in 1:retries - if status == :fail && reason == :test_failures && - pkg.name in retry_lists[registry] + for j in 1:pkg.retries + if status == :fail && reason == :test_failures times[i] = now() pkg_version, status, reason, log = sandboxed_test(config′, julia_install, pkg) @@ -686,7 +683,7 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; duration = (now()-times[i]) / Millisecond(1000) push!(result, [config.julia, julia_version, config.compiled, - pkg.name, pkg.uuid, pkg_version, + pkg.name, pkg_version, status, reason, duration, log]) running[i] = nothing end @@ -711,23 +708,3 @@ function evaluate(configs::Vector{Configuration}, pkgs::Vector; return result end - -""" - evaluate(configs::Vector{Configuration}=[Configuration()], - pkg_names::Vector{String}=[]]; registry=General, update_registry=true, kwargs...) - -Run all tests for all packages in the registry `registry`, or only for the packages as -identified by their name in `pkgnames`, using the configurations from `configs`. -The registry is first updated if `update_registry` is set to true. - -Refer to `sandboxed_test`[@ref] and `sandboxed_julia`[@ref] for more possible -keyword arguments. -""" -function evaluate(configs::Vector{Configuration}=[Configuration()], - pkg_names::Vector{String}=String[]; - registry::String=DEFAULT_REGISTRY, update_registry::Bool=true, kwargs...) - prepare_registry(registry; update=update_registry) - pkgs = read_pkgs(pkg_names) - - evaluate(configs, pkgs; registry, kwargs...) -end diff --git a/src/julia.jl b/src/julia.jl index 56eee790f..7eaa0d9c5 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -231,7 +231,7 @@ function build_julia(repo_path::String; return install_dir end -function prepare_julia(config::Configuration) +function install_julia(config::Configuration) if isempty(config.buildflags) # check if it's an official release dir = get_julia_release(config.julia) diff --git a/src/registry.jl b/src/registry.jl index 1d389d2f7..20a64b8b9 100644 --- a/src/registry.jl +++ b/src/registry.jl @@ -1,62 +1,21 @@ const DEFAULT_REGISTRY = "General" -const skip_lists = Dict{String,Vector{String}}() -const retry_lists = Dict{String,Vector{String}}() - -registries_file() = joinpath(dirname(@__DIR__), "deps", "Registries.toml") -read_registries() = TOML.parsefile(registries_file()) - -""" - prepare_registry([name]) - -Prepare the given registry for use by this package and update it if `update` is true. -`name` must correspond to an existing stanza in the `deps/Registries.toml` file. -""" -function prepare_registry(name=DEFAULT_REGISTRY; update::Bool=false) - reg = read_registries()[name] - regspec = RegistrySpec(name = name, url = reg["url"], uuid = UUID(reg["uuid"])) - - # clone and update the registry - if !any(existing_regspec -> existing_regspec.name == name, - Pkg.Registry.reachable_registries()) - Pkg.Types.clone_or_cp_registries([regspec]) - elseif update - Pkg.Registry.update(name) - end - - # read some metadata - skip_lists[name] = get(reg, "skip", String[]) - retry_lists[name] = get(reg, "retry", String[]) - - return -end - """ - read_pkgs([pkgs::Vector{String}]; [registry::String]) + registry_packages(registry::String=DEFAULT_REGISTRY)::Vector{Package} -Read all packages from a registry and return them as a vector of tuples containing the -package name and registry, its UUID, and a path to it. If `pkgs` is given, only collect -packages matching the names in `pkgs` +Read all packages from a registry and return them as a vector of Package structs. """ -function read_pkgs(pkg_names::Vector{String}=String[]; registry=DEFAULT_REGISTRY) - pkg_names = Set(pkg_names) - want_all = isempty(pkg_names) - - pkg_data = [] +function registry_packages(registry::String=DEFAULT_REGISTRY; retries::Integer=2) + packages = Package[] registry_instance = only(filter(ri->ri.name == registry, Pkg.Registry.reachable_registries())) for (uuid, pkg) in registry_instance - if !want_all - pkg.name in pkg_names || continue - delete!(pkg_names, pkg.name) - end - - push!(pkg_data, pkg) - end + # TODO: read package compat info so that we can avoid testing uninstallable packages - if !want_all && !isempty(pkg_names) - @warn """did not find the following packages in the $registry registry:\n $(" - " .* join(pkg_names, '\n'))""" + push!(packages, + Package(name=pkg.name, + retries=(pkg.name in retry_list ? retries : 0))) end - return pkg_data + return packages end diff --git a/test/runtests.jl b/test/runtests.jl index 3ed2ca9b6..7f89dcfc9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,14 +1,13 @@ using PkgEval using Test -pkgnames = ["TimerOutputs", "Crayons", "Example", "Gtk"] - julia = get(ENV, "JULIA", string(VERSION)) +julia_version = tryparse(VersionNumber, julia) @testset "PkgEval using Julia $julia" begin @testset "sandbox" begin config = Configuration(; julia) - install = PkgEval.prepare_julia(config) + install = PkgEval.install_julia(config) let p = Pipe() @@ -16,33 +15,79 @@ julia = get(ENV, "JULIA", string(VERSION)) PkgEval.sandboxed_julia(config, install, `-e 'print(1337)'`; stdout=p.out) @test read(p.out, String) == "1337" end +end - # try to compare the version info - let - p = Pipe() - PkgEval.sandboxed_julia(config, install, `-e 'println(VERSION)'`; stdout=p.out) - version_str = read(p.out, String) - requested_version = tryparse(VersionNumber, julia) - if requested_version !== nothing - @test parse(VersionNumber, version_str) == requested_version +@testset "julia installation" begin + let results = evaluate([Configuration(; julia)], + [Package(; name="Example")]) + @test size(results, 1) == 1 + @test results[1, :julia_spec] == julia + if julia_version !== nothing + @test results[1, :julia_version] == julia_version end end end +@testset "package installation" begin + # by name + let results = evaluate([Configuration(; julia)], + [Package(; name="Example")]) + @test size(results, 1) == 1 + @test results[1, :name] == "Example" + @test results[1, :version] isa VersionNumber + @test results[1, :status] == :ok + end + + # specifying a version + let results = evaluate([Configuration(; julia)], + [Package(; name="Example", version=v"0.5.3")]) + @test size(results, 1) == 1 + @test results[1, :name] == "Example" + @test results[1, :version] == v"0.5.3" + @test results[1, :status] == :ok + end + + # specifying a revision + let results = evaluate([Configuration(; julia)], + [Package(; name="Example", rev="master")]) + @test size(results, 1) == 1 + @test results[1, :name] == "Example" + @test results[1, :status] == :ok + @test contains(results[1, :log], "https://github.com/JuliaLang/Example.jl.git#master") + end + + # specifying the URL + let results = evaluate([Configuration(; julia)], + [Package(; name="Example", url="https://github.com/JuliaLang/Example.jl")]) + @test size(results, 1) == 1 + @test results[1, :name] == "Example" + @test results[1, :status] == :ok + @test contains(results[1, :log], "https://github.com/JuliaLang/Example.jl#master") + end +end + @testset "time and output limits" begin # timeouts - results = evaluate([Configuration(; julia, time_limit=0.1)], pkgnames; - update_registry=false) - @test all(results.status .== :kill) && all(results.reason .== :time_limit) + let results = evaluate([Configuration(; julia, time_limit=0.1)], + [Package(; name="Example")]) + @test size(results, 1) == 1 + @test results[1, :status] == :kill && results[1, :reason] == :time_limit + end # log limit - results = evaluate([Configuration(; julia, log_limit=1)], pkgnames; - update_registry=false) - @test all(results.status .== :kill) && all(results.reason .== :log_limit) + let results = evaluate([Configuration(; julia, log_limit=1)], + [Package(; name="Example")]) + @test size(results, 1) == 1 + @test results[1, :status] == :kill && results[1, :reason] == :log_limit + end end -@testset "main entrypoint" begin - results = evaluate([Configuration(; julia)], pkgnames; update_registry=false) +@testset "complex packages" begin + # some more complicate packages that are all expected to pass tests + package_names = ["TimerOutputs", "Crayons", "Example", "Gtk"] + packages = [Package(; name) for name in package_names] + + results = evaluate([Configuration(; julia)], packages) if !(julia == "master" || julia == "nightly") @test all(results.status .== :ok) for result in eachrow(results) @@ -52,8 +97,8 @@ end end @testset "PackageCompiler" begin - results = evaluate([Configuration(; julia, compiled=true)], ["Example"]; - update_registry=false) + results = evaluate([Configuration(; julia, compiled=true)], + [Package(; name="Example")]) if !(julia == "master" || julia == "nightly") @test all(results.status .== :ok) for result in eachrow(results) From 7e755148e17527d5e7e07025747e463757088829 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 12:03:49 +0200 Subject: [PATCH 15/19] Update docstrings and README. --- README.md | 154 ++++++++---------------------------------------- src/PkgEval.jl | 2 +- src/evaluate.jl | 27 ++++++--- 3 files changed, 45 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index ae95f4f16..e451862b5 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,7 @@ ## Quick start -To use PkgEval.jl, you need to Docker and make sure you can start containers (typically, -you need to be a member of the `docker` group): - -``` -$ docker run hello-world -Hello from Docker! -This message shows that your installation appears to be working correctly. -``` - -Start by installing the package: +PkgEval is not a registered package, so you'll need to install it from Git: ```shell git clone https://github.com/JuliaCI/PkgEval.jl.git @@ -23,23 +14,22 @@ julia --project -e 'import Pkg; Pkg.instantiate()' ``` Then start Julia with `julia --project` and use the following commands to run the tests of a -list of packages on a selection of Julia versions: +list of packages on a selection of Julia configurations: ```julia julia> using PkgEval -julia> julia_versions = PkgEval.obtain_julia.(["1.3", "nightly"]) -2-element Array{VersionNumber,1}: - v"1.3.0" - v"1.4.0-DEV-3c182bc5c2" - -julia> PkgEval.run(julia_versions, ["Example"]) -2×8 DataFrames.DataFrame. Omitted printing of 1 columns -│ Row │ julia │ registry │ name │ version │ status │ reason │ duration │ -│ │ VersionNumber │ String │ String │ Version…⍰ │ Symbol │ Symbol⍰ │ Float64 │ -├─────┼─────────────────────────┼──────────┼─────────┼───────────┼────────┼─────────┼──────────┤ -│ 1 │ v"1.3.0" │ General │ Example │ v"0.5.3" │ ok │ missing │ 6.94 │ -│ 2 │ v"1.4.0-DEV-3c182bc5c2" │ General │ Example │ v"0.5.3" │ ok │ missing │ 6.948 │ +julia> config = Configuration(; julia="1.7"); + +julia> package = Package(; name="Example"); + +julia> evaluate([config], [package]) +1×9 DataFrame + Row │ julia_spec julia_version compiled name version ⋯ + │ String VersionNumber Bool String VersionN…? ⋯ +─────┼─────────────────────────────────────────────────────────── + 1 │ 1.7 1.7.0 false Example 0.5.3 ⋯ + 4 columns omitted ``` Test logs are part of this dataframe in the `log` column. For example, in this case: @@ -51,11 +41,6 @@ Installed Example ─ v0.5.3 Testing Example tests passed ``` -Other `run` methods, that offer more options and control over the testing process, are -available as well. These methods however require you to first prepare the environment -yourself, by calling `prepare_registry` to set-up the package registry, and `prepare_julia` -to download and unpack a binary version of Julia. - ## Why does my package fail? @@ -65,110 +50,19 @@ shell: ```julia julia> using PkgEval -julia> julia_version = v"1.3.0" # use `obtain_julia` if you need a specific build - -julia> julia_install = PkgEval.prepare_julia(julia_version) -julia> PkgEval.prepare_registry() - -julia> PkgEval.run_sandboxed_julia(julia_install) -``` - -Now you can install, load end test your package. If that fails because of some missing -dependency, you can just install that using the `apt` package manager within the container: - -``` -julia> # in the spawned container's Julia session, switch to REPL mode by pressing ; - -shell> sudo apt update -shell> sudo apt install ... -``` - -Once you've found the missing dependency and verified that it fixes the tests of your -package, make a [pull -request](https://github.com/JuliaComputing/PkgEval.jl/edit/master/runner/Dockerfile) to -include the dependency in the default image. - - -## Analyzing results - -Most of the time, you will want to compare the results that you obtained. For example: - -```julia -julia> result = PkgEval.run([v"1.2.0", v"1.4.0-DEV-76ebc419f0"], ["AbstractNumbers"]) -2×8 DataFrame. Omitted printing of 1 columns -│ Row │ julia │ registry │ name │ version │ status │ reason │ duration │ -│ │ VersionNumber │ String │ String │ Version…⍰ │ Symbol │ Symbol⍰ │ Float64 │ -├─────┼─────────────────────────┼──────────┼─────────────────┼───────────┼────────┼───────────────┼──────────┤ -│ 1 │ v"1.2.0" │ General │ AbstractNumbers │ v"0.2.0" │ ok │ missing │ 24.768 │ -│ 2 │ v"1.4.0-DEV-76ebc419f0" │ General │ AbstractNumbers │ v"0.2.0" │ fail │ test_failures │ 26.803 │ -``` - -If you simply want to compare two Julia versions, use `PkgEval.compare`: +julia> config = Configuration(; julia="1.7"); -```julia -julia> PkgEval.compare(result, v"1.2.0", v"1.4.0-DEV-76ebc419f0") -On v1.4.0-DEV-76ebc419f0, out of 1 packages 0 passed, 1 failed, 0 got killed and 0 were skipped. - -Comparing against v1.2.0: -- AbstractNumbers status was ok, now fail (reason: test_failures) -In summary, 0 packages now succeed, while 1 have started to fail. -``` - -For more extensive evaluations, or when more versions are involved, use `PkgEval.render` -to generate a HTML site in the `website/build` directory at the root of the repository: - -```julia -julia> PkgEval.render(result) -Generating site at /home/tim/Julia/pkg/PkgEval/site/build -``` - - -## Choosing a different version of Julia +julia> interactive_shell(config) -PkgEval ultimately needs a binary build of Julia to run tests with, but there's multiple -options to provide such a build. The easiest option is to use a version number that has -already been registered in the `Versions.toml` database, together with an URL and hash to -download an verify the file. An error will be thrown if the specific version cannot be -found. This is done automatically when the `prepare_julia` function is called (you will need -to call this method explicitly if you use a lower-level interface, i.e., anything but the -`run` function from the quick start section above): + _ _ _(_)_ | Documentation: https://docs.julialang.org + (_) | (_) (_) | + _ _ _| |_ __ _ | Type "?" for help, "]?" for Pkg help. + | | | | | | |/ _` | | + | | |_| | | | (_| | | Version 1.7.0 (2021-11-30) + _/ |\__'_|_|_|\__'_| | Official https://julialang.org/ release +|__/ | +julia> # we're in the PkgEval sandbox here ``` -julia> PkgEval.prepare_julia(v"1.2.0-nonexistent") -ERROR: Requested Julia version not found -``` - -Alternatively, you can download a named release as listed in `Releases.toml`. By calling -`obtain_julia_release` with a release name, this release will be downloaded, hashed, and -added to the `Versions.toml` database for later use. The method returns the version number -that corresponds with this added entry; you should use it when calling into other functions -of the package: - -```julia -julia_version = PkgEval.obtain_julia_release("nightly") -PkgEval.run([julia_version], ...) -``` - -For even more control, you can build Julia by calling the `perform_julia_build` function, -passing a string that identifies a branch, tag or commit in the Julia Git repository: - -```julia -julia_version = PkgEval.perform_julia_build("master") -``` - -Similarly, this function returns a version number that corresponds with an entry added to -`Versions.toml`: - -``` -["1.4.0-DEV-8f7855a7c3"] -file = "julia-1.4.0-DEV-8f7855a7c3.tar.gz" -sha = "dcd105b94906359cae52656129615a1446e7aee1e992ae9c06a15554d83a46f0" -``` - -If you get a permission error while building Julia, try to set the variable -`BINARYBUILDER_RUNNER=privileged`, restart Julia and try the build again. -To facilitate all this, there's a higher-level function `obtain_julia` that will try each of -the above methods until a valid version is found and returned. It is of course also possible -to build Julia yourself, in which case you will need to create a tarball, copy it to the -`deps/downloads` directory, and add a correct version stanza to `Versions.toml`. +Now you can install, load and test your package. diff --git a/src/PkgEval.jl b/src/PkgEval.jl index de393a1cd..bc06da315 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -18,7 +18,7 @@ storage_dir = "" skip_list = String[] retry_list = String[] -export Configuration, Package, evaluate +export Configuration, Package, evaluate, interactive_shell Base.@kwdef struct Configuration julia::String = "nightly" diff --git a/src/evaluate.jl b/src/evaluate.jl index 6853056ab..ad0b6ca7a 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -414,7 +414,7 @@ function sandboxed_test(config::Configuration, install::String, pkg::Package; kw end """ - compiled_test(install::String, pkg) + compiled_test(install::String, pkg::Package) Run the unit tests for a single package `pkg` (see `compiled_test`[@ref] for details and a list of supported keyword arguments), after first having compiled a system image that @@ -423,7 +423,7 @@ contains this package and its dependencies. To find incompatibilities, the compilation happens on an Ubuntu-based runner, while testing is performed in an Arch Linux container. """ -function compiled_test(config::Configuration, install::String, pkg; kwargs...) +function compiled_test(config::Configuration, install::String, pkg::Package; kwargs...) script = raw""" try using Dates @@ -508,16 +508,18 @@ function compiled_test(config::Configuration, install::String, pkg; kwargs...) end """ - evaluate(configs::Vector{Configuration}, packages::Vector{String}=[]]; kwargs...) + evaluate(configs::Vector{Configuration}, [packages::Vector{String}]; + ninstances=Sys.CPU_THREADS, kwargs...) -Run all tests for all packages in the registry `registry`, or only for the packages as -identified by their name in `pkgnames`, using the configurations from `configs`. -The registry is first updated if `update_registry` is set to true. +Run tests for `packages` using `configs`. If no packages are specified, default to testing +all packages in the default registry. +The `ninstances` keyword argument determines how many packages are tested in parallel. Refer to `sandboxed_test`[@ref] and `sandboxed_julia`[@ref] for more possible keyword arguments. """ -function evaluate(configs::Vector{Configuration}, packages::Vector{Package}; +function evaluate(configs::Vector{Configuration}, + packages::Vector{Package}=registry_packages(); ninstances::Integer=Sys.CPU_THREADS) # here we deal with managing execution: spawning workers, output, result I/O, etc @@ -708,3 +710,14 @@ function evaluate(configs::Vector{Configuration}, packages::Vector{Package}; return result end + +# TODO: make `install_julia` cache the returned directory so that we don't have to pass +# `install` around. This should make the inner functions directly usable again. +function interactive_shell(config::Configuration) + install = install_julia(config) + try + PkgEval.sandboxed_julia(config, install) + finally + rm(install; recursive=true) + end +end From f1865f9355d7ff2284ec8e18a9c1df040a832ccc Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 12:14:11 +0200 Subject: [PATCH 16/19] Reorganize files and imports. --- src/PkgEval.jl | 76 ++----------------------------------------------- src/evaluate.jl | 9 +++++- src/julia.jl | 7 ++--- src/types.jl | 64 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 79 deletions(-) create mode 100644 src/types.jl diff --git a/src/PkgEval.jl b/src/PkgEval.jl index bc06da315..9dbe9e275 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -1,15 +1,7 @@ module PkgEval -import Pkg.TOML using Pkg -using Base: UUID -using Dates -using ProgressMeter -using DataFrames -using Random -using Sandbox -using LazyArtifacts -using JSON +import Pkg.TOML import Scratch: @get_scratch! download_dir = "" @@ -18,75 +10,11 @@ storage_dir = "" skip_list = String[] retry_list = String[] -export Configuration, Package, evaluate, interactive_shell - -Base.@kwdef struct Configuration - julia::String = "nightly" - buildflags::Vector{String} = String[] - depwarn::Bool = false - log_limit::Int = 2^20 # 1 MB - time_limit = 60*60 # 1 hour - compiled::Bool = false - compile_time_limit::Int = 30*60 # 30 mins - - # the directory where Julia is installed in the run-time environment - julia_install_dir::String = "/opt/julia" - - # whether to launch Xvfb before starting Julia - xvfb::Bool = true - - # a list of CPUs to restrict the Julia process to (or empty if unconstrained). - # if set, JULIA_CPU_THREADS will also be set to a number equaling the number of CPUs. - cpus::Vector{Int} = Int[] - - # additional Julia arguments to pass to the process - julia_args::Cmd = `` - - # rootfs properties - distro::String = "debian" - uid::Int = 1000 - user::String = "pkgeval" - gid::Int = 1000 - group::String = "pkgeval" - home::String = "/home/pkgeval" -end - -Base.@kwdef struct Package - # source of the package; forwarded to PackageSpec - name::String - version::Union{Nothing,VersionNumber} = nothing - url::Union{Nothing,String} = nothing - rev::Union{Nothing,String} = nothing - - retries::Int = 0 -end - -# convert a Package to a tuple that's Pkg.add'able -function package_spec_tuple(pkg::Package) - spec = (;) - for field in (:name, :version, :url, :rev) - val = getfield(pkg, field) - if val !== nothing - spec = merge(spec, NamedTuple{(field,)}((val,))) - end - end - spec -end - -# copy constructor that allows overriding specific fields -function Configuration(cfg::Configuration; kwargs...) - kwargs = Dict(kwargs...) - merged_kwargs = Dict() - for field in fieldnames(Configuration) - merged_kwargs[field] = get(kwargs, field, getfield(cfg, field)) - end - Configuration(; merged_kwargs...) -end - # utils isdebug(group) = Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing +include("types.jl") include("registry.jl") include("julia.jl") include("evaluate.jl") diff --git a/src/evaluate.jl b/src/evaluate.jl index ad0b6ca7a..dfa8e3205 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -1,4 +1,11 @@ -export Configuration +export evaluate, interactive_shell + +using Dates +using Random +using DataFrames: DataFrame, nrow +using LazyArtifacts: @artifact_str +using ProgressMeter: Progress, update! +using Sandbox: Sandbox, SandboxConfig, UnprivilegedUserNamespacesExecutor, cleanup lazy_artifact(x) = @artifact_str(x) diff --git a/src/julia.jl b/src/julia.jl index 7eaa0d9c5..4b2aa47f6 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -1,8 +1,7 @@ -import Downloads import JSON -import Pkg -import Git: git -import BinaryBuilder: DirectorySource, ExecutableProduct, build_tarballs +import Downloads +using Git: git +using BinaryBuilder: DirectorySource, ExecutableProduct, build_tarballs const VERSIONS_URL = "https://julialang-s3.julialang.org/bin/versions.json" diff --git a/src/types.jl b/src/types.jl new file mode 100644 index 000000000..f2ba10276 --- /dev/null +++ b/src/types.jl @@ -0,0 +1,64 @@ +export Configuration, Package + +Base.@kwdef struct Configuration + julia::String = "nightly" + buildflags::Vector{String} = String[] + depwarn::Bool = false + log_limit::Int = 2^20 # 1 MB + time_limit = 60*60 # 1 hour + compiled::Bool = false + compile_time_limit::Int = 30*60 # 30 mins + + # the directory where Julia is installed in the run-time environment + julia_install_dir::String = "/opt/julia" + + # whether to launch Xvfb before starting Julia + xvfb::Bool = true + + # a list of CPUs to restrict the Julia process to (or empty if unconstrained). + # if set, JULIA_CPU_THREADS will also be set to a number equaling the number of CPUs. + cpus::Vector{Int} = Int[] + + # additional Julia arguments to pass to the process + julia_args::Cmd = `` + + # rootfs properties + distro::String = "debian" + uid::Int = 1000 + user::String = "pkgeval" + gid::Int = 1000 + group::String = "pkgeval" + home::String = "/home/pkgeval" +end + +Base.@kwdef struct Package + # source of the package; forwarded to PackageSpec + name::String + version::Union{Nothing,VersionNumber} = nothing + url::Union{Nothing,String} = nothing + rev::Union{Nothing,String} = nothing + + retries::Int = 0 +end + +# convert a Package to a tuple that's Pkg.add'able +function package_spec_tuple(pkg::Package) + spec = (;) + for field in (:name, :version, :url, :rev) + val = getfield(pkg, field) + if val !== nothing + spec = merge(spec, NamedTuple{(field,)}((val,))) + end + end + spec +end + +# copy constructor that allows overriding specific fields +function Configuration(cfg::Configuration; kwargs...) + kwargs = Dict(kwargs...) + merged_kwargs = Dict() + for field in fieldnames(Configuration) + merged_kwargs[field] = get(kwargs, field, getfield(cfg, field)) + end + Configuration(; merged_kwargs...) +end From d30ce661b008d637224ccb1213fbd4100e53e860 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 14:22:27 +0200 Subject: [PATCH 17/19] Memoize Julia installs. --- README.md | 2 +- src/PkgEval.jl | 7 +-- src/evaluate.jl | 152 ++++++++++++++++++----------------------------- src/julia.jl | 15 ++++- src/report.jl | 34 ----------- src/rootfs.jl | 44 ++++++++++++++ src/types.jl | 18 +++--- src/utils.jl | 20 +++++++ test/runtests.jl | 19 ++++-- 9 files changed, 163 insertions(+), 148 deletions(-) delete mode 100644 src/report.jl create mode 100644 src/rootfs.jl create mode 100644 src/utils.jl diff --git a/README.md b/README.md index e451862b5..75b788f4a 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ julia> using PkgEval julia> config = Configuration(; julia="1.7"); -julia> interactive_shell(config) +julia> PkgEval.sandboxed_julia(config) _ _ _(_)_ | Documentation: https://docs.julialang.org (_) | (_) (_) | diff --git a/src/PkgEval.jl b/src/PkgEval.jl index 9dbe9e275..325648ae2 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -10,15 +10,12 @@ storage_dir = "" skip_list = String[] retry_list = String[] -# utils -isdebug(group) = - Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing - include("types.jl") include("registry.jl") +include("rootfs.jl") include("julia.jl") include("evaluate.jl") -include("report.jl") +include("utils.jl") function __init__() global download_dir = @get_scratch!("downloads") diff --git a/src/evaluate.jl b/src/evaluate.jl index dfa8e3205..bce6671c0 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -1,51 +1,47 @@ -export evaluate, interactive_shell +export evaluate using Dates using Random using DataFrames: DataFrame, nrow -using LazyArtifacts: @artifact_str using ProgressMeter: Progress, update! using Sandbox: Sandbox, SandboxConfig, UnprivilegedUserNamespacesExecutor, cleanup -lazy_artifact(x) = @artifact_str(x) - -const rootfs_lock = ReentrantLock() -const rootfs_cache = Dict() -function create_rootfs(config::Configuration) - lock(rootfs_lock) do - get!(rootfs_cache, (config.distro, config.uid, config.user, config.gid, config.group, config.home)) do - base = lazy_artifact(config.distro) - - # a bare rootfs isn't usable out-of-the-box - derived = mktempdir() - cp(base, derived; force=true) - - # add a user and group - chmod(joinpath(derived, "etc/passwd"), 0o644) - open(joinpath(derived, "etc/passwd"), "a") do io - println(io, "$(config.user):x:$(config.uid):$(config.gid)::$(config.home):/bin/bash") - end - chmod(joinpath(derived, "etc/group"), 0o644) - open(joinpath(derived, "etc/group"), "a") do io - println(io, "$(config.group):x:$(config.gid):") - end - chmod(joinpath(derived, "etc/shadow"), 0o640) - open(joinpath(derived, "etc/shadow"), "a") do io - println(io, "$(config.user):*:::::::") - end - - # replace resolv.conf - rm(joinpath(derived, "etc/resolv.conf"); force=true) - write(joinpath(derived, "etc/resolv.conf"), read("/etc/resolv.conf")) - - return derived - end - end -end +const statusses = Dict( + :ok => "successful", + :skip => "skipped", + :fail => "unsuccessful", + :kill => "interrupted", +) +const reasons = Dict( + missing => missing, + # skip + :explicit => "package was blacklisted", + :jll => "package is a untestable wrapper package", + :unsupported => "package is not supported by this Julia version", + # fail + :unsatisfiable => "package could not be installed", + :untestable => "package does not have any tests", + :binary_dependency => "package requires a missing binary dependency", + :missing_dependency => "package is missing a package dependency", + :missing_package => "package is using an unknown package", + :test_failures => "package has test failures", + :syntax => "package has syntax issues", + :gc_corruption => "GC corruption detected", + :segfault => "a segmentation fault happened", + :abort => "the process was aborted", + :unreachable => "an unreachable instruction was executed", + :network => "networking-related issues were detected", + :unknown => "there were unidentified errors", + :uncompilable => "compilation of the package failed", + # kill + :time_limit => "test duration exceeded the time limit", + :log_limit => "test log exceeded the size limit", + :inactivity => "tests became inactive", +) """ - sandboxed_julia(config::Configuration, install::String, args=``; env=Dict(), mounts=Dict(), - wait=true, stdin=stdin, stdout=stdout, stderr=stderr, kwargs...) + sandboxed_julia(config::Configuration, args=``; env=Dict(), mounts=Dict(), wait=true, + stdin=stdin, stdout=stdout, stderr=stderr, kwargs...) Run Julia inside of a sandbox, passing the given arguments `args` to it. The argument `wait` determines if the process will be waited on. Streams can be connected using the `stdin`, @@ -54,14 +50,14 @@ determines if the process will be waited on. Streams can be connected using the Further customization is possible using the `env` arg, to set environment variables, and the `mounts` argument to mount additional directories. """ -function sandboxed_julia(config::Configuration, install::String, args=``; wait=true, +function sandboxed_julia(config::Configuration, args=``; wait=true, stdin=stdin, stdout=stdout, stderr=stderr, kwargs...) # XXX: even when preferred_executor() returns UnprivilegedUserNamespacesExecutor, # sometimes a stray sudo happens at run time? no idea how. exe_typ = UnprivilegedUserNamespacesExecutor exe = exe_typ() - cmd = sandboxed_julia_cmd(config, install, exe, args; kwargs...) + cmd = sandboxed_julia_cmd(config, exe, args; kwargs...) proc = run(pipeline(cmd; stdin, stderr, stdout); wait) # TODO: introduce a --stats flag that has the sandbox trace and report on CPU, network, ... usage @@ -86,10 +82,11 @@ end const xvfb_lock = ReentrantLock() const xvfb_proc = Ref{Union{Base.Process,Nothing}}(nothing) -function sandboxed_julia_cmd(config::Configuration, install::String, executor, args=``; +function sandboxed_julia_cmd(config::Configuration, executor, args=``; env::Dict{String,String}=Dict{String,String}(), mounts::Dict{String,String}=Dict{String,String}()) rootfs = create_rootfs(config) + install = install_julia(config) read_only_maps = Dict( "/" => rootfs, config.julia_install_dir => install, @@ -186,15 +183,14 @@ function cpu_time(pid) end """ - sandboxed_script(config::Configuration, install::String, script::String, args=``) + sandboxed_script(config::Configuration, script::String, args=``) Run a Julia script `script` in non-interactive mode, returning the process status and a failure reason if any (both represented by a symbol), and the full log. Refer to `sandboxed_julia`[@ref] for more possible `keyword arguments. """ -function sandboxed_script(config::Configuration, install::String, script::String, args=``; - kwargs...) +function sandboxed_script(config::Configuration, script::String, args=``; kwargs...) @assert config.log_limit > 0 cmd = `--eval 'eval(Meta.parse(read(stdin,String)))' $args` @@ -211,7 +207,7 @@ function sandboxed_script(config::Configuration, install::String, script::String input = Pipe() output = Pipe() - proc = sandboxed_julia(config, install, cmd; env, wait=false, + proc = sandboxed_julia(config, cmd; env, wait=false, stdout=output, stderr=output, stdin=input, kwargs...) close(output.in) @@ -309,16 +305,15 @@ function sandboxed_script(config::Configuration, install::String, script::String end """ - sandboxed_test(config::Configuration, install::String, pkg; kwargs...) + sandboxed_test(config::Configuration, pkg; kwargs...) -Run the unit tests for a single package `pkg` inside of a sandbox using a Julia installation -at `install`. +Run the unit tests for a single package `pkg` inside of a sandbox according to `config`. Refer to `sandboxed_script`[@ref] for more possible `keyword arguments. """ -function sandboxed_test(config::Configuration, install::String, pkg::Package; kwargs...) +function sandboxed_test(config::Configuration, pkg::Package; kwargs...) if config.compiled - return compiled_test(config, install, pkg; kwargs...) + return compiled_test(config, pkg; kwargs...) end script = raw""" @@ -358,7 +353,7 @@ function sandboxed_test(config::Configuration, install::String, pkg::Package; kw args = `--depwarn=error $args` end - status, reason, log = sandboxed_script(config, install, script, args; kwargs...) + status, reason, log = sandboxed_script(config, script, args; kwargs...) # pick up the installed package version from the log version_match = match(Regex("Installed $(pkg.name) .+ v(.+)"), log) @@ -421,7 +416,7 @@ function sandboxed_test(config::Configuration, install::String, pkg::Package; kw end """ - compiled_test(install::String, pkg::Package) + compiled_test(config::Configuration, pkg::Package) Run the unit tests for a single package `pkg` (see `compiled_test`[@ref] for details and a list of supported keyword arguments), after first having compiled a system image that @@ -430,7 +425,7 @@ contains this package and its dependencies. To find incompatibilities, the compilation happens on an Ubuntu-based runner, while testing is performed in an Arch Linux container. """ -function compiled_test(config::Configuration, install::String, pkg::Package; kwargs...) +function compiled_test(config::Configuration, pkg::Package; kwargs...) script = raw""" try using Dates @@ -475,8 +470,7 @@ function compiled_test(config::Configuration, install::String, pkg::Package; kwa compile_config = Configuration(config; time_limit = config.compile_time_limit ) - status, reason, log = sandboxed_script(compile_config, install, script, args; mounts, - kwargs...) + status, reason, log = sandboxed_script(compile_config, script, args; mounts, kwargs...) # try to figure out the failure reason if status === nothing @@ -508,7 +502,7 @@ function compiled_test(config::Configuration, install::String, pkg::Package; kwa home="/home/user", ) version, status, reason, test_log = - sandboxed_test(test_config, install, pkg; mounts, kwargs...) + sandboxed_test(test_config, pkg; mounts, kwargs...) rm(sysimage_dir; recursive=true) return version, status, reason, log * "\n" * test_log @@ -530,17 +524,7 @@ function evaluate(configs::Vector{Configuration}, ninstances::Integer=Sys.CPU_THREADS) # here we deal with managing execution: spawning workers, output, result I/O, etc - # Julia installation - instantiated_configs = Dict() - for config in configs - install = install_julia(config) - # XXX: better get the version from a file in the tree - version_str = chomp(read(`$install/bin/julia --startup-file=no --eval "println(VERSION)"`, String)) - version = parse(VersionNumber, version_str) - instantiated_configs[config] = (install, version) - end - - jobs = vec(collect(Iterators.product(instantiated_configs, packages))) + jobs = vec(collect(Iterators.product(configs, packages))) # use a random test order to (hopefully) get a more reasonable ETA shuffle!(jobs) @@ -627,8 +611,7 @@ function evaluate(configs::Vector{Configuration}, end # NOTE: we expand the Configuration into separate columns - result = DataFrame(julia_spec = String[], - julia_version = VersionNumber[], + result = DataFrame(julia = String[], compiled = Bool[], name = String[], version = Union{Missing,VersionNumber}[], @@ -657,19 +640,19 @@ function evaluate(configs::Vector{Configuration}, push!(all_workers, @async begin try while !isempty(jobs) && !done - (config, (julia_install, julia_version)), pkg = pop!(jobs) + config, pkg = pop!(jobs) times[i] = now() running[i] = (config, pkg) # should we even test this package? if pkg.name in skip_list - push!(result, [config.julia, julia_version, config.compiled, + push!(result, [config.julia, config.compiled, pkg.name, missing, :skip, :explicit, 0, missing]) running[i] = nothing continue elseif endswith(pkg.name, "_jll") - push!(result, [config.julia, julia_version, config.compiled, + push!(result, [config.julia, config.compiled, pkg.name, missing, :skip, :jll, 0, missing]) running[i] = nothing @@ -678,20 +661,19 @@ function evaluate(configs::Vector{Configuration}, # perform an initial run config′ = Configuration(config; cpus=[i-1]) - pkg_version, status, reason, log = - sandboxed_test(config′, julia_install, pkg) + pkg_version, status, reason, log = sandboxed_test(config′, pkg) # certain packages are known to have flaky tests; retry them for j in 1:pkg.retries if status == :fail && reason == :test_failures times[i] = now() pkg_version, status, reason, log = - sandboxed_test(config′, julia_install, pkg) + sandboxed_test(config′, pkg) end end duration = (now()-times[i]) / Millisecond(1000) - push!(result, [config.julia, julia_version, config.compiled, + push!(result, [config.julia, config.compiled, pkg.name, pkg_version, status, reason, duration, log]) running[i] = nothing @@ -708,23 +690,7 @@ function evaluate(configs::Vector{Configuration}, finally stop_work() println() - - # clean-up - for (config, (install,version)) in instantiated_configs - rm(install; recursive=true) - end end return result end - -# TODO: make `install_julia` cache the returned directory so that we don't have to pass -# `install` around. This should make the inner functions directly usable again. -function interactive_shell(config::Configuration) - install = install_julia(config) - try - PkgEval.sandboxed_julia(config, install) - finally - rm(install; recursive=true) - end -end diff --git a/src/julia.jl b/src/julia.jl index 4b2aa47f6..2b3ddc31b 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -230,7 +230,7 @@ function build_julia(repo_path::String; return install_dir end -function install_julia(config::Configuration) +function _install_julia(config::Configuration) if isempty(config.buildflags) # check if it's an official release dir = get_julia_release(config.julia) @@ -262,3 +262,16 @@ function install_julia(config::Configuration) rm(repo; recursive=true) end end + +const julia_lock = ReentrantLock() +const julia_cache = Dict() +function install_julia(config::Configuration) + lock(julia_lock) do + key = (config.julia, config.buildflags) + dir = get(julia_cache, key, nothing) + if dir === nothing || !isdir(dir) + julia_cache[key] = _install_julia(config) + end + return julia_cache[key] + end +end diff --git a/src/report.jl b/src/report.jl deleted file mode 100644 index 3333b202f..000000000 --- a/src/report.jl +++ /dev/null @@ -1,34 +0,0 @@ -# details on status codes - -const statusses = Dict( - :ok => "successful", - :skip => "skipped", - :fail => "unsuccessful", - :kill => "interrupted", -) -const reasons = Dict( - missing => missing, - # skip - :explicit => "package was blacklisted", - :jll => "package is a untestable wrapper package", - :unsupported => "package is not supported by this Julia version", - # fail - :unsatisfiable => "package could not be installed", - :untestable => "package does not have any tests", - :binary_dependency => "package requires a missing binary dependency", - :missing_dependency => "package is missing a package dependency", - :missing_package => "package is using an unknown package", - :test_failures => "package has test failures", - :syntax => "package has syntax issues", - :gc_corruption => "GC corruption detected", - :segfault => "a segmentation fault happened", - :abort => "the process was aborted", - :unreachable => "an unreachable instruction was executed", - :network => "networking-related issues were detected", - :unknown => "there were unidentified errors", - :uncompilable => "compilation of the package failed", - # kill - :time_limit => "test duration exceeded the time limit", - :log_limit => "test log exceeded the size limit", - :inactivity => "tests became inactive", -) diff --git a/src/rootfs.jl b/src/rootfs.jl new file mode 100644 index 000000000..c61d4c126 --- /dev/null +++ b/src/rootfs.jl @@ -0,0 +1,44 @@ +using LazyArtifacts: @artifact_str + +lazy_artifact(x) = @artifact_str(x) + +function _create_rootfs(config::Configuration) + base = lazy_artifact(config.distro) + + # a bare rootfs isn't usable out-of-the-box + derived = mktempdir() + cp(base, derived; force=true) + + # add a user and group + chmod(joinpath(derived, "etc/passwd"), 0o644) + open(joinpath(derived, "etc/passwd"), "a") do io + println(io, "$(config.user):x:$(config.uid):$(config.gid)::$(config.home):/bin/bash") + end + chmod(joinpath(derived, "etc/group"), 0o644) + open(joinpath(derived, "etc/group"), "a") do io + println(io, "$(config.group):x:$(config.gid):") + end + chmod(joinpath(derived, "etc/shadow"), 0o640) + open(joinpath(derived, "etc/shadow"), "a") do io + println(io, "$(config.user):*:::::::") + end + + # replace resolv.conf + rm(joinpath(derived, "etc/resolv.conf"); force=true) + write(joinpath(derived, "etc/resolv.conf"), read("/etc/resolv.conf")) + + return derived +end + +const rootfs_lock = ReentrantLock() +const rootfs_cache = Dict() +function create_rootfs(config::Configuration) + lock(rootfs_lock) do + key = (config.distro, config.uid, config.user, config.gid, config.group, config.home) + dir = get(julia_cache, key, nothing) + if dir === nothing || !isdir(dir) + rootfs_cache[key] = _create_rootfs(config) + end + return rootfs_cache[key] + end +end diff --git a/src/types.jl b/src/types.jl index f2ba10276..49bda9859 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,8 +1,18 @@ export Configuration, Package Base.@kwdef struct Configuration + # Julia properties julia::String = "nightly" buildflags::Vector{String} = String[] + + # rootfs properties + distro::String = "debian" + uid::Int = 1000 + user::String = "pkgeval" + gid::Int = 1000 + group::String = "pkgeval" + home::String = "/home/pkgeval" + depwarn::Bool = false log_limit::Int = 2^20 # 1 MB time_limit = 60*60 # 1 hour @@ -21,14 +31,6 @@ Base.@kwdef struct Configuration # additional Julia arguments to pass to the process julia_args::Cmd = `` - - # rootfs properties - distro::String = "debian" - uid::Int = 1000 - user::String = "pkgeval" - gid::Int = 1000 - group::String = "pkgeval" - home::String = "/home/pkgeval" end Base.@kwdef struct Package diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 000000000..6e007cac3 --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,20 @@ +isdebug(group) = + Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing + +function purge() + lock(julia_lock) do + for dir in values(julia_cache) + rm(dir; recursive=true) + end + empty!(julia_cache) + end + + lock(rootfs_lock) do + for dir in values(rootfs_cache) + rm(dir; recursive=true) + end + empty!(rootfs_cache) + end + + return +end diff --git a/test/runtests.jl b/test/runtests.jl index 7f89dcfc9..9cd68ac09 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,24 +7,29 @@ julia_version = tryparse(VersionNumber, julia) @testset "sandbox" begin config = Configuration(; julia) - install = PkgEval.install_julia(config) let p = Pipe() close(p.in) - PkgEval.sandboxed_julia(config, install, `-e 'print(1337)'`; stdout=p.out) + PkgEval.sandboxed_julia(config, `-e 'print(1337)'`; stdout=p.out) @test read(p.out, String) == "1337" end + + # try to compare the version info + if julia_version !== nothing + p = Pipe() + close(p.in) + PkgEval.sandboxed_julia(config, `-e 'println(VERSION)'`; stdout=p.out) + version_str = read(p.out, String) + @test parse(VersionNumber, version_str) == julia_version + end end @testset "julia installation" begin let results = evaluate([Configuration(; julia)], [Package(; name="Example")]) @test size(results, 1) == 1 - @test results[1, :julia_spec] == julia - if julia_version !== nothing - @test results[1, :julia_version] == julia_version - end + @test results[1, :julia] == julia end end @@ -107,4 +112,6 @@ end end end +PkgEval.purge() + end From cb31372d2acaf1742a9b6efe3f9fe7a6bdd977f9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 14:47:16 +0200 Subject: [PATCH 18/19] Post-process the host triplet to ensure we can fetch a release. --- src/julia.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/julia.jl b/src/julia.jl index 2b3ddc31b..46e2ea880 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -2,6 +2,7 @@ import JSON import Downloads using Git: git using BinaryBuilder: DirectorySource, ExecutableProduct, build_tarballs +using Base.BinaryPlatforms: Platform, triplet const VERSIONS_URL = "https://julialang-s3.julialang.org/bin/versions.json" @@ -42,8 +43,10 @@ function get_julia_release(spec::String) files = versions[string(version_spec)]["files"] # find a file entry for our machine + platform = parse(Platform, Sys.MACHINE) # don't use Sys.MACHINE directly as it may + # contain unnecessary tags, like -pc- i = findfirst(files) do file - file["triplet"] == Sys.MACHINE + file["triplet"] == triplet(platform) end if isnothing(i) @error "Release unavailable for $(Sys.MACHINE)" @@ -101,7 +104,7 @@ end function get_julia_build(repo) @debug "Trying to download a Julia build..." repo_details = get_repo_details(repo) - if Sys.MACHINE == "x86_64-linux-gnu" + if Sys.islinux() && Sys.ARCH == :x86_64 url = "https://julialangnightlies.s3.amazonaws.com/bin/linux/x64/$(repo_details.version.major).$(repo_details.version.minor)/julia-$(repo_details.shorthash)-linux64.tar.gz" else @debug "Don't know how to get build for $(Sys.MACHINE)" @@ -218,7 +221,7 @@ function build_julia(repo_path::String; install_dir = mktempdir() mktempdir() do dir product_hashes = cd(dir) do - build_tarballs(binarybuilder_args, "julia", repo_details.version, sources, + build_tarballs([], "julia", repo_details.version, sources, script, platforms, products, dependencies, preferred_gcc_version=v"7", skip_audit=true, verbose=isdebug(:binarybuilder)) From 578203ec791ad73ea423f4fcd8549e4477ae0ed8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 15:11:14 +0200 Subject: [PATCH 19/19] Update CI hashes. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3daf37be5..ec2c153d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,10 +18,10 @@ jobs: arch: - x64 build_spec: - - v1.7.0 # entry straight in Versions.toml - - nightly # entry from Builds.toml - - 8cb458c6dcd8e067a3bd430b006efb0dfde56cf9 # directly from Git, never built - - master # directly from Git, likely built + - v1.7.0 # release from versions.json + - nightly # special release + - master # directly from Git, likely built by CI + - master~500 # directly from Git, unlikely built by CI env: JULIA_DEBUG: PkgEval JULIA: ${{ matrix.build_spec }}