From c084a638005d2af4e2d70a5b9caca59791db8cb6 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 16:07:01 +0200 Subject: [PATCH 1/4] Prototype rr integration. --- src/evaluate.jl | 65 +++++++++++++++++++++++++++++++++++++++++++++--- src/types.jl | 1 + test/runtests.jl | 18 +++++++++++--- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/evaluate.jl b/src/evaluate.jl index bce6671c0..c5c9c882f 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -335,25 +335,62 @@ function sandboxed_test(config::Configuration, pkg::Package; kwargs...) print("\n\n", '#'^80, "\n# Testing: $(now())\n#\n\n") - Pkg.test(package_spec.name) + if get(ENV, "PKGEVAL_RR", "false") == "true" + Pkg.test(package_spec.name; julia_args=`--bug-report=rr-local`) + else + Pkg.test(package_spec.name) + end println("\nPkgEval succeeded") + catch err print("\nPkgEval failed: ") showerror(stdout, err) Base.show_backtrace(stdout, catch_backtrace()) println() + + if get(ENV, "PKGEVAL_RR", "false") == "true" + print("\n\n", '#'^80, "\n# BugReporting post-processing: $(now())\n#\n\n") + + # pack-up our rr trace. this is expensive, so we only do it for failures. + # it also needs to happen in a clean environment, or BugReporting's deps + # could affect/be affected by the tested package's dependencies. + Pkg.activate(; temp=true) + Pkg.add("BugReporting") + try + using BugReporting + trace_dir = BugReporting.default_rr_trace_dir() + trace = BugReporting.find_latest_trace(trace_dir) + BugReporting.compress_trace(trace, "/traces/$(ARGS[1]).tar.zst") + println("\nBugReporting succeeded") + catch err + print("\nBugReporting failed: ") + showerror(stdout, err) + Base.show_backtrace(stdout, catch_backtrace()) + println() + end + end finally print("\n\n", '#'^80, "\n# PkgEval teardown: $(now())\n#\n\n") end""" - # generate a PackageSpec we'll use to install the package args = `$(repr(package_spec_tuple(pkg)))` if config.depwarn args = `--depwarn=error $args` end - status, reason, log = sandboxed_script(config, script, args; kwargs...) + mounts = Dict{String,String}() + env = Dict{String,String}() + if config.rr + trace_dir = mktempdir() + trace_file = joinpath(trace_dir, "$(pkg.name).tar.zst") + mounts["/traces"] = trace_dir + env["PKGEVAL_RR"] = "true" + haskey(ENV, "PKGEVAL_RR_BUCKET") || + @warn maxlog=1 "PKGEVAL_RR_BUCKET not set; will not be uploading rr traces" + end + + status, reason, log = sandboxed_script(config, script, args; mounts, env, kwargs...) # pick up the installed package version from the log version_match = match(Regex("Installed $(pkg.name) .+ v(.+)"), log) @@ -412,6 +449,28 @@ function sandboxed_test(config::Configuration, pkg::Package; kwargs...) end end + if config.rr + # upload an rr trace for interesting failures + # TODO: re-use BugReporting.jl + if status == :fail && reason in [:gc_corruption, :segfault, :abort, :unreachable] && + haskey(ENV, "PKGEVAL_RR_BUCKET") + bucket = ENV["PKGEVAL_RR_BUCKET"] + unixtime = round(Int, datetime2unix(now())) + trace_unique_name = "$(pkg.name)-$(unixtime).tar.zst" + if isfile(trace_file) + f = retry(delays=Base.ExponentialBackOff(n=5, first_delay=5, max_delay=300)) do + Base.run(`s3cmd put --quiet $trace_file s3://$(bucket)/$(trace_unique_name)`) + Base.run(`s3cmd setacl --quiet --acl-public s3://$(bucket)/$(trace_unique_name)`) + end + f() + log *= "Uploaded rr trace to https://s3.amazonaws.com/$(bucket)/$(trace_unique_name)" + else + log *= "Testing did not produce an rr trace." + end + end + rm(trace_dir; recursive=true) + end + return version, status, reason, log end diff --git a/src/types.jl b/src/types.jl index 49bda9859..4be0bc7b1 100644 --- a/src/types.jl +++ b/src/types.jl @@ -18,6 +18,7 @@ Base.@kwdef struct Configuration time_limit = 60*60 # 1 hour compiled::Bool = false compile_time_limit::Int = 30*60 # 30 mins + rr::Bool = false # the directory where Julia is installed in the run-time environment julia_install_dir::String = "/opt/julia" diff --git a/test/runtests.jl b/test/runtests.jl index 9cd68ac09..df899a507 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -104,11 +104,21 @@ end @testset "PackageCompiler" begin results = evaluate([Configuration(; julia, compiled=true)], [Package(; name="Example")]) + @test size(results, 1) == 1 if !(julia == "master" || julia == "nightly") - @test all(results.status .== :ok) - for result in eachrow(results) - @test occursin("Testing $(result.name) tests passed", result.log) - end + @test results[1, :status] == :ok + @test contains(results[1, :log], "Testing Example tests passed") + end +end + +@testset "rr" begin + results = evaluate([Configuration(; julia, rr=true)], + [Package(; name="Example")]) + @test all(results.status .== :ok) + @test contains(results[1, :log], "BugReporting") + if !(julia == "master" || julia == "nightly") + @test results[1, :status] == :ok + @test contains(results[1, :log], "Testing Example tests passed") end end From 5b74dc98cff5e19cfe0f83b3258f756911958613 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 17:01:19 +0200 Subject: [PATCH 2/4] Fix environment variable passing. --- src/evaluate.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/evaluate.jl b/src/evaluate.jl index c5c9c882f..518b84a29 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -190,17 +190,20 @@ failure reason if any (both represented by a symbol), and the full log. Refer to `sandboxed_julia`[@ref] for more possible `keyword arguments. """ -function sandboxed_script(config::Configuration, script::String, args=``; kwargs...) +function sandboxed_script(config::Configuration, script::String, args=``; + env::Dict{String,String}=Dict{String,String}(), kwargs...) @assert config.log_limit > 0 cmd = `--eval 'eval(Meta.parse(read(stdin,String)))' $args` - env = Dict( + env = merge(env, Dict( + # we're likely running many instances, so avoid overusing the CPU "JULIA_PKG_PRECOMPILE_AUTO" => "0", + # package hacks "PYTHON" => "", "R_HOME" => "*" - ) + )) if haskey(ENV, "JULIA_PKG_SERVER") env["JULIA_PKG_SERVER"] = ENV["JULIA_PKG_SERVER"] end From 48fa23997820e017eda7bec13771b6cf9183ffe3 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 17:32:10 +0200 Subject: [PATCH 3/4] Cache packages and their compilecache across invocations. Hopefully this improves the package load time a bit. --- src/PkgEval.jl | 2 ++ src/evaluate.jl | 34 ++++++++++++++++++++++++++-------- src/utils.jl | 22 +++++++++++++++++++--- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/PkgEval.jl b/src/PkgEval.jl index 325648ae2..331ff6583 100644 --- a/src/PkgEval.jl +++ b/src/PkgEval.jl @@ -22,6 +22,8 @@ function __init__() mkpath(joinpath(download_dir, "srccache")) global storage_dir = @get_scratch!("storage") + mkpath(joinpath(storage_dir, "artifacts")) + mkpath(joinpath(storage_dir, "packages")) # read Packages.toml packages = TOML.parsefile(joinpath(dirname(@__DIR__), "Packages.toml")) diff --git a/src/evaluate.jl b/src/evaluate.jl index 518b84a29..ede20ebdd 100644 --- a/src/evaluate.jl +++ b/src/evaluate.jl @@ -39,6 +39,20 @@ const reasons = Dict( :inactivity => "tests became inactive", ) +const compiled_lock = ReentrantLock() +const compiled_cache = Dict() +function get_compilecache(config::Configuration) + lock(compiled_lock) do + key = (config.julia, config.buildflags, + config.distro, config.uid, config.user, config.gid, config.group, config.home) + dir = get(compiled_cache, key, nothing) + if dir === nothing || !isdir(dir) + compiled_cache[key] = mktempdir() + end + return compiled_cache[key] + end +end + """ sandboxed_julia(config::Configuration, args=``; env=Dict(), mounts=Dict(), wait=true, stdin=stdin, stdout=stdout, stderr=stderr, kwargs...) @@ -87,16 +101,20 @@ function sandboxed_julia_cmd(config::Configuration, executor, args=``; mounts::Dict{String,String}=Dict{String,String}()) rootfs = create_rootfs(config) install = install_julia(config) + registries = joinpath(first(DEPOT_PATH), "registries") read_only_maps = Dict( - "/" => rootfs, - config.julia_install_dir => install, - "/usr/local/share/julia/registries" => joinpath(first(DEPOT_PATH), "registries"), + "/" => rootfs, + config.julia_install_dir => install, + "/usr/local/share/julia/registries" => registries ) - artifacts_path = joinpath(storage_dir, "artifacts") - mkpath(artifacts_path) + compiled = get_compilecache(config) + packages = joinpath(storage_dir, "packages") + artifacts = joinpath(storage_dir, "artifacts") read_write_maps = merge(mounts, Dict( - joinpath(config.home, ".julia/artifacts") => artifacts_path + joinpath(config.home, ".julia", "compiled") => compiled, + joinpath(config.home, ".julia", "packages") => packages, + joinpath(config.home, ".julia", "artifacts") => artifacts )) env = merge(env, Dict( @@ -107,7 +125,7 @@ function sandboxed_julia_cmd(config::Configuration, executor, args=``; # use the provided registry # NOTE: putting a registry in a non-primary depot entry makes Pkg use it as-is, - # without needingb to set Pkg.UPDATED_REGISTRY_THIS_SESSION. + # without needing to set Pkg.UPDATED_REGISTRY_THIS_SESSION. "JULIA_DEPOT_PATH" => "::/usr/local/share/julia", # some essential env vars (since we don't run from a shell) @@ -396,7 +414,7 @@ function sandboxed_test(config::Configuration, pkg::Package; kwargs...) status, reason, log = sandboxed_script(config, script, args; mounts, env, kwargs...) # pick up the installed package version from the log - version_match = match(Regex("Installed $(pkg.name) .+ v(.+)"), log) + version_match = match(Regex("\\+ $(pkg.name) v(\\S+)"), log) version = if version_match !== nothing try VersionNumber(version_match.captures[1]) diff --git a/src/utils.jl b/src/utils.jl index 6e007cac3..18dacfd14 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,7 +1,23 @@ isdebug(group) = Base.CoreLogging.current_logger_for_env(Base.CoreLogging.Debug, group, PkgEval) !== nothing +""" + PkgEval.purge() + +Remove temporary files and folders that are unlikely to be re-used in the future, e.g., +temporary Julia installs or compilation cache of packages. + +Artifacts that are more likely to be re-used in the future, e.g., downloaded Julia builds +or check-outs of Git repositories, are saved in scratch spaces instead. +""" function purge() + lock(rootfs_lock) do + for dir in values(rootfs_cache) + rm(dir; recursive=true) + end + empty!(rootfs_cache) + end + lock(julia_lock) do for dir in values(julia_cache) rm(dir; recursive=true) @@ -9,11 +25,11 @@ function purge() empty!(julia_cache) end - lock(rootfs_lock) do - for dir in values(rootfs_cache) + lock(compiled_lock) do + for dir in values(compiled_cache) rm(dir; recursive=true) end - empty!(rootfs_cache) + empty!(compiled_cache) end return From b4e6a397efdb77f17ecf3faf7a303d4f843bfd6b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 29 Jul 2022 18:18:26 +0200 Subject: [PATCH 4/4] Disable rr testing on CI. --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index df899a507..2376eb4dc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -111,7 +111,7 @@ end end end -@testset "rr" begin +haskey(ENV, "CI") || @testset "rr" begin results = evaluate([Configuration(; julia, rr=true)], [Package(; name="Example")]) @test all(results.status .== :ok)