diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6d6bcd2..d15c07a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: shell: julia --color=yes {0} run: | using Pkg, Dates - Pkg.add(["IJulia", "Literate", "PrettyTables"]) + Pkg.add(["IJulia", "Literate", "PrettyTables", "JSON"]) Pkg.activate(".") Pkg.instantiate() Pkg.precompile() diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index d3e40fc3..f5f1e157 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -47,12 +47,15 @@ jobs: python-version: '3.x' - name: Cache python uses: actions/cache@v4 + if: ${{ contains(runner.name, 'GitHub Actions') }} id: cache-py with: - key: ${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} - path: ${{ env.pythonLocation }} + key: ${{ runner.os }}-pip-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} + path: | + ${{ env.pythonLocation }}/lib + ${{ env.pythonLocation }}/bin - name: Install Python dependencies - if: ${{ steps.cache-py.outputs.cache-hit != 'true' }} + if: ${{ !contains(runner.name, 'GitHub Actions') || steps.cache-py.outputs.cache-hit != 'true' }} run: pip install -r requirements.txt - name: Build website run: jupyter-book build ${DIR} --builder linkcheck diff --git a/Manifest.toml b/Manifest.toml index 656b1e38..9233ce25 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -83,9 +83,9 @@ version = "7.9.0" [[deps.ArrayLayouts]] deps = ["FillArrays", "LinearAlgebra"] -git-tree-sha1 = "0330bc3e828a05d1073553fb56f9695d73077370" +git-tree-sha1 = "33207a8be6267bc389d0701e97a9bce6a4de68eb" uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" -version = "1.9.1" +version = "1.9.2" weakdeps = ["SparseArrays"] [deps.ArrayLayouts.extensions] @@ -166,9 +166,9 @@ version = "3.4.2" [[deps.CSV]] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] -git-tree-sha1 = "a44910ceb69b0d44fe262dd451ab11ead3ed0be8" +git-tree-sha1 = "6c834533dc1fabd820c1db03c839bf97e45a3fab" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "0.10.13" +version = "0.10.14" [[deps.Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] @@ -346,9 +346,9 @@ version = "1.6.1" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "0f4b5d62a88d8f59003e43c25a8a90de9eb76317" +git-tree-sha1 = "97d79461925cdb635ee32116978fc735b9463a39" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.18" +version = "0.18.19" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -494,9 +494,9 @@ version = "0.6.8" [[deps.DynamicPolynomials]] deps = ["Future", "LinearAlgebra", "MultivariatePolynomials", "MutableArithmetics", "Pkg", "Reexport", "Test"] -git-tree-sha1 = "0bb0a6f812213ecc8fbbcf472f4a993036858971" +git-tree-sha1 = "0c056035f7de73b203a5295a22137f96fc32ad46" uuid = "7c1d4256-1411-5781-91ec-d7bc3513ac07" -version = "0.5.5" +version = "0.5.6" [[deps.EnumX]] git-tree-sha1 = "bdb1942cd4c45e3c678fd11569d5cccd80976237" @@ -859,9 +859,9 @@ version = "3.0.2+0" [[deps.JuliaFormatter]] deps = ["CSTParser", "CommonMark", "DataStructures", "Glob", "Pkg", "PrecompileTools", "Tokenize"] -git-tree-sha1 = "e57262abcc8463dc8676b4bcc2ef07df40e4986a" +git-tree-sha1 = "1c4880cb70a5c6c87ea36deccc3d7f9e7969c18c" uuid = "98e50ef6-434e-11e9-1051-2b60c6c9e899" -version = "1.0.55" +version = "1.0.56" [[deps.JumpProcesses]] deps = ["ArrayInterface", "DataStructures", "DiffEqBase", "DocStringExtensions", "FunctionWrappers", "Graphs", "LinearAlgebra", "Markdown", "PoissonRandom", "Random", "RandomNumbers", "RecursiveArrayTools", "Reexport", "SciMLBase", "StaticArrays", "SymbolicIndexingInterface", "UnPack"] @@ -1149,9 +1149,9 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MatrixFactorizations]] deps = ["ArrayLayouts", "LinearAlgebra", "Printf", "Random"] -git-tree-sha1 = "fd4530d3c921d85c2fefce7e75d34d7acfefca61" +git-tree-sha1 = "6731e0574fa5ee21c02733e397beb133df90de35" uuid = "a3b82374-2e81-5b9e-98ce-41277c0e4c87" -version = "2.1.1" +version = "2.2.0" [[deps.MaybeInplace]] deps = ["ArrayInterface", "LinearAlgebra", "MacroTools", "SparseArrays"] @@ -1404,9 +1404,9 @@ version = "1.4.1" [[deps.Plots]] deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "UnitfulLatexify", "Unzip"] -git-tree-sha1 = "3bdfa4fa528ef21287ef659a89d686e8a1bcb1a9" +git-tree-sha1 = "442e1e7ac27dd5ff8825c3fa62fbd1e86397974b" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.40.3" +version = "1.40.4" [deps.Plots.extensions] FileIOExt = "FileIO" @@ -1566,9 +1566,9 @@ version = "3.13.0" [[deps.RecursiveFactorization]] deps = ["LinearAlgebra", "LoopVectorization", "Polyester", "PrecompileTools", "StrideArraysCore", "TriangularSolve"] -git-tree-sha1 = "8bc86c78c7d8e2a5fe559e3721c0f9c9e303b2ed" +git-tree-sha1 = "c04dacfc546591d43c39dc529c922d6a06a5a694" uuid = "f2c3362d-daeb-58d1-803e-2bc74f2840b4" -version = "0.2.21" +version = "0.2.22" [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" @@ -1628,9 +1628,9 @@ version = "0.6.42" [[deps.SciMLBase]] deps = ["ADTypes", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "SciMLStructures", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"] -git-tree-sha1 = "d15c65e25615272e1b1c5edb1d307484c7942824" +git-tree-sha1 = "8aed4375fa906e0248b422b71c059a2a57d882ae" uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" -version = "2.31.0" +version = "2.32.0" [deps.SciMLBase.extensions] SciMLBaseChainRulesCoreExt = "ChainRulesCore" @@ -1855,10 +1855,10 @@ uuid = "789caeaf-c7a9-5a7d-9973-96adeb23e2a0" version = "6.65.1" [[deps.StrideArraysCore]] -deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface", "ThreadingUtilities"] -git-tree-sha1 = "d6415f66f3d89c615929af907fdc6a3e17af0d8c" +deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface", "ThreadingUtilities"] +git-tree-sha1 = "b164d4dc04d7072066b725b7906e56331b170004" uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da" -version = "0.5.2" +version = "0.5.3" [[deps.StringManipulation]] deps = ["PrecompileTools"] @@ -1889,9 +1889,9 @@ version = "5.2.2+0" [[deps.SymbolicIndexingInterface]] deps = ["Accessors", "ArrayInterface", "MacroTools", "RuntimeGeneratedFunctions", "StaticArraysCore"] -git-tree-sha1 = "4b7f4c80449d8baae8857d55535033981862619c" +git-tree-sha1 = "40ea524431a92328cd73582d1820a5b08247a40f" uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5" -version = "0.3.15" +version = "0.3.16" [[deps.SymbolicLimits]] deps = ["SymbolicUtils"] @@ -1907,9 +1907,9 @@ version = "1.5.1" [[deps.Symbolics]] deps = ["ArrayInterface", "Bijections", "ConstructionBase", "DataStructures", "DiffRules", "Distributions", "DocStringExtensions", "DomainSets", "DynamicPolynomials", "ForwardDiff", "IfElse", "LaTeXStrings", "LambertW", "Latexify", "Libdl", "LinearAlgebra", "LogExpFunctions", "MacroTools", "Markdown", "NaNMath", "PrecompileTools", "RecipesBase", "Reexport", "Requires", "RuntimeGeneratedFunctions", "SciMLBase", "Setfield", "SparseArrays", "SpecialFunctions", "StaticArrays", "SymbolicIndexingInterface", "SymbolicLimits", "SymbolicUtils"] -git-tree-sha1 = "280c17e091a24283a59eedfc00a02026ec984b09" +git-tree-sha1 = "4104548fff14d7370b278ee767651d6ec61eb195" uuid = "0c5d862f-8b57-4792-8d23-62f2024744c7" -version = "5.27.1" +version = "5.28.0" [deps.Symbolics.extensions] SymbolicsGroebnerExt = "Groebner" diff --git a/run.jl b/run.jl index 1f12ab64..1691e30c 100644 --- a/run.jl +++ b/run.jl @@ -1,4 +1,3 @@ - using Distributed using PrettyTables using SHA @@ -6,96 +5,142 @@ using IJulia @everywhere begin ENV["GKSwstype"] = "100" - using Literate, Pkg + using Literate, Pkg, JSON Pkg.activate(Base.current_project()) end -basedir = get(ENV, "DOCDIR", "docs") # Defaults to docs/ -cachedir = get(ENV, "NBCACHE", ".cache") # Defaults to .cache/ -ipynbs = String[] -litnbs = String[] - -# Collect the list of notebooks -for (root, dirs, files) in walkdir(basedir) - for file in files - if endswith(file, ".ipynb") || endswith(file, ".jl") - nb = joinpath(root, file) - shaval = read(nb, String) |> sha256 |> bytes2hex - @info "Notebook $(nb): hash=$(shaval)" - shafilename = joinpath(cachedir, root, splitext(file)[1] * ".sha") - # Cache hit - if isfile(shafilename) && read(shafilename, String) == shaval - @info "Notebook $(nb) cache hits and will not be executed." - # Cache miss - else - @info "Notebook $(nb) cache misses. Writing hash to $(shafilename)." - mkpath(dirname(shafilename)) - write(shafilename, shaval) - if endswith(file, ".ipynb") - push!(ipynbs, nb) - elseif endswith(file, ".jl") - push!(litnbs, nb) - end +# Strip SVG output from a Jupyter notebook +@everywhere function strip_svg(ipynb) + @info "Stripping SVG in $(ipynb)" + nb = open(JSON.parse, ipynb, "r") + for cell in nb["cells"] + !haskey(cell, "outputs") && continue + for output in cell["outputs"] + !haskey(output, "data") && continue + datadict = output["data"] + if haskey(datadict, "image/png") || haskey(datadict, "image/jpeg") + delete!(datadict, "text/html") + delete!(datadict, "image/svg+xml") end end end + rm(ipynb) + open(ipynb, "w") do io + JSON.print(io, nb, 1) + end + return ipynb end # Remove cached notebook and sha files if there is no corresponding notebook -for (root, dirs, files) in walkdir(cachedir) - for file in files - if endswith(file, ".ipynb") || endswith(file, ".sha") - fn = joinpath(joinpath(splitpath(root)[2:end]), splitext(file)[1]) - nb = fn * ".ipynb" - lit = fn * ".jl" - if !isfile(nb) && !isfile(lit) - fullfn = joinpath(root, file) - @info "Notebook $(nb) or $(lit) not found. Removing $(fullfn)." - rm(fullfn) +function clean_cache(cachedir) + for (root, dirs, files) in walkdir(cachedir) + for file in files + if endswith(file, ".ipynb") || endswith(file, ".sha") + fn = joinpath(joinpath(splitpath(root)[2:end]), splitext(file)[1]) + nb = fn * ".ipynb" + lit = fn * ".jl" + if !isfile(nb) && !isfile(lit) + fullfn = joinpath(root, file) + @info "Notebook $(nb) or $(lit) not found. Removing $(fullfn)." + rm(fullfn) + end + end + end + end +end + +function list_notebooks(basedir, cachedir) + ipynbs = String[] + litnbs = String[] + + for (root, dirs, files) in walkdir(basedir) + for file in files + if endswith(file, ".ipynb") || endswith(file, ".jl") + nb = joinpath(root, file) + shaval = read(nb, String) |> sha256 |> bytes2hex + @info "Notebook $(nb): hash=$(shaval)" + shafilename = joinpath(cachedir, root, splitext(file)[1] * ".sha") + # Cache hit + if isfile(shafilename) && read(shafilename, String) == shaval + @info "Notebook $(nb) cache hits and will not be executed." + # Cache miss + else + @info "Notebook $(nb) cache misses. Writing hash to $(shafilename)." + mkpath(dirname(shafilename)) + write(shafilename, shaval) + if endswith(file, ".ipynb") + push!(ipynbs, nb) + elseif endswith(file, ".jl") + push!(litnbs, nb) + end + end end end end + return (;ipynbs, litnbs) end -# Execute literate notebooks in worker process(es) -ts_lit = pmap(litnbs; on_error=ex->NaN) do nb - outdir = joinpath(cachedir, dirname(nb)) - @elapsed Literate.notebook(nb, outdir; mdstrings=true) +@everywhere function run_literate(file, cachedir; rmsvg=true) + outpath = joinpath(abspath(pwd()), cachedir, dirname(file)) + mkpath(outpath) + ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true) + rmsvg && strip_svg(ipynb) + return ipynb end -# Remove worker processes in Distributed.jl -rmprocs(workers()) +function main(; + basedir=get(ENV, "DOCDIR", "docs"), + cachedir=get(ENV, "NBCACHE", ".cache"), printtable=true, + rmsvg=true + ) + + mkpath(cachedir) + clean_cache(cachedir) + + (;ipynbs, litnbs) = list_notebooks(basedir, cachedir) + + # Execute literate notebooks in worker process(es) + ts_lit = pmap(litnbs; on_error=ex -> NaN) do nb + @elapsed run_literate(nb, cachedir; rmsvg) + end + + # Remove worker processes to release some memory + rmprocs(workers()) -# Debug notebooks one by one if there are errors -for (nb, t) in zip(litnbs, ts_lit) - if isnan(t) - println("Debugging notebook: ", nb) - try - withenv("JULIA_DEBUG" => "Literate") do - Literate.notebook(nb, dirname(nb); mdstrings=true) + # Debug notebooks one by one if there are errors + for (nb, t) in zip(litnbs, ts_lit) + if isnan(t) + println("Debugging notebook: ", nb) + try + withenv("JULIA_DEBUG" => "Literate") do + run_literate(nb, cachedir; rmsvg) + end + catch e + println(e) end - catch e - println(e) end end -end -any(isnan, ts_lit) && error("Please check literate notebook error(s).") + any(isnan, ts_lit) && error("Please check literate notebook error(s).") -# Install IJulia kernel -IJulia.installkernel("Julia", "--project=@.", "--heap-size-hint=3G") + # Install IJulia kernel + IJulia.installkernel("Julia", "--project=@.", "--heap-size-hint=3G") -# nbconvert command array -ntasks = parse(Int, get(ENV, "NBCONVERT_JOBS", "1")) -kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)" -execute = ifelse(get(ENV, "ALLOWERRORS", " ") == "true", "--execute --allow-errors", "--execute") -timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1") -cmds = [`jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(joinpath(abspath(pwd()), cachedir, nb)) $(nb)` for nb in ipynbs] + # nbconvert command array + ntasks = parse(Int, get(ENV, "NBCONVERT_JOBS", "1")) + kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)" + execute = ifelse(get(ENV, "ALLOWERRORS", " ") == "true", "--execute --allow-errors", "--execute") + timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1") + cmds = [`jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(joinpath(abspath(pwd()), cachedir, nb)) $(nb)` for nb in ipynbs] + + # Run the nbconvert commands in parallel + ts_ipynb = asyncmap(cmds; ntasks) do cmd + @elapsed run(cmd) + end -# Run the nbconvert commands in parallel -ts_ipynb = asyncmap(cmds; ntasks) do cmd - @elapsed run(cmd) + # Print execution result + printtable && pretty_table([litnbs ts_lit; ipynbs ts_ipynb], header=["Notebook", "Elapsed (s)"]) end -# Print execution result -pretty_table([litnbs ts_lit; ipynbs ts_ipynb], header=["Notebook", "Elapsed (s)"]) +# Run code +main()