Add literate CI (#25)

.github/workflows/ci.yml

@@ -12,9 +12,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  NJOBS: '4'
+  NBCONVERT_JOBS: '4'
+  LITERATE_PROC: '2'
   EXTRA_ARGS: ''   # Extra arguments for nbconvert
-  CACHE_NUM: '1'
+  CACHE_NUM: '2'
   JULIA_CONDAPKG_BACKEND: 'Null'
   JULIA_CI: 'true'
   NBCACHE: '.cache'
@@ -44,9 +45,9 @@ jobs:
       id: cache-nb
       with:
         path: ${{ env.NBCACHE }}
-        key: ${{ runner.os }}-nb-${{ env.CACHE_NUM }}-${{ hashFiles('src/**','Project.toml', 'Manifest.toml') }}-${{ hashFiles('docs/**/*.ipynb') }}
+        key: ${{ runner.os }}-nb-${{ env.CACHE_NUM }}-${{ hashFiles('src/**', 'Project.toml', 'Manifest.toml') }}-${{ hashFiles('docs/**/*.ipynb', 'docs/**/*.jl') }}
         restore-keys: |
-          ${{ runner.os }}-nb-${{ env.CACHE_NUM }}-${{ hashFiles('src/**','Project.toml', 'Manifest.toml') }}-
+          ${{ runner.os }}-nb-${{ env.CACHE_NUM }}-${{ hashFiles('src/**', 'Project.toml', 'Manifest.toml') }}-
     - name: Read Julia version
       uses: SebRollen/[email protected]
       id: read_toml
@@ -58,35 +59,42 @@ jobs:
       with:
         version: ${{ steps.read_toml.outputs.value }}
         show-versioninfo: 'true'
-    - name: Cache Julia packages
-      uses: actions/cache@v4
+    - name: Restore Julia packages
+      uses: actions/cache/restore@v4
       id: cache-julia
       with:
         path: |
           ~/.julia
           !~/.julia/registries
-        key: ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ hashFiles('src/**','Project.toml', 'Manifest.toml') }}
+        key: ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ hashFiles('src/**', 'Project.toml', 'Manifest.toml') }}
         restore-keys: |
           ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-
     - name: Install Julia packages
       if: ${{ steps.cache-julia.outputs.cache-hit != 'true' }}
-      run: |
-        julia --color=yes -e 'using Pkg; Pkg.add(["IJulia"])'
-        julia --color=yes --project=@. -e 'using Pkg, Dates; Pkg.instantiate(); Pkg.precompile(); Pkg.gc(collect_delay=Day(0))'
-    - name: Install IJulia kernel and calculate cache
-      run: |
-        julia --color=yes -e 'using IJulia; IJulia.installkernel("Julia", "--project=@.", "--heap-size-hint=3G")'
-        julia --color=yes caching.jl
-    - name: Execute Notebooks
+      env:
+        PYTHON: ${{ env.pythonLocation }}/python
+      run: julia --color=yes instantiate.jl
+    - name: Save Julia packages
+      uses: actions/cache/save@v4
+      if: ${{ steps.cache-julia.outputs.cache-hit != 'true' }}
+      with:
+        path: |
+          ~/.julia
+          !~/.julia/registries
+        key: ${{ steps.cache-julia.outputs.cache-primary-key }}
+    - name: Install IJulia kernel and run literate notebooks
+      if: ${{ steps.cache-nb.outputs.cache-hit != 'true' }}
+      run: julia --color=yes -p ${{ env.LITERATE_PROC }} --heap-size-hint=4G literate.jl
+    - name: Execute Jupyter notebooks
       run: >
-        parallel -a nbs.txt --joblog /tmp/job.log -j${NJOBS} jupyter nbconvert --to notebook
+        parallel -a ipynbs.txt --joblog job.log -j ${{ env.NBCONVERT_JOBS }} jupyter nbconvert --to notebook
         --execute ${{ env.EXTRA_ARGS }}
         --ExecutePreprocessor.timeout=-1
         --ExecutePreprocessor.kernel_name=julia-1.$(julia -e 'print(VERSION.minor)')
         --output ${{ github.workspace }}/${{ env.NBCACHE }}/{}
         {}
-    - name: Show execution stats
-      run: cat /tmp/job.log
+    - name: Show GUN parallel execution stats
+      run: cat job.log
     - name: Copy back built notebooks
       run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
     - name: Build website

docs/_toc.yml

@@ -5,4 +5,6 @@ format: jb-book
 root: index
 chapters:
   - file: plots
+  - file: pythonplot
   - file: sub/plots
+  - file: sub/plots-lit

docs/pythonplot.jl

@@ -0,0 +1,23 @@
+#===
+# Plotting with PythonPlot.jl
+===#
+
+import PythonPlot as plt
+using Random
+Random.seed!(2022)
+
+#---
+
+plt.figure()
+plt.plot(1:5, rand(1:6, 5))
+plt.gcf()
+
+# ## Runtime information
+
+import Pkg
+Pkg.status()
+
+#---
+
+import InteractiveUtils
+InteractiveUtils.versioninfo()

docs/sub/plots-lit.jl

@@ -0,0 +1,22 @@
+#===
+# Plotting by Plots.jl
+
+===#
+
+using Plots
+using Random
+Random.seed!(2022)
+
+#---
+
+plot(rand(1:6, 5))
+
+# ## Runtime information
+
+import Pkg
+Pkg.status()
+
+#---
+
+import InteractiveUtils
+InteractiveUtils.versioninfo()

instantiate.jl

@@ -0,0 +1,7 @@
+using Pkg, Dates
+
+Pkg.add(["IJulia", "Literate", "PrettyTables"])
+Pkg.activate(".")
+Pkg.instantiate()
+Pkg.precompile()
+Pkg.gc(collect_delay=Day(0))

literate.jl

@@ -0,0 +1,92 @@
+
+using Distributed
+using PrettyTables
+using SHA
+using IJulia
+
+@everywhere begin
+    ENV["GKSwstype"] = "100"
+    using Literate, Pkg
+    Pkg.activate(Base.current_project())
+end
+
+outfile = "ipynbs.txt"
+basedir = get(ENV, "DOCDIR", "docs") # Defaults to docs/
+cachedir = get(ENV, "NBCACHE", ".cache") # Defaults to .cache/
+ipynbs = String[]
+litnbs = String[]
+
+# Collect the list of notebooks
+for (root, dirs, files) in walkdir(basedir)
+    for file in files
+        if endswith(file, ".ipynb") || endswith(file, ".jl")
+            nb = joinpath(root, file)
+            shaval = read(nb, String) |> sha256 |> bytes2hex
+            @info "Notebook $(nb): hash=$(shaval)"
+            shafilename = joinpath(cachedir, root, splitext(file)[1] * ".sha")
+            # Cache hit
+            if isfile(shafilename) && read(shafilename, String) == shaval
+                @info "Notebook $(nb) cache hits and will not be executed."
+            # Cache miss
+            else
+                @info "Notebook $(nb) cache misses. Writing hash to $(shafilename)."
+                mkpath(dirname(shafilename))
+                write(shafilename, shaval)
+                if endswith(file, ".ipynb")
+                    push!(ipynbs, nb)
+                elseif endswith(file, ".jl")
+                    push!(litnbs, nb)
+                end
+            end
+        end
+    end
+end
+
+# Remove cached notebook and sha files if there is no respective notebook
+for (root, dirs, files) in walkdir(cachedir)
+    for file in files
+        if endswith(file, ".ipynb") || endswith(file, ".sha")
+            fn = joinpath(joinpath(splitpath(root)[2:end]), splitext(file)[1])
+            nb = fn * ".ipynb"
+            lit = fn * ".jl"
+            if !isfile(nb) && !isfile(lit)
+                fullfn = joinpath(root, file)
+                @info "Notebook $(nb) or $(lit) not found. Removing $(fullfn)."
+                rm(fullfn)
+            end
+        end
+    end
+end
+
+# Execute literate notebooks in worker process(es)
+ts = pmap(litnbs; on_error=ex->NaN) do nb
+    @elapsed Literate.notebook(nb, dirname(nb); mdstrings=true)
+end
+
+pretty_table([litnbs ts], header=["Notebook", "Elapsed (s)"])
+
+# Debug notebooks one by one if there are errors
+for (nb, t) in zip(litnbs, ts)
+    if isnan(t)
+        println("Debugging notebook: ", nb)
+        try
+            withenv("JULIA_DEBUG" => "Literate") do
+                Literate.notebook(nb, dirname(nb); mdstrings=true)
+            end
+        catch e
+            println(e)
+        end
+    end
+end
+
+any(isnan, ts) && error("Please check literate notebook error(s).")
+
+# Install IJulia kernel
+IJulia.installkernel("Julia", "--project=@.", "--heap-size-hint=3G")
+
+# Write the list of jupyter notebooks to be executed
+open(outfile, "w") do f
+    for i in ipynbs
+        println(f, i)
+    end
+end