asyncmap (#27)

close #26

.github/workflows/ci.yml

@@ -14,8 +14,8 @@ concurrency:
 env:
   NBCONVERT_JOBS: '4'
   LITERATE_PROC: '2'
-  EXTRA_ARGS: ''   # Extra arguments for nbconvert
-  CACHE_NUM: '3'
+  ALLOWERRORS: 'false'
+  CACHE_NUM: '1'
   JULIA_CONDAPKG_BACKEND: 'Null'
   JULIA_CI: 'true'
   NBCACHE: '.cache'
@@ -83,18 +83,8 @@ jobs:
           ~/.julia
           !~/.julia/registries
         key: ${{ steps.cache-julia.outputs.cache-primary-key }}
-    - name: Install IJulia kernel and run literate notebooks
-      run: julia --color=yes -p ${{ env.LITERATE_PROC }} --heap-size-hint=4G literate.jl
-    - name: Execute Jupyter notebooks
-      run: >
-        parallel -a ipynbs.txt --joblog job.log -j ${{ env.NBCONVERT_JOBS }} jupyter nbconvert --to notebook
-        --execute ${{ env.EXTRA_ARGS }}
-        --ExecutePreprocessor.timeout=-1
-        --ExecutePreprocessor.kernel_name=julia-1.$(julia -e 'print(VERSION.minor)')
-        --output ${{ github.workspace }}/${{ env.NBCACHE }}/{}
-        {}
-    - name: Show GNU parallel execution stats
-      run: cat job.log
+    - name: Run notebooks
+      run: julia --color=yes -p ${{ env.LITERATE_PROC }} --heap-size-hint=4G run.jl
     - name: Copy back built notebooks
       run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
     - name: Build website

literate.jl → run.jl

@@ -64,8 +64,12 @@ ts = pmap(litnbs; on_error=ex->NaN) do nb
     @elapsed Literate.notebook(nb, outdir; mdstrings=true)
 end
 
+# Show literate notebook execution results
 pretty_table([litnbs ts], header=["Notebook", "Elapsed (s)"])
 
+# Remove worker processes in Distributed.jl
+rmprocs(workers())
+
 # Debug notebooks one by one if there are errors
 for (nb, t) in zip(litnbs, ts)
     if isnan(t)
@@ -85,9 +89,17 @@ any(isnan, ts) && error("Please check literate notebook error(s).")
 # Install IJulia kernel
 IJulia.installkernel("Julia", "--project=@.", "--heap-size-hint=3G")
 
-# Write the list of jupyter notebooks to be executed
-open(outfile, "w") do f
-    for i in ipynbs
-        println(f, i)
-    end
+# nbconvert command array
+ntasks = parse(Int, get(ENV, "NBCONVERT_JOBS", "1"))
+kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)"
+execute = ifelse(get(ENV, "ALLOWERRORS", " ") == "true", "--execute --allow-errors", "--execute")
+timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1")
+cmds = [`jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(joinpath(abspath(pwd()), cachedir, nb)) $(nb)` for nb in ipynbs]
+
+# Run the nbconvert commands in parallel
+ts = asyncmap(cmds; ntasks) do cmd
+    @elapsed run(cmd)
 end
+
+# Print execution result
+pretty_table([ipynbs ts], header=["Notebook", "Elapsed (s)"])