Revert "test no docker (#250)"

This reverts commit f35fabb.
sosiristseng · Oct 14, 2024 · c175364 · c175364
1 parent c56ce9a
commit c175364
Show file tree

Hide file tree

Showing 7 changed files with 109 additions and 83 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,3 @@
+.git/
+.cache/
+.github/workflows
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -8,3 +8,9 @@ updates:
     directory: "/"
     schedule:
       interval: daily
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: daily
+    labels:
+      - automerge
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,59 +14,34 @@ concurrency:
 env:
   ALLOWERRORS: 'false'
   NBCACHE: '.cache'
-  JULIA_CONDAPKG_BACKEND: 'Null'
-  JULIA_NUM_THREADS: 'auto'
-  JULIA_CI: 'true'
-  UV_SYSTEM_PYTHON: 1
 
 jobs:
   setup:
+    permissions:
+      packages: write
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
-      hash: ${{ steps.hash.outputs.value }}
-      ver: ${{ steps.hash.outputs.ver }}
+      hash: ${{ steps.img.outputs.hash }}
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      id: setup-python
-      with:
-        python-version: '3.x'
-    - name: Read Julia version
-      uses: SebRollen/[email protected]
-      id: read_toml
-      with:
-        file: 'Manifest.toml'
-        field: 'julia_version'
-    - name: Setup Julia
-      uses: julia-actions/setup-julia@v2
+    - name: Login to ghcr.io
+      uses: docker/login-action@v3
       with:
-        version: ${{ steps.read_toml.outputs.value }}
-    - name: Get environment hash
-      id: hash
+        registry: ghcr.io
+        username: ${{ github.repository_owner	}}
+        password: ${{ github.token }}
+    - name: Get docker image hash
+      id: img
+      run: echo "hash=${{ hashFiles('requirements.txt', 'Project.toml', 'Manifest.toml', 'src/**', 'env.Dockerfile') }}" >> "$GITHUB_OUTPUT"
+    - name: Build Docker container
+      env:
+        IMG: ghcr.io/${{ github.repository }}:${{ steps.img.outputs.hash }}
       run: |
-        echo "value=${{ hashFiles('Project.toml', 'Manifest.toml', 'src/**') }}" >> "$GITHUB_OUTPUT"
-        echo "ver=${{ runner.os }}-julia-${{ steps.read_toml.outputs.value }}" >> "$GITHUB_OUTPUT"
-    - name: Cache Julia packages
-      if: ${{ contains(runner.name, 'GitHub Actions') }}
-      uses: actions/cache@v4
-      id: cache-julia
-      with:
-        path: ~/.julia
-        key: ${{ steps.hash.outputs.ver }}-${{ steps.hash.outputs.value }}
-        restore-keys: |
-          ${{ steps.hash.outputs.ver }}-
-    - name: Install Julia packages
-      if: ${{ runner.environment == 'self-hosted' || steps.cache-julia.outputs.cache-hit != 'true' }}
-      shell: julia --color=yes {0}
-      run: |
-        using Pkg
-        Pkg.add(["IJulia", "Literate", "JSON"])
-        Pkg.activate(".")
-        Pkg.instantiate()
-        Pkg.precompile()
+        docker manifest inspect ${IMG} && exit 0
+        docker build -f env.Dockerfile -t ${IMG} .
+        docker push ${IMG}
     - name: List notebooks as a JSON array
       id: set-matrix
       run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs",recursive=True)))')" >> "$GITHUB_OUTPUT"
@@ -81,6 +56,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NB: docs/${{ matrix.notebook }}
+      IMG: ghcr.io/${{ github.repository }}:${{ needs.setup.outputs.hash }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -89,39 +65,35 @@ jobs:
         id: nb-cache
         with:
           path: ${{ env.NBCACHE }}
-          key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        with:
-          python-version: '3.x'
-      - name: Install uv
+          key: ${{ runner.os }}-notebook-${{ hashFiles(env.NB) }}-${{ needs.setup.outputs.hash }}
+      - name: Pull docker image
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: Install Python dependencies
-        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: uv pip install -r requirements.txt
-      - name: Read Julia version
+        run: |
+          docker pull ${{ env.IMG }}
+          docker images ${{ env.IMG }}
+      - name: Execute notebook
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        uses: SebRollen/[email protected]
-        id: read_toml
-        with:
-          file: 'Manifest.toml'
-          field: 'julia_version'
-      - name: Setup Julia
+        run: >
+          docker run -w /tmp -v ${{ github.workspace }}:/tmp
+          -e NB=${{ env.NB }}
+          -e NBCACHE=${{ env.NBCACHE }}
+          -e WORKSPACE=${{ github.workspace }}
+          -e ALLOWERRORS=${{ env.ALLOWERRORS }}
+          ${{ env.IMG }}
+          julia execute.jl
+      - name: Claim output cache folder
+        run: |
+          sudo chown -R $USER ${{ env.NBCACHE }}
+          ls -R ${{ env.NBCACHE }}
+      - name: Cache notebook
+        uses: actions/cache/save@v4
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        uses: julia-actions/setup-julia@v2
         with:
-          version: ${{ steps.read_toml.outputs.value }}
-      - name: Restore Julia packages
-        if: ${{ runner.environment == 'github-hosted' && steps.nb-cache.outputs.cache-hit != 'true'}}
-        uses: actions/cache@v4
-        with:
-          path: ~/.julia
-          key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}
-      - name: Execute notebook
-        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: julia --project=@. execute.jl
+          path: ${{ env.NBCACHE }}
+          key: ${{ steps.nb-cache.outputs.cache-primary-key }}
+      - name: Convert artifact Name
+        id: art
+        run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
       - name: Upload Notebook
         uses: actions/upload-artifact@v4
         with:
@@ -146,6 +118,7 @@ jobs:
         run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
       - name: Setup Python
         uses: actions/setup-python@v5
+        id: setup-python
         with:
           python-version: '3.x'
       - name: Install uv

diff --git a/.github/workflows/rm-old-image.yml b/.github/workflows/rm-old-image.yml
@@ -0,0 +1,20 @@
+name: Docker image retention
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 1' # Every week
+
+jobs:
+  clean:
+    name: Delete old images
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+    - uses: snok/[email protected]
+      with:
+        account: user
+        token: ${{ secrets.GITHUB_TOKEN }}
+        image-names: ${{ github.event.repository.name }}
+        cut-off: 2w
+        dry-run: false
diff --git a/.github/workflows/update-manifest.yml b/.github/workflows/update-manifest.yml
@@ -26,14 +26,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Setup Julia
-        uses: julia-actions/setup-julia@v2
-        with:
-          version: '1'
       - name: Update Julia dependencies
-        env:
-          JULIA_PKG_PRECOMPILE_AUTO: '0'
-        run: julia --project=@. --color=yes -e 'using Pkg; Pkg.update()'
+        run: >
+          docker run --rm -w /tmp
+          -v ${{ github.workspace }}:/tmp
+          -e JULIA_PKG_PRECOMPILE_AUTO=0
+          julia:latest
+          julia --project=@. --color=yes -e 'import Pkg; Pkg.update()'
       # Authenticate with a custom GitHub APP
       # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens
       - name: Generate token for PR

diff --git a/env.Dockerfile b/env.Dockerfile
@@ -0,0 +1,27 @@
+FROM julia:1.10.5 as julia
+FROM ghcr.io/astral-sh/uv:latest as uv
+FROM python:3.12.5-slim
+
+# System config
+ENV UV_SYSTEM_PYTHON '1'
+ENV JULIA_CI 'true'
+ENV JULIA_NUM_THREADS 'auto'
+# Let PythonCall use built-in python
+ENV JULIA_CONDAPKG_BACKEND 'Null'
+ENV JULIA_PATH '/usr/local/julia/'
+ENV JULIA_DEPOT_PATH '/srv/juliapkg/'
+ENV PATH ${JULIA_PATH}/bin:${PATH}
+COPY --from=julia ${JULIA_PATH} ${JULIA_PATH}
+COPY --from=uv /uv /bin/uv
+
+WORKDIR /app
+
+# Python dependencies
+COPY requirements.txt .
+RUN uv pip install --no-cache -r requirements.txt
+
+# Julia dependencies
+COPY Project.toml Manifest.toml ./
+COPY src/ src
+RUN julia --color=yes -e 'using Pkg; Pkg.add(["IJulia", "Literate", "JSON"]); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \
+    julia --color=yes --project=@. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
diff --git a/execute.jl b/execute.jl
@@ -1,16 +1,16 @@
 using Literate
 using JSON
 using Pkg
-using IJulia
+Pkg.activate(Base.current_project())
 
 ENV["GKSwstype"] = "100"
+file = get(ENV, "NB", "test.ipynb")
+cachedir = get(ENV, "NBCACHE", ".cache")
 
 function main(; rmsvg=true)
-    file = get(ENV, "NB", "test.ipynb")
     if endswith(file, ".jl")
         run_literate(file; rmsvg)
     elseif endswith(file, ".ipynb")
-        IJulia.installkernel("Julia", "--project=@.")
         run_ipynb(file)
     else
         error("$(file) is not a valid notebook file!")
@@ -40,7 +40,6 @@ function strip_svg(ipynb)
 end
 
 function run_literate(file; rmsvg=true)
-    cachedir = get(ENV, "NBCACHE", ".cache")
     outpath = joinpath(abspath(pwd()), cachedir, dirname(file))
     mkpath(outpath)
     ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true)
@@ -49,7 +48,6 @@ function run_literate(file; rmsvg=true)
 end
 
 function run_ipynb(file)
-    cachedir = get(ENV, "NBCACHE", ".cache")
     outpath = joinpath(abspath(pwd()), cachedir, file)
     mkpath(dirname(outpath))
     kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)"