From c175364aa7b1772b70fb119a4543ebd128fdf91c Mon Sep 17 00:00:00 2001 From: Wen Wei Tseng Date: Mon, 14 Oct 2024 17:00:47 +0800 Subject: [PATCH] Revert "test no docker (#250)" This reverts commit f35fabb7f24e1a73e68b7a8b79a5d6e3881fdfb2. --- .dockerignore | 3 + .github/dependabot.yml | 6 ++ .github/workflows/ci.yml | 115 ++++++++++---------------- .github/workflows/rm-old-image.yml | 20 +++++ .github/workflows/update-manifest.yml | 13 ++- env.Dockerfile | 27 ++++++ execute.jl | 8 +- 7 files changed, 109 insertions(+), 83 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/rm-old-image.yml create mode 100644 env.Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2311a50 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git/ +.cache/ +.github/workflows diff --git a/.github/dependabot.yml b/.github/dependabot.yml index afaf0bb..f278062 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,9 @@ updates: directory: "/" schedule: interval: daily + - package-ecosystem: docker + directory: "/" + schedule: + interval: daily + labels: + - automerge diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcc3d8e..2efde21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,59 +14,34 @@ concurrency: env: ALLOWERRORS: 'false' NBCACHE: '.cache' - JULIA_CONDAPKG_BACKEND: 'Null' - JULIA_NUM_THREADS: 'auto' - JULIA_CI: 'true' - UV_SYSTEM_PYTHON: 1 jobs: setup: + permissions: + packages: write runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - hash: ${{ steps.hash.outputs.value }} - ver: ${{ steps.hash.outputs.ver }} + hash: ${{ steps.img.outputs.hash }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - id: setup-python - with: - python-version: '3.x' - - name: Read Julia version - uses: SebRollen/toml-action@v1.2.0 - id: read_toml - with: - file: 'Manifest.toml' - field: 'julia_version' - - name: Setup Julia - uses: julia-actions/setup-julia@v2 + - name: Login to ghcr.io + uses: docker/login-action@v3 with: - version: ${{ steps.read_toml.outputs.value }} - - name: Get environment hash - id: hash + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + - name: Get docker image hash + id: img + run: echo "hash=${{ hashFiles('requirements.txt', 'Project.toml', 'Manifest.toml', 'src/**', 'env.Dockerfile') }}" >> "$GITHUB_OUTPUT" + - name: Build Docker container + env: + IMG: ghcr.io/${{ github.repository }}:${{ steps.img.outputs.hash }} run: | - echo "value=${{ hashFiles('Project.toml', 'Manifest.toml', 'src/**') }}" >> "$GITHUB_OUTPUT" - echo "ver=${{ runner.os }}-julia-${{ steps.read_toml.outputs.value }}" >> "$GITHUB_OUTPUT" - - name: Cache Julia packages - if: ${{ contains(runner.name, 'GitHub Actions') }} - uses: actions/cache@v4 - id: cache-julia - with: - path: ~/.julia - key: ${{ steps.hash.outputs.ver }}-${{ steps.hash.outputs.value }} - restore-keys: | - ${{ steps.hash.outputs.ver }}- - - name: Install Julia packages - if: ${{ runner.environment == 'self-hosted' || steps.cache-julia.outputs.cache-hit != 'true' }} - shell: julia --color=yes {0} - run: | - using Pkg - Pkg.add(["IJulia", "Literate", "JSON"]) - Pkg.activate(".") - Pkg.instantiate() - Pkg.precompile() + docker manifest inspect ${IMG} && exit 0 + docker build -f env.Dockerfile -t ${IMG} . + docker push ${IMG} - name: List notebooks as a JSON array id: set-matrix run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs",recursive=True)))')" >> "$GITHUB_OUTPUT" @@ -81,6 +56,7 @@ jobs: runs-on: ubuntu-latest env: NB: docs/${{ matrix.notebook }} + IMG: ghcr.io/${{ github.repository }}:${{ needs.setup.outputs.hash }} steps: - name: Checkout uses: actions/checkout@v4 @@ -89,39 +65,35 @@ jobs: id: nb-cache with: path: ${{ env.NBCACHE }} - key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} - - name: Setup Python - uses: actions/setup-python@v5 - if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - with: - python-version: '3.x' - - name: Install uv + key: ${{ runner.os }}-notebook-${{ hashFiles(env.NB) }}-${{ needs.setup.outputs.hash }} + - name: Pull docker image if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - name: Install Python dependencies - if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: uv pip install -r requirements.txt - - name: Read Julia version + run: | + docker pull ${{ env.IMG }} + docker images ${{ env.IMG }} + - name: Execute notebook if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - uses: SebRollen/toml-action@v1.2.0 - id: read_toml - with: - file: 'Manifest.toml' - field: 'julia_version' - - name: Setup Julia + run: > + docker run -w /tmp -v ${{ github.workspace }}:/tmp + -e NB=${{ env.NB }} + -e NBCACHE=${{ env.NBCACHE }} + -e WORKSPACE=${{ github.workspace }} + -e ALLOWERRORS=${{ env.ALLOWERRORS }} + ${{ env.IMG }} + julia execute.jl + - name: Claim output cache folder + run: | + sudo chown -R $USER ${{ env.NBCACHE }} + ls -R ${{ env.NBCACHE }} + - name: Cache notebook + uses: actions/cache/save@v4 if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - uses: julia-actions/setup-julia@v2 with: - version: ${{ steps.read_toml.outputs.value }} - - name: Restore Julia packages - if: ${{ runner.environment == 'github-hosted' && steps.nb-cache.outputs.cache-hit != 'true'}} - uses: actions/cache@v4 - with: - path: ~/.julia - key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }} - - name: Execute notebook - if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: julia --project=@. execute.jl + path: ${{ env.NBCACHE }} + key: ${{ steps.nb-cache.outputs.cache-primary-key }} + - name: Convert artifact Name + id: art + run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" - name: Upload Notebook uses: actions/upload-artifact@v4 with: @@ -146,6 +118,7 @@ jobs: run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/ - name: Setup Python uses: actions/setup-python@v5 + id: setup-python with: python-version: '3.x' - name: Install uv diff --git a/.github/workflows/rm-old-image.yml b/.github/workflows/rm-old-image.yml new file mode 100644 index 0000000..e053549 --- /dev/null +++ b/.github/workflows/rm-old-image.yml @@ -0,0 +1,20 @@ +name: Docker image retention +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' # Every week + +jobs: + clean: + name: Delete old images + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: snok/container-retention-policy@v3.0.0 + with: + account: user + token: ${{ secrets.GITHUB_TOKEN }} + image-names: ${{ github.event.repository.name }} + cut-off: 2w + dry-run: false diff --git a/.github/workflows/update-manifest.yml b/.github/workflows/update-manifest.yml index 32d69c7..bf51de2 100644 --- a/.github/workflows/update-manifest.yml +++ b/.github/workflows/update-manifest.yml @@ -26,14 +26,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Julia - uses: julia-actions/setup-julia@v2 - with: - version: '1' - name: Update Julia dependencies - env: - JULIA_PKG_PRECOMPILE_AUTO: '0' - run: julia --project=@. --color=yes -e 'using Pkg; Pkg.update()' + run: > + docker run --rm -w /tmp + -v ${{ github.workspace }}:/tmp + -e JULIA_PKG_PRECOMPILE_AUTO=0 + julia:latest + julia --project=@. --color=yes -e 'import Pkg; Pkg.update()' # Authenticate with a custom GitHub APP # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens - name: Generate token for PR diff --git a/env.Dockerfile b/env.Dockerfile new file mode 100644 index 0000000..7fe641b --- /dev/null +++ b/env.Dockerfile @@ -0,0 +1,27 @@ +FROM julia:1.10.5 as julia +FROM ghcr.io/astral-sh/uv:latest as uv +FROM python:3.12.5-slim + +# System config +ENV UV_SYSTEM_PYTHON '1' +ENV JULIA_CI 'true' +ENV JULIA_NUM_THREADS 'auto' +# Let PythonCall use built-in python +ENV JULIA_CONDAPKG_BACKEND 'Null' +ENV JULIA_PATH '/usr/local/julia/' +ENV JULIA_DEPOT_PATH '/srv/juliapkg/' +ENV PATH ${JULIA_PATH}/bin:${PATH} +COPY --from=julia ${JULIA_PATH} ${JULIA_PATH} +COPY --from=uv /uv /bin/uv + +WORKDIR /app + +# Python dependencies +COPY requirements.txt . +RUN uv pip install --no-cache -r requirements.txt + +# Julia dependencies +COPY Project.toml Manifest.toml ./ +COPY src/ src +RUN julia --color=yes -e 'using Pkg; Pkg.add(["IJulia", "Literate", "JSON"]); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \ + julia --color=yes --project=@. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' diff --git a/execute.jl b/execute.jl index a7a6c91..7064fcc 100644 --- a/execute.jl +++ b/execute.jl @@ -1,16 +1,16 @@ using Literate using JSON using Pkg -using IJulia +Pkg.activate(Base.current_project()) ENV["GKSwstype"] = "100" +file = get(ENV, "NB", "test.ipynb") +cachedir = get(ENV, "NBCACHE", ".cache") function main(; rmsvg=true) - file = get(ENV, "NB", "test.ipynb") if endswith(file, ".jl") run_literate(file; rmsvg) elseif endswith(file, ".ipynb") - IJulia.installkernel("Julia", "--project=@.") run_ipynb(file) else error("$(file) is not a valid notebook file!") @@ -40,7 +40,6 @@ function strip_svg(ipynb) end function run_literate(file; rmsvg=true) - cachedir = get(ENV, "NBCACHE", ".cache") outpath = joinpath(abspath(pwd()), cachedir, dirname(file)) mkpath(outpath) ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true) @@ -49,7 +48,6 @@ function run_literate(file; rmsvg=true) end function run_ipynb(file) - cachedir = get(ENV, "NBCACHE", ".cache") outpath = joinpath(abspath(pwd()), cachedir, file) mkpath(dirname(outpath)) kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)"