Merge pull request #270 from AlexsLemonade/jashapiro/265-slim-docker

Update python requirements and create slim docker image

.Rprofile

@@ -1,6 +1,2 @@
-# Configure BioCManager to use Posit Public Package Manager:
-options(BioC_mirror = "https://p3m.dev/bioconductor")
-options(BIOCONDUCTOR_CONFIG_FILE = "https://p3m.dev/bioconductor/config.yaml")
-
 # activate renv
 source("renv/activate.R")

.github/workflows/build-docker.yaml

@@ -1,48 +1,63 @@
-name: Build scpca-tools image
+name: Build scpca-tools images
 
 on:
   push:
     branches:
       - main
     tags:
-      - 'v*'
+      - "v*"
   pull_request:
     branches:
       - main
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: alexslemonade/scpca-tools
+  REGISTRY_USER: alexslemonade
 
 jobs:
   build_and_push:
     runs-on: ubuntu-22.04
     permissions:
       contents: read
       packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image_name: scpcatools
+            dockerfile: Dockerfile
+          - image_name: scpcatools-slim
+            dockerfile: scpcatools_slim.Dockerfile
+          - image_name: scpcatools-reports
+            dockerfile: scpcatools_reports.Dockerfile
+          - image_name: scpcatools-scvi
+            dockerfile: scpcatools_scvi.Dockerfile
+          - image_name: scpcatools-seurat
+            dockerfile: scpcatools_seurat.Dockerfile
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker metadata
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.REGISTRY_USER }}/${{ matrix.image_name }}
           tags: |
             type=semver,pattern={{raw}}
             type=edge,branch=main
 
-      - uses: docker/build-push-action@v4
+      - uses: docker/build-push-action@v5
         with:
           context: "{{defaultContext}}:docker"
+          file: ${{ matrix.dockerfile }}
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 # All available hooks: https://pre-commit.com/hooks.html
 # R specific hooks: https://github.com/lorenzwalthert/precommit
 repos:
--   repo: https://github.com/lorenzwalthert/precommit
-    rev: v0.3.2.9027
+  - repo: https://github.com/lorenzwalthert/precommit
+    rev: v0.4.2
     hooks:
-    -   id: style-files
+      - id: style-files
         args: [--style_pkg=styler, --style_fun=tidyverse_style]
-    # codemeta must be above use-tidy-description when both are used
-    # -   id: codemeta-description-updated
-    -   id: use-tidy-description
-    -   id: spell-check
+      # codemeta must be above use-tidy-description when both are used
+      # -   id: codemeta-description-updated
+      - id: use-tidy-description
+      - id: spell-check
         exclude: >
           (?x)^(
           .*\.[rR]|
@@ -35,22 +35,24 @@ repos:
           (.*/|)DESCRIPTION|
           (.*/|)NAMESPACE|
           (.*/|)renv/settings\.dcf|
-          (.*/|)renv\.lock|
-          (.*/|)requirements\.txt|
-          (.*/|)Dockerfile|
+          (.*/|)renv.*\.lock|
+          (.*/|)requirements.*\.in|
+          (.*/|)requirements.*\.txt|
+          (.*/|).*Dockerfile|
           (.*/|)WORDLIST|
           \.github/workflows/.*|
           data/.*|
           )$
-    -   id: lintr
-    -   id: parsable-R
-    -   id: no-browser-statement
-    -   id: no-debug-statement
-    -   id: deps-in-desc
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+      - id: lintr
+      - id: parsable-R
+      - id: no-browser-statement
+      - id: no-debug-statement
+      - id: deps-in-desc
+        exclude: docker/.*
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
     hooks:
-    -   id: check-added-large-files
-        args: ['--maxkb=200']
-    -   id: end-of-file-fixer
+      - id: check-added-large-files
+        args: ["--maxkb=200"]
+      - id: end-of-file-fixer
         exclude: '\.Rd'

docker/Dockerfile

@@ -1,11 +1,7 @@
-FROM rocker/r-ver:4.3.3
+FROM bioconductor/r-ver:3.18
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.source https://github.com/AlexsLemonade/scpcaTools
 
-COPY scripts/install_scpca_deps.sh .
-
-RUN bash ./install_scpca_deps.sh
-
 #### R packages
 # Use renv for R packages
 ENV RENV_CONFIG_CACHE_ENABLED FALSE
@@ -15,14 +11,15 @@ WORKDIR /usr/local/renv
 COPY renv.lock renv.lock
 # restore renv and remove cache files
 RUN Rscript -e "renv::restore()" && \
-      rm -rf ~/.cache/R/renv && \
-      rm -rf /tmp/downloaded_packages && \
-      rm -rf /tmp/Rtmp*
+  rm -rf ~/.cache/R/renv && \
+  rm -rf /tmp/downloaded_packages && \
+  rm -rf /tmp/Rtmp*
 
 # Complete installation of zellkonverter conda env
 ENV BASILISK_EXTERNAL_DIR /usr/local/renv/basilisk
 RUN Rscript -e "proc <- basilisk::basiliskStart(env = zellkonverter::zellkonverterAnnDataEnv(), testload = 'anndata'); \
-                basilisk::basiliskStop(proc)"
+  basilisk::basiliskStop(proc); \
+  basilisk.utils::cleanConda()"
 
 #### Python packages
 COPY requirements.txt requirements.txt

docker/README.md

@@ -5,11 +5,34 @@ The image is built from a versioned Rocker image, with additional R packages and
 It can be built with the following command run in this directory:
 
 ```
-docker buildx build . -t scpca-tools --platform linux/amd64
+docker buildx build . -t scpcatools --platform linux/amd64
 ```
 
-Note that this image does not include RStudio, in attempt to make the image smaller. 
+Note that this image does not include RStudio, in attempt to make the image smaller.
 
 This image is built automatically on every update to `scpcaTools` (`main`) and pushed to `ghcr.io/alexslemonade/scpca-tools:edge`.
 
-Release versions of `scpcaTools` will be tagged as `ghcr.io/alexslemonade/scpca-tools:latest` and tagged by version number.
+Release versions of `scpcaTools` will be tagged as `ghcr.io/alexslemonade/scpcatools:latest` and tagged by version number.
+
+The main image includes all of the recommended packages for `scpcaTools` (aside from those only required for development), as well as the Python `scanpy` and `scvi-tools` packages and dependencies.
+
+There are also smaller images that can be built, which include only the R packages required for `scpcaTools`, with particular additions for specific purposes. These include:
+
+- `scpcatools_slim.Dockerfile`: A smaller image that includes only the required R packages for `scpcaTools`, with no additional packages.
+- `scpcatools_scvi.Dockerfile`: The slim package with the addition of the `zellkonverter` package for reading and writing `AnnData` objects, and the Python `anndata` and `scvi` packages.
+- `scpcatools_reports.Dockerfile`: The slim package with the addition `rmarkdown` and `ComplexHeatmap` and associated packages for generating reports.
+- `scpcatools_seurat.Dockerfile`: The slim package with the addition `Seurat`.
+
+These images are built automatically on every update to `scpcaTools` (`main`) and pushed to `ghcr.io/alexslemonade/scpcatool-slims:edge` and `ghcr.io/alexslemonade/scpcatools-scvi:edge`, etc.
+
+## Generating R and Python dependency files
+
+Dependency lock files, with the exception of `renv.lock`, are built using the `make-requirements.sh` script.
+This script depends on `pip-tools` and should be run using Python 3.10 to match the default version of Python in the Docker images.
+For convenience, a conda `environment.yml` file is included in this directory that can be used to create an `scpcatools-dev` environment with the necessary Python version and packages.
+
+This script will generate the `requirements_*.txt` and `renv_*.lock` files for each set of Python packages to be installed within the various Docker images.
+- The requirements files are based on `requirements_*.in` files that specify the high-level package requirements for each image.
+- The `renv_*.lock` files are strict subsets of the main `renv.lock` file that is use for development and for the full image.
+  - The main `renv.lock` file should be updated manually using `renv::snapshot()` from within the `scpcaTools` project.
+  - After updating `renv.lock`, the `make-requirements.sh` script should be run to update the `renv_*.lock` files.

docker/environment.yml

@@ -0,0 +1,7 @@
+name: scpcatools-dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.10
+  - pip-tools

docker/make-requirements.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+# This script is used to generate or update requirements/lock files for R and Python packages
+# Requires that the `pip-tools` python package is installed.
+# Before running, make sure that the renv.lock file and installed libaries are
+# consistent with renv::snapshot() or renv::restore()
+
+# Set the working directory to the directory of this file
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+# Python package lists
+pip-compile --no-annotate --strip-extras --output-file=requirements.txt requirements.in
+pip-compile --no-annotate --strip-extras --output-file=requirements_anndata.txt requirements_anndata.in
+pip-compile --no-annotate --strip-extras --output-file=requirements_scvi.txt requirements_scvi.in
+
+# slim lockfile
+Rscript scripts/make-lockfile.R -f renv_slim.lock
+
+# reports lockfile
+Rscript scripts/make-lockfile.R -f renv_reports.lock -p ComplexHeatmap,ggforce,kableExtra,rmarkdown
+
+# zellkonverter lockfile
+Rscript scripts/make-lockfile.R -f renv_zellkonverter.lock -p zellkonverter
+
+# Seurat lockfile
+Rscript scripts/make-lockfile.R -f renv_seurat.lock -p Seurat