From 5bf11d5733ac935f54b6808f6f8190da30118033 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 3 Sep 2024 13:47:19 -0700 Subject: [PATCH 1/4] Split off stat20 user image. --- .circleci/config.yml | 25 ----- deployments/stat20/hubploy.yaml | 7 +- deployments/stat20/image/Dockerfile | 92 ------------------- deployments/stat20/image/README.md | 5 + deployments/stat20/image/class-libs.R | 20 ---- deployments/stat20/image/environment.yml | 25 ----- deployments/stat20/image/file-locks | 13 --- .../stat20/image/infra-requirements.txt | 29 ------ .../stat20/image/install-mambaforge.bash | 42 --------- .../image/r-packages/2023-fall-stat-20.r | 50 ---------- deployments/stat20/image/rstudio-prefs.json | 3 - 11 files changed, 6 insertions(+), 305 deletions(-) delete mode 100644 deployments/stat20/image/Dockerfile create mode 100644 deployments/stat20/image/README.md delete mode 100644 deployments/stat20/image/class-libs.R delete mode 100644 deployments/stat20/image/environment.yml delete mode 100644 deployments/stat20/image/file-locks delete mode 100644 deployments/stat20/image/infra-requirements.txt delete mode 100755 deployments/stat20/image/install-mambaforge.bash delete mode 100644 deployments/stat20/image/r-packages/2023-fall-stat-20.r delete mode 100644 deployments/stat20/image/rstudio-prefs.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fa02f25d..396b89b7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -319,12 +319,6 @@ jobs: hubploy deploy --timeout 30m shiny hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy stat20 - command: | - hubploy deploy --timeout 30m stat20 hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy ugr01 command: | @@ -505,15 +499,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: stat20 - name: stat20 image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: ugr01 name: ugr01 image build @@ -687,15 +672,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: stat20 - name: stat20 image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: ugr01 name: ugr01 image build @@ -728,7 +704,6 @@ workflows: - nature image build - publichealth image build - shiny image build - - stat20 image build filters: branches: diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index dd982a4a7..518b3560a 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,10 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/stat20/image/Dockerfile b/deployments/stat20/image/Dockerfile deleted file mode 100644 index e83591c4a..000000000 --- a/deployments/stat20/image/Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -FROM rocker/geospatial:4.4.1 -# https://github.com/rocker-org/rocker-versioned2/wiki/geospatial_e06f866673fa - -ENV NB_USER rstudio -ENV NB_UID 1000 -ENV CONDA_DIR /srv/conda - -# Set ENV for all programs... -ENV PATH ${CONDA_DIR}/bin:$PATH - -# Pick up rocker's default TZ -ENV TZ=Etc/UTC - -# And set ENV for R! It doesn't read from the environment... -RUN echo "TZ=${TZ}" >> /usr/local/lib/R/etc/Renviron.site -RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site - -# Add PATH to /etc/profile so it gets picked up by the terminal -RUN echo "PATH=${PATH}" >> /etc/profile -RUN echo "export PATH" >> /etc/profile - -ENV HOME /home/${NB_USER} - -WORKDIR ${HOME} - -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -# tini is necessary because it is our ENTRYPOINT below. -RUN apt-get update && \ - apt-get -qq install \ - less \ - tini \ - fonts-symbola \ - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-plain-generic \ - > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# While quarto is included with rocker/verse, we sometimes need different -# versions than the default. For example a newer version might fix bugs. -ENV _QUARTO_VERSION=1.4.549 -RUN curl -L -o /tmp/quarto.deb https://github.com/quarto-dev/quarto-cli/releases/download/v${_QUARTO_VERSION}/quarto-${_QUARTO_VERSION}-linux-amd64.deb -RUN apt-get update > /dev/null && \ - apt-get install /tmp/quarto.deb > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - rm -f /tmp/quarto.deb - -ENV SHINY_SERVER_URL https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-1.5.21.1012-amd64.deb -RUN curl --silent --location --fail ${SHINY_SERVER_URL} > /tmp/shiny-server.deb && \ - apt install --no-install-recommends --yes /tmp/shiny-server.deb && \ - rm /tmp/shiny-server.deb - -# google-chrome is for pagedown; chromium doesn't work nicely with it (snap?) -RUN wget --quiet -O /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - apt-get update > /dev/null && \ - apt-get -qq install /tmp/chrome.deb > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - rm -f /tmp/chrome.deb - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/environment.yml -COPY infra-requirements.txt /tmp/infra-requirements.txt -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -COPY class-libs.R /tmp/class-libs.R - -COPY r-packages/2023-fall-stat-20.r /tmp/r-packages/ -RUN r /tmp/r-packages/2023-fall-stat-20.r - -# Configure locking behavior -COPY file-locks /etc/rstudio/file-locks - -# Disable visual markdown editing by default -COPY rstudio-prefs.json /etc/rstudio/rstudio-prefs.json - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/stat20/image/README.md b/deployments/stat20/image/README.md new file mode 100644 index 000000000..1c5508f86 --- /dev/null +++ b/deployments/stat20/image/README.md @@ -0,0 +1,5 @@ +# Stat20 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/stat20-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat20-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/stat20/image/class-libs.R b/deployments/stat20/image/class-libs.R deleted file mode 100644 index 1c9343537..000000000 --- a/deployments/stat20/image/class-libs.R +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env Rscript - -class_libs_install_version <- function(class_name, class_libs) { - print(paste("Installing packages for", class_name)) - for (i in seq(1, length(class_libs), 2)) { - installed_packages <- rownames(installed.packages()) - package_name = class_libs[i] - version = class_libs[i+1] - # Only install packages if they haven't already been installed! - # devtools doesn't do that by default - if (!package_name %in% installed_packages) { - print(paste("Installing", package_name, version)) - devtools::install_version(package_name, version, quiet=TRUE) - } else { - # FIXME: This ignores version incompatibilities :'( - print(paste("Not installing", package_name, " as it is already installed")) - } - } - print(paste("Done installing packages for", class_name)) -} diff --git a/deployments/stat20/image/environment.yml b/deployments/stat20/image/environment.yml deleted file mode 100644 index f69fba4a5..000000000 --- a/deployments/stat20/image/environment.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: stat20 - -channels: -- conda-forge - -dependencies: -- syncthing==1.22.2 -- jupyter-server-proxy==4.2.0 -- jupyter-rsession-proxy==2.2.0 -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* -# for usage analysis dashboard (to be removed during sp 24 maintenance window) -- pandas==2.2.1 -- matplotlib==3.8.3 -- pip: - - -r /tmp/infra-requirements.txt - # For push authentication to GitHub - - gh-scoped-creds==4.1 - - jupyter-shiny-proxy==1.1 - # for notebook exporting - - nbconvert==7.6.0 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - - shiny==0.8.1 # for usage analysis dashboard (to be removed during sp 24 maintenance window) - diff --git a/deployments/stat20/image/file-locks b/deployments/stat20/image/file-locks deleted file mode 100644 index 7b1a3fcf4..000000000 --- a/deployments/stat20/image/file-locks +++ /dev/null @@ -1,13 +0,0 @@ -# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking - -# rocker sets this to advisory, but this might be causing NFS issues. -# lets set it to the default (default: linkbased) -lock-type=linkbased - -# we'll also reduce the frequency by 1/3 -refresh-rate=60 -timeout-interval=90 - -# log attempts -# enable-logging=1 -# log-file=/tmp/rstudio-locking.log diff --git a/deployments/stat20/image/infra-requirements.txt b/deployments/stat20/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/stat20/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/stat20/image/install-mambaforge.bash b/deployments/stat20/image/install-mambaforge.bash deleted file mode 100755 index 812319edc..000000000 --- a/deployments/stat20/image/install-mambaforge.bash +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION=22.9.0-2 - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -# make sure we don't do anything funky with user's $HOME -# since this is run as root -unset HOME - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/deployments/stat20/image/r-packages/2023-fall-stat-20.r b/deployments/stat20/image/r-packages/2023-fall-stat-20.r deleted file mode 100644 index 83039e84d..000000000 --- a/deployments/stat20/image/r-packages/2023-fall-stat-20.r +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env Rscript - -source("/tmp/class-libs.R") - -class_name = "2023 Fall Stat 20" - -class_libs = c( - "fivethirtyeight", "0.6.2", - "gapminder", "0.3.0", - "janitor", "2.2.0", - "openintro", "2.4.0", - "pagedown", "0.16", - "palmerpenguins", "0.1.1", - "patchwork", "1.1.2", - "showtext", "0.9-4", - "swirl", "2.4.5", - "tidycensus", "1.1", - "tidymodels", "0.1.4", - "tigris", "1.5", - "unvotes", "0.3.0", - "xaringanthemer", "0.4.1", - "rmarkdown", "2.22", - "plotly", "4.10.1", - "reshape2", "1.4.4", - "kableExtra", "1.3.4", - "infer", "1.0.4", - "countdown", "0.4.0", - "ggrepel", "0.9.3", - "ggthemes", "4.2.4", - "latex2exp", "0.9.6", - "markdown", "1.7", - "downlit", "0.4.3", - "xml2", "1.3.4", - "gt", "0.9.0", - "quarto", "1.2", - "fs", "1.6.3", - "rsample", "1.2.1" -) - -class_libs_install_version(class_name, class_libs) - -devtools::install_github("mdbeckman/dcData", ref="56888a6") -devtools::install_github("hadley/emo@3f03b11") -devtools::install_github("andrewpbray/boxofdata@8afd934") -devtools::install_github("tidymodels/infer@2806a69") -devtools::install_github("stat20/stat20data@11b4377") - -# file.symlink("/opt/shared/stat20/stat20data", "/usr/local/lib/R/site-library/stat20data") - -print(paste("Done installing packages for",class_name)) diff --git a/deployments/stat20/image/rstudio-prefs.json b/deployments/stat20/image/rstudio-prefs.json deleted file mode 100644 index 0647cffbf..000000000 --- a/deployments/stat20/image/rstudio-prefs.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "visual_markdown_editing_is_default": false -} From fef58698740b307c94fed85f74dc8dfc5c68d23b Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 3 Sep 2024 13:53:15 -0700 Subject: [PATCH 2/4] Add stat20 to workflow. --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 4ff7da1d5..969ae17a5 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -102,7 +102,7 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub staging - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -196,4 +196,4 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub prod - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20) From 0ffe11bff2cd1151119811d346c5d0aedad1d917 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" Date: Tue, 3 Sep 2024 20:55:27 +0000 Subject: [PATCH 3/4] update stat20 image tag to d3fd1f663330: deployments/stat20/hubploy.yaml --- deployments/stat20/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index 518b3560a..9fd637c16 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:PLACEHOLDER + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:d3fd1f663330 cluster: provider: gcloud From 3db70fd320ca72fd77f9809a718c43625bfa58f2 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" Date: Tue, 3 Sep 2024 23:14:57 +0000 Subject: [PATCH 4/4] update stat20 image tag to 5eed7cbfc7ff: deployments/stat20/hubploy.yaml --- deployments/stat20/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index 9fd637c16..e9a4038e0 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:d3fd1f663330 + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:5eed7cbfc7ff cluster: provider: gcloud