From 15a6a71bc2f7252832aae60a1940031aa44d6e43 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 23 Sep 2024 10:43:05 -0700 Subject: [PATCH 1/4] moving ischool image to its own repo --- .circleci/config.yml | 25 ---- deployments/ischool/hubploy.yaml | 8 +- deployments/ischool/image/Dockerfile | 107 ------------------ deployments/ischool/image/README.md | 5 + deployments/ischool/image/environment.yml | 23 ---- deployments/ischool/image/file-locks | 13 --- .../ischool/image/infra-requirements.txt | 29 ----- .../ischool/image/install-mambaforge.bash | 38 ------- 8 files changed, 6 insertions(+), 242 deletions(-) delete mode 100644 deployments/ischool/image/Dockerfile create mode 100644 deployments/ischool/image/README.md delete mode 100644 deployments/ischool/image/environment.yml delete mode 100644 deployments/ischool/image/file-locks delete mode 100644 deployments/ischool/image/infra-requirements.txt delete mode 100755 deployments/ischool/image/install-mambaforge.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d59b0f9a..dce3a85af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -217,12 +217,6 @@ jobs: hubploy deploy --timeout 30m highschool hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy ischool - command: | - hubploy deploy --timeout 30m ischool hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy prob140 command: | @@ -282,15 +276,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: ischool - name: ischool image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod deploy: jobs: @@ -330,15 +315,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: ischool - name: ischool image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging # Build images only during the staging deploy. All merges # to prod need to go via staging, so prod should *never* # use images not built for staging. By enforcing this at the @@ -349,7 +325,6 @@ workflows: - data100 image build - datahub image build - edx image build - - ischool image build filters: branches: diff --git a/deployments/ischool/hubploy.yaml b/deployments/ischool/hubploy.yaml index 9e7582c88..de342d2fd 100644 --- a/deployments/ischool/hubploy.yaml +++ b/deployments/ischool/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ischool-user-image - path: image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ischool-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/ischool/image/Dockerfile b/deployments/ischool/image/Dockerfile deleted file mode 100644 index 72835b8f0..000000000 --- a/deployments/ischool/image/Dockerfile +++ /dev/null @@ -1,107 +0,0 @@ -FROM rocker/geospatial:4.4.1 - -ENV NB_USER rstudio -ENV NB_UID 1000 -ENV CONDA_DIR /srv/conda - -# Set ENV for all programs... -ENV PATH ${CONDA_DIR}/bin:$PATH - -# And set ENV for R! It doesn't read from the environment... -RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site - -# Add PATH to /etc/profile so it gets picked up by the terminal -RUN echo "PATH=${PATH}" >> /etc/profile -RUN echo "export PATH" >> /etc/profile - -ENV HOME /home/${NB_USER} - -WORKDIR ${HOME} - -# Install packages needed by notebook-as-pdf -# nodejs for installing notebook / jupyterhub from source -# libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 -RUN apt-get update > /dev/null && \ - apt-get install --yes \ - less \ - fonts-symbola \ - libx11-xcb1 \ - libxtst6 \ - libxrandr2 \ - libasound2 \ - libpangocairo-1.0-0 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libgtk-3-0 \ - libnss3 \ - libnspr4 \ - libxss1 \ - nodejs \ - npm \ - texlive-xetex \ - texlive-fonts-extra \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-lang-chinese \ - texlive-plain-generic \ - tini > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/environment.yml -COPY infra-requirements.txt /tmp/infra-requirements.txt -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -# DH-327, very similar to what was done for datahub in DH-164 -ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} -RUN playwright install chromium - -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -# Install extra R packages -RUN install2.r --error --skipinstalled \ - ## general interest - anytime \ - Hmisc \ - lmtest \ - lubridate \ - latex2exp \ - MASS \ - patchwork \ - stargazer \ - tidymodels \ - wooldridge \ - ## machine learning - caret \ - parsnip \ - torch \ - ## 241 - AER \ - randomizr \ - ## 271 - fable \ - feasts \ - fpp3 \ - gamlr \ - plm \ - lme4 \ - Quandl \ - quantmod \ - tseries \ - tsibble \ - vcd \ - vars \ - zoo - -# Use simpler locking strategy -COPY file-locks /etc/rstudio/file-locks - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/ischool/image/README.md b/deployments/ischool/image/README.md new file mode 100644 index 000000000..9c460fd99 --- /dev/null +++ b/deployments/ischool/image/README.md @@ -0,0 +1,5 @@ +# ischool Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/ischool-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/ischool-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/ischool/image/environment.yml b/deployments/ischool/image/environment.yml deleted file mode 100644 index 28fd179df..000000000 --- a/deployments/ischool/image/environment.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: ischool - -channels: -- conda-forge - -dependencies: -- syncthing==1.25.0 -- jupyter-rsession-proxy==2.2.0 -# https://github.com/berkeley-dsep-infra/datahub/issues/5251 -- nodejs=16 # code-server requires node < 17 -- jupyter-server-proxy==4.2.0 -- jupyter-vscode-proxy==0.5 -- code-server==4.10.1 -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* -- pip: - - -r /tmp/infra-requirements.txt - - otter-grader==3.1.4 - # DH-327 - - nbconvert[webpdf]==7.16.4 - # for notebook exporting - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 diff --git a/deployments/ischool/image/file-locks b/deployments/ischool/image/file-locks deleted file mode 100644 index 7b1a3fcf4..000000000 --- a/deployments/ischool/image/file-locks +++ /dev/null @@ -1,13 +0,0 @@ -# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking - -# rocker sets this to advisory, but this might be causing NFS issues. -# lets set it to the default (default: linkbased) -lock-type=linkbased - -# we'll also reduce the frequency by 1/3 -refresh-rate=60 -timeout-interval=90 - -# log attempts -# enable-logging=1 -# log-file=/tmp/rstudio-locking.log diff --git a/deployments/ischool/image/infra-requirements.txt b/deployments/ischool/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/ischool/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/ischool/image/install-mambaforge.bash b/deployments/ischool/image/install-mambaforge.bash deleted file mode 100755 index 38c5cefcd..000000000 --- a/deployments/ischool/image/install-mambaforge.bash +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION='23.1.0-1' - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root From db5989cbeb206ea2d08131db9c83ea716d84684e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 24 Sep 2024 09:47:21 -0700 Subject: [PATCH 2/4] enable ci/cd for ischool --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 13a5c8e57..f2cc35b74 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102 ischool) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102 ischool) From 53d05ae4b7389ce42a09ac9256e733c547336a99 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" Date: Tue, 24 Sep 2024 17:01:03 +0000 Subject: [PATCH 3/4] update ischool image tag to 0be26ba905cb: deployments/ischool/hubploy.yaml --- deployments/ischool/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/ischool/hubploy.yaml b/deployments/ischool/hubploy.yaml index de342d2fd..1b08b012a 100644 --- a/deployments/ischool/hubploy.yaml +++ b/deployments/ischool/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ischool-user-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ischool-user-image:0be26ba905cb cluster: provider: gcloud From 78f65d8dd8ddbecc1cfbce93d2678cb8a8d6eac2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 24 Sep 2024 10:03:39 -0700 Subject: [PATCH 4/4] adding ischool to repo list --- scripts/user-image-management/repos.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/user-image-management/repos.txt b/scripts/user-image-management/repos.txt index 6864d75f3..eeeb68632 100644 --- a/scripts/user-image-management/repos.txt +++ b/scripts/user-image-management/repos.txt @@ -7,6 +7,7 @@ git@github.com:berkeley-dsep-infra/data102-user-image.git git@github.com:berkeley-dsep-infra/dev-primary-image.git git@github.com:berkeley-dsep-infra/dev-secondary-image.git git@github.com:berkeley-dsep-infra/eecs-user-image.git +git@github.com:berkeley-dsep-infra/ischool-user-image.git git@github.com:berkeley-dsep-infra/julia-user-image.git git@github.com:berkeley-dsep-infra/logodev-user-image.git git@github.com:berkeley-dsep-infra/nature-user-image.git