From 684412dd092b3d13277268d8203e63686de48728 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Thu, 8 Aug 2024 13:34:06 -0700 Subject: [PATCH 001/190] Reinstall coreutils to get man pages. This ensures that man pages for basic tools exist. There were previously being excluded by dpkg. --- deployments/datahub/images/default/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deployments/datahub/images/default/Dockerfile b/deployments/datahub/images/default/Dockerfile index 49062d34d..451815ffb 100644 --- a/deployments/datahub/images/default/Dockerfile +++ b/deployments/datahub/images/default/Dockerfile @@ -22,7 +22,11 @@ RUN apt-get -qq update --yes && \ RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER} # Do not exclude manpages from being installed. -RUN sed -i -e '/usr.share.man/s/^/#/' /etc/dpkg/dpkg.cfg.d/excludes +RUN sed -i '/usr.share.man/s/^/#/' /etc/dpkg/dpkg.cfg.d/excludes + +# Reinstall coreutils so that basic man pages are installed. Due to dpkg's +# exclusion, they were not originally installed. +RUN apt --reinstall install coreutils # Install all apt packages COPY apt.txt /tmp/apt.txt From 02a5a6ec39798d319cca10b4c8b277977ed6800a Mon Sep 17 00:00:00 2001 From: Greg Merritt Date: Mon, 12 Aug 2024 10:45:04 -0700 Subject: [PATCH 002/190] user pool node resize - Deployment Manager check --- vendor/google/ugr/gke/configs/node_pool_ugr01.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml index 5979cc1af..f742d5e80 100644 --- a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml +++ b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml @@ -11,6 +11,6 @@ resources: dateSuffix: '2024-07-07' initialNodeCount: 1 diskSizeGb: 100 - machineType: n2-standard-2 + machineType: n2-standard-4 minNodeCount: 1 maxNodeCount: 20 From 45bd4de62ae22bb0c99b3a8fdeacec350ab586b3 Mon Sep 17 00:00:00 2001 From: Greg Merritt Date: Mon, 12 Aug 2024 11:05:59 -0700 Subject: [PATCH 003/190] Oops -- not all fields are in-place updatable. Trying w/ min nodes instead. --- vendor/google/ugr/gke/configs/node_pool_ugr01.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml index f742d5e80..4d69373c7 100644 --- a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml +++ b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml @@ -11,6 +11,6 @@ resources: dateSuffix: '2024-07-07' initialNodeCount: 1 diskSizeGb: 100 - machineType: n2-standard-4 - minNodeCount: 1 + machineType: n2-standard-2 + minNodeCount: 2 maxNodeCount: 20 From 125999d6f40f8d33b169d890462cdec1a812a200 Mon Sep 17 00:00:00 2001 From: Greg Merritt Date: Mon, 12 Aug 2024 11:10:52 -0700 Subject: [PATCH 004/190] Settings updated, but pool nodes unchanged. Trying both initial++ & min++ --- vendor/google/ugr/gke/configs/node_pool_ugr01.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml index 4d69373c7..fc9619719 100644 --- a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml +++ b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml @@ -9,8 +9,8 @@ resources: clusterName: ugresearch-cluster region: us-central1 dateSuffix: '2024-07-07' - initialNodeCount: 1 + initialNodeCount: 2 diskSizeGb: 100 machineType: n2-standard-2 - minNodeCount: 2 + minNodeCount: 3 maxNodeCount: 20 From 68bc0e02980c2402419bb800cfb0373e4c0b35c4 Mon Sep 17 00:00:00 2001 From: Greg Merritt Date: Mon, 12 Aug 2024 11:14:41 -0700 Subject: [PATCH 005/190] Back to defaults. ("initial" was obeyed, but "min++" seems to not cause ++.) --- vendor/google/ugr/gke/configs/node_pool_ugr01.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml index fc9619719..5979cc1af 100644 --- a/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml +++ b/vendor/google/ugr/gke/configs/node_pool_ugr01.yaml @@ -9,8 +9,8 @@ resources: clusterName: ugresearch-cluster region: us-central1 dateSuffix: '2024-07-07' - initialNodeCount: 2 + initialNodeCount: 1 diskSizeGb: 100 machineType: n2-standard-2 - minNodeCount: 3 + minNodeCount: 1 maxNodeCount: 20 From 33ef944b5e219cde861ad82c48f838bbe71a365c Mon Sep 17 00:00:00 2001 From: Greg Merritt Date: Mon, 12 Aug 2024 11:23:51 -0700 Subject: [PATCH 006/190] Deployment update behavior notes added to ugr vendor gke README.md --- vendor/google/ugr/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vendor/google/ugr/README.md b/vendor/google/ugr/README.md index 4ca1367d6..30a2b61e0 100644 --- a/vendor/google/ugr/README.md +++ b/vendor/google/ugr/README.md @@ -85,4 +85,6 @@ Run the shell script to deploy: ## Change a deployment -To change an existing deployment, edit the parameters to be changed in the deployment's yaml config and run its associated shell script. \ No newline at end of file +To change an existing deployment, edit the parameters to be changed in the deployment's yaml config and run its associated shell script. + +Note that not all parameters can be updated in place. For example, node pool autoscaling (such as min/max numbers and on/off) can be updated on the "hot" deployment, but other config settiongs (e.g., machine type and disk size) require the resources to first be undeployed, then deployed anew with the new configuration. From 4d45689aac8d04aea0d4d2a4eed20139281cffd0 Mon Sep 17 00:00:00 2001 From: Greg Merritt Date: Mon, 12 Aug 2024 11:25:10 -0700 Subject: [PATCH 007/190] README.md typo fixed --- vendor/google/ugr/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/google/ugr/README.md b/vendor/google/ugr/README.md index 30a2b61e0..7d1667975 100644 --- a/vendor/google/ugr/README.md +++ b/vendor/google/ugr/README.md @@ -87,4 +87,4 @@ Run the shell script to deploy: To change an existing deployment, edit the parameters to be changed in the deployment's yaml config and run its associated shell script. -Note that not all parameters can be updated in place. For example, node pool autoscaling (such as min/max numbers and on/off) can be updated on the "hot" deployment, but other config settiongs (e.g., machine type and disk size) require the resources to first be undeployed, then deployed anew with the new configuration. +Note that not all parameters can be updated in place. For example, node pool autoscaling (such as min/max numbers and on/off) can be updated on the "hot" deployment, but other config settings (e.g., machine type and disk size) require the resources to first be undeployed, then deployed anew with the new configuration. From e48665f7f9cd2b21c11c30a39709a1ad09828053 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Mon, 12 Aug 2024 17:55:45 -0700 Subject: [PATCH 008/190] Set up Datahub for MBA 247 --- deployments/datahub/config/common.yaml | 14 ++++++++++++++ .../images/default/r-packages/2023-fall-mba-247.r | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index 0cc9c5680..8aff4a141 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -64,6 +64,20 @@ jupyterhub: - course::1535811::enrollment_type::teacher - course::1535811::enrollment_type::ta + # MBA 247, Fall 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5923 + course-staff-1536576: + # description: Enable course staff to view and access servers. + # # this role provides permissions to... + scopes: + - admin-ui + - list:users!group=course::1536576 + - admin:servers!group=course::1536576 + - access:servers!group=course::1536576 + # # this role will be assigned to... + groups: + - course::1536576::enrollment_type::teacher + - course::1536576::enrollment_type::ta + nodeSelector: hub.jupyter.org/pool-name: core-pool-2024-05-08 initContainers: diff --git a/deployments/datahub/images/default/r-packages/2023-fall-mba-247.r b/deployments/datahub/images/default/r-packages/2023-fall-mba-247.r index 31b6da47c..660f6184f 100644 --- a/deployments/datahub/images/default/r-packages/2023-fall-mba-247.r +++ b/deployments/datahub/images/default/r-packages/2023-fall-mba-247.r @@ -14,6 +14,7 @@ class_libs = c( "pROC", "1.18.4", "rpart.plot", "3.1.1", "randomForest","4.7-1.1", - "xgboost","1.7.5.1" + "xgboost","1.7.5.1", + "factoextra","1.0.7" ) class_libs_install_version(class_name, class_libs) From 33d133930e08676625f1d66db97821b7e20b7838 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 13 Aug 2024 10:28:54 -0700 Subject: [PATCH 009/190] bumping labeler to v5 --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 07b21837d..2a340279e 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -9,4 +9,4 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 From bb8532dce164b83bd76d49a4449aa7a3cd188f1c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 13 Aug 2024 10:52:36 -0700 Subject: [PATCH 010/190] preparing for more hubploy testing --- deployments/logodev/hubploy.yaml | 5 +- deployments/logodev/image/apt.txt | 93 ------------------- deployments/logodev/image/environment.yml | 65 ------------- .../logodev/image/infra-requirements.txt | 29 ------ deployments/logodev/image/postBuild | 6 -- 5 files changed, 1 insertion(+), 197 deletions(-) delete mode 100644 deployments/logodev/image/apt.txt delete mode 100644 deployments/logodev/image/environment.yml delete mode 100644 deployments/logodev/image/infra-requirements.txt delete mode 100644 deployments/logodev/image/postBuild diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 7760f94a1..36e817577 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,10 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a02e39fbaedb - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:829f9eaececd registry: provider: gcloud gcloud: diff --git a/deployments/logodev/image/apt.txt b/deployments/logodev/image/apt.txt deleted file mode 100644 index a6c4589ed..000000000 --- a/deployments/logodev/image/apt.txt +++ /dev/null @@ -1,93 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download tools -curl -wget - -# Core text editors on a *nix box: vim -vim - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# powerful terminal-based file manager, better than the one in JLab -mc - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -## This section adds tools for desktop environment usage -dbus-x11 -xorg -xubuntu-icon-theme -xfce4 -xfce4-goodies -xclip -xsel -firefox -chromium-browser - -# GUI text editors -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml deleted file mode 100644 index 386806bea..000000000 --- a/deployments/logodev/image/environment.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: data100-FA23 - -channels: -- conda-forge -- pytorch - -dependencies: -- python==3.11.* -- git==2.39.1 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 -- folium==0.14.0 -- h5netcdf==1.0.2 -- ipywidgets==8.0.7 -- jupysql==0.8.0 -- jupyter-archive==3.4.0 -- matplotlib==3.7.1 -- mdit-py-plugins==0.4.0 -- numpy==1.24.2 -- pandas==2.2.2 -- plotly==5.13.1 -- requests==2.28.2 -- scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 -- seaborn==0.12.2 -- statsmodels==0.14.0 -- tensorflow-cpu==2.12.1 -- sqlalchemy==2.0.16 -- mlxtend==0.23.0 -# Spring 2024 data 100 -- pytorch==2.1.2 -- cpuonly==2.0 -- transformers==4.37.1 -# Spring 2024 table demos -- lxml==5.1.0 -# Spring 2024 Econ 148 Packages -- geopandas==0.14.2 -- geopy==2.4.1 -- lifelines==0.27.8 -- pycountry==22.3.5 -- pip -- pip: - # - -r infra-requirements.txt - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 - - nbconvert[webpdf] - # - pyppeteer==2.0.0 - - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 - - otter-grader==5.4.0 - - duckdb==0.10.1 - - duckdb_engine==0.11.2 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 diff --git a/deployments/logodev/image/infra-requirements.txt b/deployments/logodev/image/infra-requirements.txt deleted file mode 100644 index b01c9d80d..000000000 --- a/deployments/logodev/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/logodev/image/postBuild b/deployments/logodev/image/postBuild deleted file mode 100644 index 9514de431..000000000 --- a/deployments/logodev/image/postBuild +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -npm install -g playwright -playwright install chromium From b3c24f0bfc3c129874a9885122ece9e6d63af270 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 13 Aug 2024 11:15:47 -0700 Subject: [PATCH 011/190] debumping labeler for now --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 2a340279e..07b21837d 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -9,4 +9,4 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@v5 + - uses: actions/labeler@v4 From 3572e0edb784ee27677bc9d8fbcb59ef1eb321cf Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Wed, 14 Aug 2024 09:47:44 -0700 Subject: [PATCH 012/190] moving data102 to small-courses-filestore --- deployments/data102/config/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/data102/config/common.yaml b/deployments/data102/config/common.yaml index ef4e3626c..1b602bb81 100644 --- a/deployments/data102/config/common.yaml +++ b/deployments/data102/config/common.yaml @@ -1,7 +1,7 @@ nfsPVC: enabled: true nfs: - serverIP: 172.31.130.242 + serverIP: 10.185.238.42 jupyterhub: scheduling: From 2abbc5d21e2f1e16ad36e25cd0b628e3ae3aa930 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 14 Aug 2024 14:30:35 -0700 Subject: [PATCH 013/190] update chp to 4.6.2 --- hub/Chart.yaml | 2 +- hub/values.yaml | 4 ++-- node-placeholder/Chart.yaml | 2 +- node-placeholder/values.yaml | 5 +++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hub/Chart.yaml b/hub/Chart.yaml index e613e06cb..6bc9df43d 100644 --- a/hub/Chart.yaml +++ b/hub/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v1 appVersion: '1.0' description: Deployment Chart for JupyterHub name: hub -version: 0.0.1-0.dev.git.7608.ha33ba6ef +version: 20240731-224556 diff --git a/hub/values.yaml b/hub/values.yaml index 0efd1c28f..b127ebac0 100644 --- a/hub/values.yaml +++ b/hub/values.yaml @@ -40,7 +40,7 @@ jupyterhub: # https://github.com/consideRatio/configurable-http-proxy/commits/main/ # image: - tag: 4.6.1-fork + tag: 4.6.2 extraCommandLineFlags: # set the timeout and proxyTimeout to 24 hours (arg is in milliseconds) # https://github.com/http-party/node-http-proxy?tab=readme-ov-file#options @@ -241,7 +241,7 @@ jupyterhub: # Generated by chartpress image: name: gcr.io/ucb-datahub-2018/jupyterhub-hub - tag: '0.0.1-0.dev.git.7607.h4497642d' + tag: '20240731-224556' networkPolicy: enabled: true # interNamespaceAccessLabels=accept makes the hub pod's associated diff --git a/node-placeholder/Chart.yaml b/node-placeholder/Chart.yaml index 0f7bcc317..4db5f93e7 100644 --- a/node-placeholder/Chart.yaml +++ b/node-placeholder/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.1-0.dev.git.7608.ha33ba6ef +version: 20240731-224556 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index f6760f10b..679e2901c 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -4,7 +4,7 @@ image: repository: us-central1-docker.pkg.dev/ucb-datahub-2018/core/node-placeholder-scaler pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.1-0.dev.git.6946.hdccbf2f4" + tag: "20240731-224556" imagePullSecrets: [] nameOverride: "" @@ -48,7 +48,8 @@ priorityClass: tolerations: -calendarUrl: https://calendar.google.com/calendar/ical/c_s47m3m1nuj3s81187k3b2b5s5o%40group.calendar.google.com/public/basic.ics +calendarUrl: + https://calendar.google.com/calendar/ical/c_s47m3m1nuj3s81187k3b2b5s5o%40group.calendar.google.com/public/basic.ics grafana: tags: From 5baf2e523ecf03a062f25ca2f13fe7b6bc54138d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:13:03 -0700 Subject: [PATCH 014/190] adding hubploy testing --- .github/workflows/deploy-to-staging.yaml | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/deploy-to-staging.yaml diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml new file mode 100644 index 000000000..d6e750723 --- /dev/null +++ b/.github/workflows/deploy-to-staging.yaml @@ -0,0 +1,37 @@ +name: Deploy images to hubs + +on: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + files_ignore: | + .github/** + images/** \ No newline at end of file From 2dc3f5a219353b03b233139b0ad38f7cdcf5894c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:17:07 -0700 Subject: [PATCH 015/190] bumping jupyterhub --- deployments/a11y/image/environment.yml | 2 +- deployments/a11y/image/infra-requirements.txt | 2 +- deployments/astro/image/infra-requirements.txt | 2 +- deployments/biology/image/infra-requirements.txt | 2 +- deployments/cee/image/infra-requirements.txt | 2 +- deployments/data100/image/environment.yml | 2 +- deployments/data100/image/infra-requirements.txt | 2 +- deployments/data101/image/environment.yml | 2 +- deployments/data101/image/infra-requirements.txt | 2 +- deployments/data102/image/environment.yml | 2 +- deployments/data102/image/infra-requirements.txt | 2 +- deployments/data8/image/environment.yml | 2 +- deployments/data8/image/infra-requirements.txt | 2 +- deployments/datahub/images/default/infra-requirements.txt | 2 +- deployments/dev/images/default/infra-requirements.txt | 2 +- deployments/dev/images/secondary/environment.yml | 2 +- deployments/dev/images/secondary/infra-requirements.txt | 2 +- deployments/edx/image/environment.yml | 2 +- deployments/edx/image/infra-requirements.txt | 2 +- deployments/eecs/image/infra-requirements.txt | 2 +- deployments/ischool/image/infra-requirements.txt | 2 +- deployments/julia/image/infra-requirements.txt | 2 +- deployments/logodev/image/environment.yml | 2 +- deployments/logodev/image/infra-requirements.txt | 2 +- deployments/publichealth/image/infra-requirements.txt | 2 +- deployments/shiny/image/environment.yml | 2 +- deployments/shiny/image/infra-requirements.txt | 2 +- deployments/stat159/image/environment.yml | 2 +- deployments/stat159/image/infra-requirements.txt | 2 +- deployments/stat20/image/infra-requirements.txt | 2 +- deployments/ugr01/image/environment.yml | 2 +- deployments/ugr01/image/infra-requirements.txt | 2 +- scripts/infra-packages/requirements.txt | 2 +- 33 files changed, 33 insertions(+), 33 deletions(-) diff --git a/deployments/a11y/image/environment.yml b/deployments/a11y/image/environment.yml index a67f3415d..8723ab955 100644 --- a/deployments/a11y/image/environment.yml +++ b/deployments/a11y/image/environment.yml @@ -11,7 +11,7 @@ dependencies: - jupyter_server==2.7.0 - jupyter-book==0.15.1 - jupyter-resource-usage==0.7.1 -- jupyterhub==4.1.5 +- jupyterhub==4.1.6 - matplotlib==3.7.1 - nbgitpuller==1.2.1 - nbclassic==1.0.0 diff --git a/deployments/a11y/image/infra-requirements.txt b/deployments/a11y/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/a11y/image/infra-requirements.txt +++ b/deployments/a11y/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/astro/image/infra-requirements.txt b/deployments/astro/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/astro/image/infra-requirements.txt +++ b/deployments/astro/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/biology/image/infra-requirements.txt b/deployments/biology/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/biology/image/infra-requirements.txt +++ b/deployments/biology/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/cee/image/infra-requirements.txt b/deployments/cee/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/cee/image/infra-requirements.txt +++ b/deployments/cee/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/data100/image/environment.yml b/deployments/data100/image/environment.yml index ffbd747dd..3d0da5bb0 100644 --- a/deployments/data100/image/environment.yml +++ b/deployments/data100/image/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/deployments/data100/image/infra-requirements.txt b/deployments/data100/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/data100/image/infra-requirements.txt +++ b/deployments/data100/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/data101/image/environment.yml b/deployments/data101/image/environment.yml index 7436db03a..5c255e877 100644 --- a/deployments/data101/image/environment.yml +++ b/deployments/data101/image/environment.yml @@ -100,7 +100,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbconvert[webpdf]==7.16.4 - pytest-notebook==0.8.1 - pymongo==4.4.1 diff --git a/deployments/data101/image/infra-requirements.txt b/deployments/data101/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/data101/image/infra-requirements.txt +++ b/deployments/data101/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/data102/image/environment.yml b/deployments/data102/image/environment.yml index 2e74cf4b6..0d70715dd 100644 --- a/deployments/data102/image/environment.yml +++ b/deployments/data102/image/environment.yml @@ -46,7 +46,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbconvert[webpdf]==7.16.4 - gh-scoped-creds==4.1 - nb2pdf==0.6.2 diff --git a/deployments/data102/image/infra-requirements.txt b/deployments/data102/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/data102/image/infra-requirements.txt +++ b/deployments/data102/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/data8/image/environment.yml b/deployments/data8/image/environment.yml index ad3b5a542..d94563421 100644 --- a/deployments/data8/image/environment.yml +++ b/deployments/data8/image/environment.yml @@ -4,7 +4,7 @@ dependencies: - python==3.11.* # Packages ported from infra-reqs -- jupyterhub==4.1.5 +- jupyterhub==4.1.6 - jupyter-resource-usage==1.0.1 - nbgitpuller==1.2.1 - syncthing==1.18.6 diff --git a/deployments/data8/image/infra-requirements.txt b/deployments/data8/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/data8/image/infra-requirements.txt +++ b/deployments/data8/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/datahub/images/default/infra-requirements.txt b/deployments/datahub/images/default/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/datahub/images/default/infra-requirements.txt +++ b/deployments/datahub/images/default/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/dev/images/default/infra-requirements.txt b/deployments/dev/images/default/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/dev/images/default/infra-requirements.txt +++ b/deployments/dev/images/default/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml index 2da3a6263..13e05aea6 100644 --- a/deployments/dev/images/secondary/environment.yml +++ b/deployments/dev/images/secondary/environment.yml @@ -7,7 +7,7 @@ channels: dependencies: - python==3.11.* - git==2.39.1 -- jupyterhub==4.1.5 +- jupyterhub==4.1.6 - jupyter_app_launcher==0.2.1 - jupyter-cache==1.0.0 - jupyter-resource-usage==1.0.0 diff --git a/deployments/dev/images/secondary/infra-requirements.txt b/deployments/dev/images/secondary/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/dev/images/secondary/infra-requirements.txt +++ b/deployments/dev/images/secondary/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/edx/image/environment.yml b/deployments/edx/image/environment.yml index df01bc927..99235e91a 100644 --- a/deployments/edx/image/environment.yml +++ b/deployments/edx/image/environment.yml @@ -39,7 +39,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - jupyterlab-accessible-themes==0.2.0 - nbconvert==7.6.0 - pytest-notebook==0.8.1 diff --git a/deployments/edx/image/infra-requirements.txt b/deployments/edx/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/edx/image/infra-requirements.txt +++ b/deployments/edx/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/eecs/image/infra-requirements.txt b/deployments/eecs/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/eecs/image/infra-requirements.txt +++ b/deployments/eecs/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/ischool/image/infra-requirements.txt b/deployments/ischool/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/ischool/image/infra-requirements.txt +++ b/deployments/ischool/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/julia/image/infra-requirements.txt b/deployments/julia/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/julia/image/infra-requirements.txt +++ b/deployments/julia/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml index 386806bea..c4de2ee40 100644 --- a/deployments/logodev/image/environment.yml +++ b/deployments/logodev/image/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/deployments/logodev/image/infra-requirements.txt b/deployments/logodev/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/logodev/image/infra-requirements.txt +++ b/deployments/logodev/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/publichealth/image/infra-requirements.txt b/deployments/publichealth/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/publichealth/image/infra-requirements.txt +++ b/deployments/publichealth/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/shiny/image/environment.yml b/deployments/shiny/image/environment.yml index abc7bc0b1..88a784784 100644 --- a/deployments/shiny/image/environment.yml +++ b/deployments/shiny/image/environment.yml @@ -6,7 +6,7 @@ dependencies: - jupyter-server-proxy==4.2.0 - jupyter-rsession-proxy==2.2.0 - jupyter-syncthing-proxy==1.0.3 -- jupyterhub==4.1.5 +- jupyterhub==4.1.6 - jupyterlab==4.0.11 - nbgitpuller==1.2.1 - notebook==7.0.7 diff --git a/deployments/shiny/image/infra-requirements.txt b/deployments/shiny/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/shiny/image/infra-requirements.txt +++ b/deployments/shiny/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/stat159/image/environment.yml b/deployments/stat159/image/environment.yml index b3f7a0358..0ce2a9864 100644 --- a/deployments/stat159/image/environment.yml +++ b/deployments/stat159/image/environment.yml @@ -150,7 +150,7 @@ dependencies: - pip: - jupysql==0.10.5 - ipytest==0.13.0 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbval==0.10.0 - nbdime==3.2.0 diff --git a/deployments/stat159/image/infra-requirements.txt b/deployments/stat159/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/stat159/image/infra-requirements.txt +++ b/deployments/stat159/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/stat20/image/infra-requirements.txt b/deployments/stat20/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/stat20/image/infra-requirements.txt +++ b/deployments/stat20/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/deployments/ugr01/image/environment.yml b/deployments/ugr01/image/environment.yml index 9ee6fc0fa..058130f05 100644 --- a/deployments/ugr01/image/environment.yml +++ b/deployments/ugr01/image/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/deployments/ugr01/image/infra-requirements.txt b/deployments/ugr01/image/infra-requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/deployments/ugr01/image/infra-requirements.txt +++ b/deployments/ugr01/image/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 diff --git a/scripts/infra-packages/requirements.txt b/scripts/infra-packages/requirements.txt index b01c9d80d..0fb0bd930 100644 --- a/scripts/infra-packages/requirements.txt +++ b/scripts/infra-packages/requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 9643afd46ee759ab31f961d2161d9de122589d37 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:23:03 -0700 Subject: [PATCH 016/190] also bump helm chart --- hub/requirements.yaml | 2 +- images/hub/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/requirements.yaml b/hub/requirements.yaml index 59f3b765b..bb1f726e6 100644 --- a/hub/requirements.yaml +++ b/hub/requirements.yaml @@ -4,5 +4,5 @@ dependencies: # get the helm chart version from here: https://github.com/jupyterhub/helm-chart # correlate the helm chart version w/jupyterhub here: # https://hub.jupyter.org/helm-chart/#development-releases-jupyterhub - version: 3.3.7 + version: 3.3.8 repository: https://jupyterhub.github.io/helm-chart/ diff --git a/images/hub/Dockerfile b/images/hub/Dockerfile index 6ff9a5c96..89fe0861f 100644 --- a/images/hub/Dockerfile +++ b/images/hub/Dockerfile @@ -1,6 +1,6 @@ # Should match the hub image used by version of chart in hub/requirements.yaml # If that changes, this should be changed too! -FROM jupyterhub/k8s-hub:3.3.7 +FROM jupyterhub/k8s-hub:3.3.8 USER root From 9ac1d78e2fdc77e0ed57b93e18831b7e8c596c96 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 15 Aug 2024 10:23:43 -0700 Subject: [PATCH 017/190] moving dlab and publichealth to small-course-filestore --- deployments/dlab/config/common.yaml | 2 +- deployments/publichealth/config/common.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/dlab/config/common.yaml b/deployments/dlab/config/common.yaml index ae7b435d6..0ab293b78 100644 --- a/deployments/dlab/config/common.yaml +++ b/deployments/dlab/config/common.yaml @@ -1,7 +1,7 @@ nfsPVC: enabled: true nfs: - serverIP: 10.50.250.234 + serverIP: 10.185.238.42 jupyterhub: scheduling: diff --git a/deployments/publichealth/config/common.yaml b/deployments/publichealth/config/common.yaml index 44f92504f..884246423 100644 --- a/deployments/publichealth/config/common.yaml +++ b/deployments/publichealth/config/common.yaml @@ -8,7 +8,7 @@ etcGitConfig: nfsPVC: enabled: true nfs: - serverIP: 10.204.227.122 + serverIP: 10.185.238.42 jupyterhub: scheduling: From f4c46c3655a22cd2cc28f85e88d5f9490a83e138 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:35:32 -0700 Subject: [PATCH 018/190] fixing astro environment --- deployments/astro/image/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 325a15304..ce81f1033 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -72,7 +72,7 @@ dependencies: - lightkurve - ipyparallel>=8.1.0 - jupyter-tensorboard>=0.2.0 - - line_profiler>-3.4.0 + - line_profiler>=3.4.0 - snakeviz>=2.1.1 - memory_profiler>=0.60.0 - flask>=2.0.2 From f384f3d1a3061f35ea800c25d4c2ac3f59f7d9a1 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:46:05 -0700 Subject: [PATCH 019/190] chartpressed --- hub/Chart.yaml | 2 +- hub/values.yaml | 2 +- node-placeholder/Chart.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hub/Chart.yaml b/hub/Chart.yaml index 6bc9df43d..50444a64a 100644 --- a/hub/Chart.yaml +++ b/hub/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v1 appVersion: '1.0' description: Deployment Chart for JupyterHub name: hub -version: 20240731-224556 +version: 20240731-224556.git.8057.h9643afd4 diff --git a/hub/values.yaml b/hub/values.yaml index b127ebac0..95b93d9a9 100644 --- a/hub/values.yaml +++ b/hub/values.yaml @@ -241,7 +241,7 @@ jupyterhub: # Generated by chartpress image: name: gcr.io/ucb-datahub-2018/jupyterhub-hub - tag: '20240731-224556' + tag: '20240731-224556.git.8057.h9643afd4' networkPolicy: enabled: true # interNamespaceAccessLabels=accept makes the hub pod's associated diff --git a/node-placeholder/Chart.yaml b/node-placeholder/Chart.yaml index 4db5f93e7..7753e9f66 100644 --- a/node-placeholder/Chart.yaml +++ b/node-placeholder/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 20240731-224556 +version: 20240731-224556.git.8054.h2abbc5d2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 2339589dcdbe46d17bc25e3a596596fd2a86d93f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 09:50:16 -0700 Subject: [PATCH 020/190] placeholder commit --- .github/workflows/deploy-to-staging.yaml | 30 +++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index d6e750723..c041d210d 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -1,5 +1,5 @@ -name: Deploy images to hubs - +name: Deploy images to staging hubs +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it on: push: branches: @@ -9,6 +9,12 @@ jobs: deploy: runs-on: ubuntu-latest steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout uses: actions/checkout@v4 @@ -28,10 +34,18 @@ jobs: pip install -r requirements.txt pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v44 + - name: Install Google Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.0 with: - files_ignore: | - .github/** - images/** \ No newline at end of file + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + + - name: Install SOPS + run: | + echo $SOPS_ACCOUNT_KEY > ${HOME}/repo/sops.key + echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/repo/sops.key' >> ${BASH_ENV} + mkdir -p ${HOME}/repo/bin + curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops + chmod 755 ${HOME}/repo/bin/sops + echo 'export PATH="${HOME}/repo/bin:${PATH}"' >> ${BASH_ENV} \ No newline at end of file From e361daf40aad74889888315bcec01638ea083ffd Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 11:27:58 -0700 Subject: [PATCH 021/190] add github workflow for linting yaml --- .github/workflows/yaml-lint.yaml | 14 ++++++++++++++ requirements.txt | 1 + 2 files changed, 15 insertions(+) create mode 100644 .github/workflows/yaml-lint.yaml diff --git a/.github/workflows/yaml-lint.yaml b/.github/workflows/yaml-lint.yaml new file mode 100644 index 000000000..d38c990a5 --- /dev/null +++ b/.github/workflows/yaml-lint.yaml @@ -0,0 +1,14 @@ +--- +on: pull-request # yamllint disable-line rule:truthy + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install yamllint + run: pip install yamllint==1.35.1 + + - name: Lint YAML files + run: yamllint --no-warnings . diff --git a/requirements.txt b/requirements.txt index da3a1ba42..4eacc64bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ chardet requests==2.31.0 # requests==2.32.3 urllib3<2.0.0 +yamllint==1.35.1 From 2dc547189098c002794161ef7659c8661b01fc2c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 11:34:05 -0700 Subject: [PATCH 022/190] minor tweaks --- .github/workflows/yaml-lint.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/yaml-lint.yaml b/.github/workflows/yaml-lint.yaml index d38c990a5..b5feb3464 100644 --- a/.github/workflows/yaml-lint.yaml +++ b/.github/workflows/yaml-lint.yaml @@ -1,5 +1,6 @@ ---- -on: pull-request # yamllint disable-line rule:truthy +name: "Yaml lint" +on: + - pull-request # yamllint disable-line rule:truthy jobs: lint: From 9879146e0d5e0de974bfd18e9b3371a41d9f0402 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 11:35:58 -0700 Subject: [PATCH 023/190] testing yamllint --- chartpress.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/chartpress.yaml b/chartpress.yaml index eaaacc141..4b8049529 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -1,3 +1,4 @@ +asdfasdf charts: - name: hub imagePrefix: gcr.io/ucb-datahub-2018/jupyterhub- From 1981cd3f750f0166c8313018568fc35741c5081e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 11:40:14 -0700 Subject: [PATCH 024/190] fixing event name --- .github/workflows/yaml-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/yaml-lint.yaml b/.github/workflows/yaml-lint.yaml index b5feb3464..787d5e3fa 100644 --- a/.github/workflows/yaml-lint.yaml +++ b/.github/workflows/yaml-lint.yaml @@ -1,6 +1,6 @@ name: "Yaml lint" on: - - pull-request # yamllint disable-line rule:truthy + - pull_request # yamllint disable-line rule:truthy jobs: lint: From 217263f067dfc8eca9f2f6f275c8865c6b29c615 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 16 Aug 2024 11:41:24 -0700 Subject: [PATCH 025/190] fixing chartpress.yaml --- chartpress.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/chartpress.yaml b/chartpress.yaml index 4b8049529..eaaacc141 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -1,4 +1,3 @@ -asdfasdf charts: - name: hub imagePrefix: gcr.io/ucb-datahub-2018/jupyterhub- From b84d3707b00b0ae6122fa606ebddbd35843741c8 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 10:28:27 -0700 Subject: [PATCH 026/190] initial commit to test gcloud github actions --- .github/workflows/deploy-to-staging.yaml | 22 +++++++++++++++------- deployments/logodev/hubploy.yaml | 5 ----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index c041d210d..275cfcc46 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -1,6 +1,7 @@ name: Deploy images to staging hubs # use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it on: + workflow_dispatch: push: branches: - staging @@ -28,24 +29,31 @@ jobs: with: fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + - name: print out labels just for testing + run: echo "${{ steps.pr-labels.outputs.labels }}" + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + hubploy --help + hubploy deploy --help - - name: Install Google Cloud SDK - uses: google-github-actions/setup-gcloud@v0.2.0 + - name: Auth to gcloud + uses: google-github-actions/auth@v2 with: + credentials_json: ${{ secrets.GCP_SA_KEY }} project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true + + - name: Install Google Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: gcloud info + run: gcloud info - name: Install SOPS run: | - echo $SOPS_ACCOUNT_KEY > ${HOME}/repo/sops.key - echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/repo/sops.key' >> ${BASH_ENV} mkdir -p ${HOME}/repo/bin curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops chmod 755 ${HOME}/repo/bin/sops - echo 'export PATH="${HOME}/repo/bin:${PATH}"' >> ${BASH_ENV} \ No newline at end of file diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 36e817577..fd2b36d2a 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -2,11 +2,6 @@ images: images: # temporary update - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:829f9eaececd - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json cluster: provider: gcloud From c8103463e858f8198a6f81cb6e4a83eb9db1c8d4 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 10:32:40 -0700 Subject: [PATCH 027/190] why is there a conflict --- deployments/logodev/image/apt.txt | 93 +++++++++++++++++++ deployments/logodev/image/environment.yml | 65 +++++++++++++ .../logodev/image/infra-requirements.txt | 29 ++++++ deployments/logodev/image/postBuild | 6 ++ 4 files changed, 193 insertions(+) create mode 100644 deployments/logodev/image/apt.txt create mode 100644 deployments/logodev/image/environment.yml create mode 100644 deployments/logodev/image/infra-requirements.txt create mode 100644 deployments/logodev/image/postBuild diff --git a/deployments/logodev/image/apt.txt b/deployments/logodev/image/apt.txt new file mode 100644 index 000000000..a6c4589ed --- /dev/null +++ b/deployments/logodev/image/apt.txt @@ -0,0 +1,93 @@ +# Some linux packages for basic terminal work, particularly +# oriented at users new to Unix/cmd line environments. + +# installing less as more just isn't enough +less + +# Basic unix tools +man +man-db +manpages-posix +manpages-dev +manpages-posix-dev + +# Download tools +curl +wget + +# Core text editors on a *nix box: vim +vim + +# A couple of CLI editors that are easier than vim +# micro # currently not working on 18.04 +nano +jed +jed-extra + +# powerful terminal-based file manager, better than the one in JLab +mc + +# for easily managing multiple repositories with one command (perl-doc +# is needed for its help pages to work) +mr +perl-doc + +# Regular build tools for compiling common stuff +build-essential +gfortran + +# Dependencies for nbconvert +texlive-xetex +texlive-fonts-recommended +texlive-plain-generic +# https://github.com/berkeley-dsep-infra/datahub/issues/3719 +texlive-lang-chinese +lmodern + +# Other useful document-related tools +pandoc +latexdiff + +# Some useful git utilities use basic Ruby +ruby + +# Other niceties for command-line work and life +ack # powerful grep-like tool +pydf # colorized disk usage +tmux +screen +htop +nnn # cmd line file manager +zsh +rsync +tig # console UI for git +multitail + +# For later, these are not available in 18.04 +#browsh # text-based web browser, occasionally handy +#dasel # json/yml/csv/etc data wrangling at the terminal +#fzf # fuzzy file finder + +## This section adds tools for desktop environment usage +dbus-x11 +xorg +xubuntu-icon-theme +xfce4 +xfce4-goodies +xclip +xsel +firefox +chromium-browser + +# GUI text editors +vim-gtk3 +gedit + +# Git clients and tools +git-gui +gitg +qgit +meld + +# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 +zip diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml new file mode 100644 index 000000000..c4de2ee40 --- /dev/null +++ b/deployments/logodev/image/environment.yml @@ -0,0 +1,65 @@ +name: data100-FA23 + +channels: +- conda-forge +- pytorch + +dependencies: +- python==3.11.* +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- nbgitpuller==1.2.1 +- notebook==7.0.7 +- folium==0.14.0 +- h5netcdf==1.0.2 +- ipywidgets==8.0.7 +- jupysql==0.8.0 +- jupyter-archive==3.4.0 +- matplotlib==3.7.1 +- mdit-py-plugins==0.4.0 +- numpy==1.24.2 +- pandas==2.2.2 +- plotly==5.13.1 +- requests==2.28.2 +- scikit-image==0.19.3 +- scikit-learn==1.2.2 +- scipy==1.10.1 +- seaborn==0.12.2 +- statsmodels==0.14.0 +- tensorflow-cpu==2.12.1 +- sqlalchemy==2.0.16 +- mlxtend==0.23.0 +# Spring 2024 data 100 +- pytorch==2.1.2 +- cpuonly==2.0 +- transformers==4.37.1 +# Spring 2024 table demos +- lxml==5.1.0 +# Spring 2024 Econ 148 Packages +- geopandas==0.14.2 +- geopy==2.4.1 +- lifelines==0.27.8 +- pycountry==22.3.5 +- pip +- pip: + # - -r infra-requirements.txt + - ipywidgets==8.0.7 + # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) + # - jupyter_collaboration==1.0.1 + - jupyterhub==4.1.6 + - nbconvert[webpdf] + # - pyppeteer==2.0.0 + - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 + - otter-grader==5.4.0 + - duckdb==0.10.1 + - duckdb_engine==0.11.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/logodev/image/infra-requirements.txt b/deployments/logodev/image/infra-requirements.txt new file mode 100644 index 000000000..0fb0bd930 --- /dev/null +++ b/deployments/logodev/image/infra-requirements.txt @@ -0,0 +1,29 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +notebook==7.0.7 +jupyterlab==4.0.11 +nbgitpuller==1.2.1 +jupyter-resource-usage==1.0.1 +# Matches version in images/hub/Dockerfile +jupyterhub==4.1.6 +appmode==0.8.0 +ipywidgets==8.0.7 +jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling +# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 +RISE==5.7.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.3 +# jupyter archival tool for easy user downloads +jupyter-archive==3.4.0 diff --git a/deployments/logodev/image/postBuild b/deployments/logodev/image/postBuild new file mode 100644 index 000000000..9514de431 --- /dev/null +++ b/deployments/logodev/image/postBuild @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +npm install -g playwright +playwright install chromium From 763f4ef0a33e381031db41d96f7c35682cf5d946 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 10:40:58 -0700 Subject: [PATCH 028/190] whoopsie --- .github/workflows/deploy-to-staging.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index 275cfcc46..5bbe82a64 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -43,7 +43,7 @@ jobs: - name: Auth to gcloud uses: google-github-actions/auth@v2 with: - credentials_json: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GKE_KEY }} project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Install Google Cloud SDK From 7e9f239dafc0f35f53e76a119275465249eb5ed8 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 11:47:02 -0700 Subject: [PATCH 029/190] testing hubploy --- .github/workflows/deploy-to-staging.yaml | 55 +++++++++++++++++------- deployments/logodev/hubploy.yaml | 2 +- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index 5bbe82a64..34a616f9d 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -16,44 +16,69 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Check out the image repo uses: actions/checkout@v4 with: fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - name: print out labels just for testing - run: echo "${{ steps.pr-labels.outputs.labels }}" + - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + HUBS=() + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == hub-* ]]; then + label=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$label" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + echo "Hubs to deploy: $HUBS" + echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' - name: Install dependencies + if: ${{ env.DEPLOY }} run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - hubploy --help - hubploy deploy --help - name: Auth to gcloud + if: ${{ env.DEPLOY }} uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GKE_KEY }} project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} uses: google-github-actions/setup-gcloud@v2 - - name: gcloud info - run: gcloud info - - name: Install SOPS + if: ${{ env.DEPLOY }} run: | mkdir -p ${HOME}/repo/bin curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops chmod 755 ${HOME}/repo/bin/sops + echo "${HOME}/repo/bin" >> $GITHUB_PATH + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy hubs to staging + if: ${{ env.DEPLOY }} + run: | + for hub in $DEPLOY_HUBS[@]; do + echo "Deploying $hub to staging" + hubploy --verbose deploy $hub hub staging + done diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index fd2b36d2a..f6a83f0fd 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:829f9eaececd + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a9443779a3a9 cluster: provider: gcloud From ea0b0b24c5a9e7e025bd66e9e8f3c8c9cbb0b3ce Mon Sep 17 00:00:00 2001 From: shane Date: Mon, 19 Aug 2024 11:52:10 -0700 Subject: [PATCH 030/190] Update deploy-to-staging.yaml --- .github/workflows/deploy-to-staging.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index 34a616f9d..f5f373eca 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -78,7 +78,7 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | - for hub in $DEPLOY_HUBS[@]; do + for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" hubploy --verbose deploy $hub hub staging done From df686dbeb67f42f8710d03a533c838285387956a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 11:54:39 -0700 Subject: [PATCH 031/190] hopefully the final test --- deployments/logodev/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index f6a83f0fd..28ea5c8da 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a9443779a3a9 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:71e05556317f cluster: provider: gcloud From 2ff3b377c78d4554aee6fbc96b4ae4f5985e55f2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 12:14:09 -0700 Subject: [PATCH 032/190] set env var for sops to use for auth --- .github/workflows/deploy-to-staging.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index f5f373eca..a31c36d03 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -78,6 +78,7 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | + GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GKE_KEY }} for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" hubploy --verbose deploy $hub hub staging From 14ac6e24f0af139aae8a84b28aa2ef2cbf231288 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 13:00:50 -0700 Subject: [PATCH 033/190] update variable name to match secret in github repo --- .github/workflows/deploy-to-staging.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index a31c36d03..d3726026a 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -78,7 +78,7 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | - GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GKE_KEY }} + GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.SOPS_KEY }} for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" hubploy --verbose deploy $hub hub staging From f5959277709451bd0da0385dda2174b22f55c36a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 13:02:24 -0700 Subject: [PATCH 034/190] trigger a deployment to staging --- deployments/logodev/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 28ea5c8da..f6a83f0fd 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:71e05556317f + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a9443779a3a9 cluster: provider: gcloud From 8b1addc55fd3b0d7a7da25ae9946412be6dcd4a8 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 13:14:20 -0700 Subject: [PATCH 035/190] remove repo from paths etc --- .github/workflows/deploy-to-staging.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index d3726026a..b2ef46eab 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -62,10 +62,10 @@ jobs: - name: Install SOPS if: ${{ env.DEPLOY }} run: | - mkdir -p ${HOME}/repo/bin - curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops - chmod 755 ${HOME}/repo/bin/sops - echo "${HOME}/repo/bin" >> $GITHUB_PATH + mkdir -p ${HOME}/bin + curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH - name: Install Helm if: ${{ env.DEPLOY }} @@ -78,7 +78,8 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | - GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.SOPS_KEY }} + echo ${{ secrets.SOPS_KEY }} > ${HOME}/sops.key + export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/sops.key" for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" hubploy --verbose deploy $hub hub staging From 9b93f96bcb8625ace574c6e3d818d1b32cc1324d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 13:41:32 -0700 Subject: [PATCH 036/190] whee this is FUN --- .github/workflows/deploy-to-staging.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index b2ef46eab..6084b2a97 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -63,10 +63,18 @@ jobs: if: ${{ env.DEPLOY }} run: | mkdir -p ${HOME}/bin - curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/bin/sops + curl -sSL https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux -o ${HOME}/bin/sops chmod 755 ${HOME}/bin/sops echo "${HOME}/bin" >> $GITHUB_PATH + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + - name: Install Helm if: ${{ env.DEPLOY }} run: | @@ -78,8 +86,6 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | - echo ${{ secrets.SOPS_KEY }} > ${HOME}/sops.key - export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/sops.key" for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" hubploy --verbose deploy $hub hub staging From d55fb5874023d1dc0e2da54b7e5ad3d33bcd7b90 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 13:44:34 -0700 Subject: [PATCH 037/190] env updates for fall2024; change course number and mem reqs --- deployments/astro/config/common.yaml | 6 +- deployments/astro/image/environment.yml | 116 +++++++++++------------- 2 files changed, 57 insertions(+), 65 deletions(-) diff --git a/deployments/astro/config/common.yaml b/deployments/astro/config/common.yaml index eb8caee98..c97af25a8 100644 --- a/deployments/astro/config/common.yaml +++ b/deployments/astro/config/common.yaml @@ -92,9 +92,9 @@ jupyterhub: custom: group_profiles: # Astro Lab 128, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5596 - course::1532117: # Temporarily grant 8G RAM to all students - mem_limit: 8G - mem_guarantee: 8G + course::1537100: # Temporarily bump down to 4G RAM to all students + mem_limit: 4G + mem_guarantee: 4G admin: extraVolumeMounts: - name: home diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index ce81f1033..353b406a4 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -6,97 +6,89 @@ channels: dependencies: - python=3.11.* -- jupyter-server-proxy==4.2.0 +- jupyter-server-proxy>=4.3 # A linux desktop environment - websockify - -- syncthing==1.18.6 - -- numpy==1.26.* -- matplotlib==3.8.* +- numpy>=2.1.0 +- numexpr>=2.10.0 +- matplotlib==3.9.* - graphviz>=2.50.0 - psycopg2==2.9.* -- pytorch==2.3.0 -- torchvision==0.18.* -- pytorch-lightning==2.2.1 -- pyaudio==0.2.11 -- cython +- pytorch +- torchvision +- pytorch-lightning - cartopy - protobuf -- mkl -- mkl-service +- setuptools>=65.3.0 - mpich - mpi4py -- scikit-learn==1.4.* -- scikit-image==0.19.* -- scipy==1.11.3 +- mkl +- mkl-service +- netcdf4>=1.6 +- scikit-learn==1.5.* +- scikit-image==0.24.* +- scipy==1.14.* +- jax>=0.4.31 +- jaxlib>=0.4.31 +- cython==3.0.11 - pip - pip: - - jupyter-desktop-server==0.1.3 + - syncthing - -r infra-requirements.txt - - tensorflow + - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 - sympy - beautifulsoup4 - tqdm - - h5py - - netcdf4 - - numexpr - corner==2.2.* - - dask>=2021.12.0 - - dask-ml>=2021.11.30 - - distributed>=2021.12.0 - - jaxlib>=0.1.75 - - SQLAlchemy>=1.4.29 - - xarray>=0.20.2 - - nltk>=3.6.7 - - seaborn>=0.11.2 - - bqplot>=0.12.31 - - astroquery>=0.4.5 - - astropy==5.3.3 - - dustmaps>=1.0.9 - - pyvo>=1.2 - - joblib==1.2.0 - - aesara>=2.3.4 - - pymc3>=3.11.4 - - requests>=2.27.1 + - dask>=2024.8.1 + - dask-ml>=2024.4.4 + - distributed>=2024.8.1 + - SQLAlchemy>=2.0.32 + - xarray>=2024.7.0 + - nltk>=3.9.1 + - seaborn>=0.13.2 + - bqplot>=0.12.43 + - astroquery>=0.4.7 + - astropy>=6.1.2 + - dustmaps>=1.0.13 + - pyvo>=1.5.2 + - joblib==1.4.2 + - pymc>=5.16.2 + - requests>=2.32.3 - ipycanvas - - altair>=4.2.0 + - altair>=5.4.0 - vega3>=0.13.0 - - pytz>=2021.3 - - keras - - keras-preprocessing>=1.1.0 - - tensorflow-io-gcs-filesystem>=0.23.0 + - pytz>=2024.1 - lightkurve - - ipyparallel>=8.1.0 - - jupyter-tensorboard>=0.2.0 - - line_profiler>=3.4.0 - - snakeviz>=2.1.1 - - memory_profiler>=0.60.0 - - flask>=2.0.2 - - stsci.tools>=4.0.0 - - gensim>=4.1.2 + - ipyparallel + - line_profiler>=4.1.3 + - snakeviz>=2.2.0 + - memory_profiler>=0.61.0 + - flask>=3.0.3 + - stsci.tools>=4.1.0 + - gensim>=4.3.3 - tweet-preprocessor - pyLDAvis==3.4.1 - - umap-learn>=0.5.2 - - pydot>=1.4.0 - - TPOT>=0.11.7 - - tables>=3.7.0 - - aiohttp==3.8.5 - - jax + - umap-learn>=0.5.6 + - pydot>=3.0.1 + - TPOT>=0.12.2 + - tables>=3.10.1 + - aiohttp>=3.10.4 - watermark - autopep8 - - datashader==0.13.0 - - holoviews==1.14.7 - - hvplot==0.7.3 - vega_datasets - vega - pandas-bokeh - pythreejs - ipywidgets - ipyvolume + - urllib3 + - six + - ipython + - notebook - click - fire - pycodestyle @@ -104,8 +96,8 @@ dependencies: # - nb_black - pycodestyle_magic - twine - - otter-grader==3.1.4 + - otter-grader>=3.1.4 # for notebook exporting - - nbconvert[webpdf]==7.16.4 + - nbconvert>=6.4.0 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 From 832eab67af60aa846d4d049d0c48f64849159d55 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 13:54:57 -0700 Subject: [PATCH 038/190] fix broken url for sops --- .github/workflows/deploy-to-staging.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index 6084b2a97..f692b3f0f 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -63,7 +63,7 @@ jobs: if: ${{ env.DEPLOY }} run: | mkdir -p ${HOME}/bin - curl -sSL https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux -o ${HOME}/bin/sops + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops chmod 755 ${HOME}/bin/sops echo "${HOME}/bin" >> $GITHUB_PATH From 1fd2c66a30d7d29d1b7efcd3da048cd2937750ba Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 13:59:59 -0700 Subject: [PATCH 039/190] remove infrastructure install; add a few packages --- deployments/astro/image/environment.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 353b406a4..afcd7c9ea 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -18,7 +18,10 @@ dependencies: - torchvision - pytorch-lightning - cartopy +- cpuonly +- pandas>=2.2.2 - protobuf +- jupysql - setuptools>=65.3.0 - mpich - mpi4py @@ -35,7 +38,6 @@ dependencies: - pip: - syncthing - - -r infra-requirements.txt - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 @@ -98,6 +100,7 @@ dependencies: - twine - otter-grader>=3.1.4 # for notebook exporting - - nbconvert>=6.4.0 + - nbconvert[webpdf] - nb2pdf==0.6.2 - nbpdfexport==0.2.1 + - pytest-notebook==0.8.1 \ No newline at end of file From 789432bd51f0c48671751f7cbb3d3abbde3d0faf Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 14:02:46 -0700 Subject: [PATCH 040/190] add gke auth plugin --- .github/workflows/deploy-to-staging.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml index f692b3f0f..c72f4db5b 100644 --- a/.github/workflows/deploy-to-staging.yaml +++ b/.github/workflows/deploy-to-staging.yaml @@ -58,6 +58,8 @@ jobs: - name: Install Google Cloud SDK if: ${{ env.DEPLOY }} uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' - name: Install SOPS if: ${{ env.DEPLOY }} From 556dce1545ec3a3e5d8a59f2b8fcd5d71c21d793 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:07:32 -0700 Subject: [PATCH 041/190] syncthing->jupyter-syncthing-proxy --- deployments/astro/image/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index afcd7c9ea..2ea2f5304 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -37,7 +37,7 @@ dependencies: - pip - pip: - - syncthing + - jupyter-syncthing-proxy==1.0.3 - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 From 4c3701d76bce7503ca14c0a59c9841eae84122f2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 14:08:49 -0700 Subject: [PATCH 042/190] turn this off for now --- .../{deploy-to-staging.yaml => deploy-to-staging.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploy-to-staging.yaml => deploy-to-staging.yaml.disabled} (100%) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml.disabled similarity index 100% rename from .github/workflows/deploy-to-staging.yaml rename to .github/workflows/deploy-to-staging.yaml.disabled From 7c57a698ef8d1688e295789c1aed669923c56b68 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:20:55 -0700 Subject: [PATCH 043/190] remove some deps --- deployments/astro/image/environment.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 2ea2f5304..e5cb36d61 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -22,7 +22,6 @@ dependencies: - pandas>=2.2.2 - protobuf - jupysql -- setuptools>=65.3.0 - mpich - mpi4py - mkl @@ -37,8 +36,6 @@ dependencies: - pip - pip: - - jupyter-syncthing-proxy==1.0.3 - - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 - sympy From e6df54e8c4b9312f09b8c4ce24702aef0522d400 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:46:27 -0700 Subject: [PATCH 044/190] add tensorboard and jupyter goodies --- deployments/astro/image/environment.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index e5cb36d61..e6235550d 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -7,6 +7,15 @@ channels: dependencies: - python=3.11.* - jupyter-server-proxy>=4.3 +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- nbgitpuller==1.2.1 +- notebook==7.0.7 # A linux desktop environment - websockify - numpy>=2.1.0 @@ -30,6 +39,7 @@ dependencies: - scikit-learn==1.5.* - scikit-image==0.24.* - scipy==1.14.* +- tensorflow-cpu==2.12.1 - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 @@ -92,6 +102,7 @@ dependencies: - fire - pycodestyle - flake8 + - tdtax>=0.1.6 # - nb_black - pycodestyle_magic - twine @@ -100,4 +111,5 @@ dependencies: - nbconvert[webpdf] - nb2pdf==0.6.2 - nbpdfexport==0.2.1 - - pytest-notebook==0.8.1 \ No newline at end of file + - pytest-notebook==0.8.1 + - jupyter-tensorboard>=0.2.0 \ No newline at end of file From 6a957b04a267b7d2f1adbe1ada480927947a0360 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 14:46:27 -0700 Subject: [PATCH 045/190] clone staging deploy workflow and s/staging/prod/g --- .../workflows/deploy-to-prod.yaml.disabled | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 .github/workflows/deploy-to-prod.yaml.disabled diff --git a/.github/workflows/deploy-to-prod.yaml.disabled b/.github/workflows/deploy-to-prod.yaml.disabled new file mode 100644 index 000000000..0cd1a2ba0 --- /dev/null +++ b/.github/workflows/deploy-to-prod.yaml.disabled @@ -0,0 +1,94 @@ +name: Deploy images to prod hubs +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it +on: + workflow_dispatch: + push: + branches: + - prod + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Pull out any hubs that need deploying from the labels on the merge commit to prod + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + HUBS=() + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == hub-* ]]; then + label=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$label" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + echo "Hubs to deploy: $HUBS" + echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy hubs to prod + if: ${{ env.DEPLOY }} + run: | + for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do + echo "Deploying $hub to prod" + hubploy --verbose deploy $hub hub prod + done From e421e19577acf77648d2ef88b35258f468232b9b Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:53:08 -0700 Subject: [PATCH 046/190] proper version tensorflow --- deployments/astro/image/environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index e6235550d..ef3ea1fad 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -39,7 +39,8 @@ dependencies: - scikit-learn==1.5.* - scikit-image==0.24.* - scipy==1.14.* -- tensorflow-cpu==2.12.1 +- tensorflow-cpu==2.17.0 +- tensorflow==2.17.0 - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 From edb3d85b49d31430f017b548411cb4265eb14511 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 19 Aug 2024 15:05:16 -0700 Subject: [PATCH 047/190] workflows for support and node placeholder --- .../deploy-node-placeholder.yaml.disabled | 89 ++++++++++++++++++ .../workflows/deploy-support.yaml.disabled | 92 +++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 .github/workflows/deploy-node-placeholder.yaml.disabled create mode 100644 .github/workflows/deploy-support.yaml.disabled diff --git a/.github/workflows/deploy-node-placeholder.yaml.disabled b/.github/workflows/deploy-node-placeholder.yaml.disabled new file mode 100644 index 000000000..1099d5ca7 --- /dev/null +++ b/.github/workflows/deploy-node-placeholder.yaml.disabled @@ -0,0 +1,89 @@ +name: Deploy node placeholder helm chart +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it +on: + workflow_dispatch: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Check if the nbde placeholder helm chart needs to be deployed + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == node-placeholder-* ]]; then + echo "Deploying node placeholder charts!" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + + - name: Activate credentials for cluster + if: ${{ env.DEPLOY }} + run: | + sops -d -i deployments/datahub/secrets/gke-key.json + gcloud auth \ + activate-service-account \ + --key-file deployments/datahub/secrets/gke-key.json \ + gcloud container clusters \ + --region=us-central1 --project=ucb-datahub-2018 \ + get-credentials spring-2024 + + - name: Deploy node placeholder helm chart + if: ${{ env.DEPLOY }} + run: | + sops -d -i node-placeholder/secrets.yaml + helm upgrade \ + --install --wait \ + --namespace=node-placeholder node-placeholder node-placeholder \ + -f node-placeholder/secrets.yaml --debug \ No newline at end of file diff --git a/.github/workflows/deploy-support.yaml.disabled b/.github/workflows/deploy-support.yaml.disabled new file mode 100644 index 000000000..88442fc9d --- /dev/null +++ b/.github/workflows/deploy-support.yaml.disabled @@ -0,0 +1,92 @@ +name: Deploy support helm chart +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it +on: + workflow_dispatch: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Check if the support helm chart needs to be deployed + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == support-deployment ]]; then + echo "Deploying support charts!" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + + - name: Activate credentials for cluster + if: ${{ env.DEPLOY }} + run: | + sops -d -i deployments/datahub/secrets/gke-key.json + gcloud auth \ + activate-service-account \ + --key-file deployments/datahub/secrets/gke-key.json + gcloud container clusters \ + --region=us-central1 --project=ucb-datahub-2018 \ + get-credentials spring-2024 + + - name: Deploy support helm chart + if: ${{ env.DEPLOY }} + run: | + sops -d -i support/secrets.yaml + helm dep up support + helm upgrade \ + --install --wait \ + --namespace=support \ + support support/ \ + -f support/secrets.yaml \ + --set installCRDs=true --debug From e24d632402155c89f9fae71188aff21cda1f8644 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 15:11:09 -0700 Subject: [PATCH 048/190] add cudatools; remove some jupyter --- deployments/astro/image/environment.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index ef3ea1fad..3dc843cfc 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -7,15 +7,6 @@ channels: dependencies: - python=3.11.* - jupyter-server-proxy>=4.3 -- git==2.39.1 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 # A linux desktop environment - websockify - numpy>=2.1.0 @@ -27,7 +18,6 @@ dependencies: - torchvision - pytorch-lightning - cartopy -- cpuonly - pandas>=2.2.2 - protobuf - jupysql @@ -41,6 +31,7 @@ dependencies: - scipy==1.14.* - tensorflow-cpu==2.17.0 - tensorflow==2.17.0 +- cudatoolkit - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 From 4169d3ae793d69345615dd54381a0d5adb57480c Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 15:20:34 -0700 Subject: [PATCH 049/190] downgrade numpy to < 2.0 to make compat with tf --- deployments/astro/image/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 3dc843cfc..40287df28 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -9,8 +9,8 @@ dependencies: - jupyter-server-proxy>=4.3 # A linux desktop environment - websockify -- numpy>=2.1.0 -- numexpr>=2.10.0 +- numpy==1.26.4 +- numexpr>=2.8.7 - matplotlib==3.9.* - graphviz>=2.50.0 - psycopg2==2.9.* From d306279f21349ae0531147078fb13decd7401db9 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 20 Aug 2024 10:09:25 -0700 Subject: [PATCH 050/190] Don't run quarto workflow in forks. --- .github/workflows/quarto-docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/quarto-docs.yml b/.github/workflows/quarto-docs.yml index a4dfb9d76..2b74a334f 100644 --- a/.github/workflows/quarto-docs.yml +++ b/.github/workflows/quarto-docs.yml @@ -11,6 +11,9 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + # Don't run in forks, unless the forker sets a secret variable. I chose to + # use a secret rather than a var because secrets are not copied to forks. + if: github.event.repository.fork == false || secrets.ALLOW_FORK_DEPLOY == 'true' steps: - name: Check out repository uses: actions/checkout@v4 From 5b52a5f50ecd76cd0e8290fc81e0bde4492cd6d9 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 20 Aug 2024 10:16:16 -0700 Subject: [PATCH 051/190] adding sqlite to allow for notebook launch --- deployments/astro/image/environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 40287df28..351dd85c8 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -35,6 +35,7 @@ dependencies: - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 +- sqlite==3.46.0 - pip - pip: @@ -104,4 +105,4 @@ dependencies: - nb2pdf==0.6.2 - nbpdfexport==0.2.1 - pytest-notebook==0.8.1 - - jupyter-tensorboard>=0.2.0 \ No newline at end of file + - jupyter-tensorboard>=0.2.0 From 93dc07c7434885e12421bfa21862b48ca832865f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 20 Aug 2024 11:08:55 -0700 Subject: [PATCH 052/190] re-add infra reqs file --- deployments/astro/image/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 351dd85c8..af9024c5a 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -39,6 +39,7 @@ dependencies: - pip - pip: + - -r infra-requirements.txt - opencv-python>=4.5.5.62 - nose2>=0.10.0 - sympy From 95ed40f319f29485e18fd5c77694cb1c8744a3a5 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Tue, 20 Aug 2024 11:30:03 -0700 Subject: [PATCH 053/190] Removing jupyterlab-myst extension from datahub image --- deployments/datahub/images/default/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 9d5aa9aa2..9ec51db74 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -232,6 +232,6 @@ dependencies: # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157 - ibis-framework[pandas,duckdb]==9.2.0 - - jupyterlab_myst==2.4.2 + # - jupyterlab_myst==2.4.2 (Causing issues with rendering of the latex output # ATTEMPT TO PUT NEW PACKAGES IN THE CONDA LIST ABOVE FIRST, RATHER THAN PIP From 0d000a0a3d72303d64bc619b0bba88ad3a0544a5 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Tue, 20 Aug 2024 11:32:04 -0700 Subject: [PATCH 054/190] Fixing comment --- deployments/datahub/images/default/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 9ec51db74..3dcec245a 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -232,6 +232,6 @@ dependencies: # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157 - ibis-framework[pandas,duckdb]==9.2.0 - # - jupyterlab_myst==2.4.2 (Causing issues with rendering of the latex output + # - jupyterlab_myst==2.4.2 (Causing issues with rendering of the latex output for a CEE course) # ATTEMPT TO PUT NEW PACKAGES IN THE CONDA LIST ABOVE FIRST, RATHER THAN PIP From b64da2199af8ce16e35d1209eaf0e70fd6c41de6 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 20 Aug 2024 13:49:06 -0700 Subject: [PATCH 055/190] [DH-359] bumping seaborn --- deployments/datahub/config/common.yaml | 4 ++-- deployments/datahub/images/default/environment.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index 8aff4a141..91206c55f 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -168,8 +168,8 @@ jupyterhub: # DataHub Infrastructure staff # https://bcourses.berkeley.edu/courses/1524699/groups#tab-80607 course::1524699::group::all-admins: - mem_limit: 2048M - mem_guarantee: 2048M + mem_limit: 4096M + mem_guarantee: 4096M # Demog Data Event, April 1 - Sep 30, https://github.com/berkeley-dsep-infra/datahub/issues/5643 course::1534506::enrollment_type::teacher: diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 9d5aa9aa2..5dccbbb18 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -24,7 +24,7 @@ dependencies: - pandas==2.2.2 - statsmodels=0.13.5 - scikit-learn=1.4.* -- seaborn=0.11.* +- seaborn=0.13.2 # - bokeh=2.3.* - decorator=5.0.* - networkx=2.6.* From 7f0c3389eda5b36f6eaf63dc34c4045a3da20b37 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 20 Aug 2024 14:36:19 -0700 Subject: [PATCH 056/190] Add missing braces around action conditional. --- .github/workflows/quarto-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quarto-docs.yml b/.github/workflows/quarto-docs.yml index 2b74a334f..8f903f041 100644 --- a/.github/workflows/quarto-docs.yml +++ b/.github/workflows/quarto-docs.yml @@ -13,7 +13,7 @@ jobs: contents: write # Don't run in forks, unless the forker sets a secret variable. I chose to # use a secret rather than a var because secrets are not copied to forks. - if: github.event.repository.fork == false || secrets.ALLOW_FORK_DEPLOY == 'true' + if: ${{ github.event.repository.fork == false || secrets.ALLOW_FORK_DEPLOY == 'true' }} steps: - name: Check out repository uses: actions/checkout@v4 From 01fd7c4acdb42890c524c2223da0630b309f9561 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 21 Aug 2024 10:23:27 -0700 Subject: [PATCH 057/190] bumping placeholder values --- node-placeholder/values.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 679e2901c..099da8136 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -79,7 +79,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 biology: nodeSelector: hub.jupyter.org/pool-name: biology-pool @@ -87,7 +87,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 cee: nodeSelector: hub.jupyter.org/pool-name: cee-pool @@ -103,7 +103,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 207314055168 - replicas: 0 + replicas: 1 data101: nodeSelector: hub.jupyter.org/pool-name: data101-pool @@ -111,7 +111,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 data102: nodeSelector: hub.jupyter.org/pool-name: data102-pool @@ -119,7 +119,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 data8: nodeSelector: hub.jupyter.org/pool-name: data8-pool @@ -127,7 +127,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 datahub: nodeSelector: hub.jupyter.org/pool-name: datahub-pool @@ -143,7 +143,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 29247442944 - replicas: 1 + replicas: 0 dlab: nodeSelector: hub.jupyter.org/pool-name: dlab-pool @@ -151,7 +151,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 eecs: nodeSelector: hub.jupyter.org/pool-name: eecs-pool @@ -159,7 +159,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 ischool: nodeSelector: hub.jupyter.org/pool-name: ischool-pool @@ -175,7 +175,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 stat159: nodeSelector: hub.jupyter.org/pool-name: stat159-pool @@ -191,7 +191,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 r: nodeSelector: hub.jupyter.org/pool-name: r-pool @@ -199,7 +199,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 small-courses: nodeSelector: hub.jupyter.org/pool-name: small-courses-pool @@ -207,4 +207,4 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 From 3d19e3d909f05bb1d28a1c40a7ba3018f4e517d5 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 21 Aug 2024 10:33:01 -0700 Subject: [PATCH 058/190] bumping cee placeholder value --- node-placeholder/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 099da8136..2c3932b5b 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -95,7 +95,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 data100: nodeSelector: hub.jupyter.org/pool-name: data100-pool From 1bf9e6fdf28ac310972b91d1ff349aef793c3f2a Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" Date: Thu, 22 Aug 2024 17:59:01 +0000 Subject: [PATCH 059/190] update logodev image tag to c44b36bfede9: deployments/logodev/hubploy.yaml --- deployments/logodev/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index f6a83f0fd..d532f0a5b 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a9443779a3a9 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:c44b36bfede9 cluster: provider: gcloud From 39885f72f868daa863b8069816e21b99f122078c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 11:04:33 -0700 Subject: [PATCH 060/190] enable merge to staging for demo --- .../{deploy-to-staging.yaml.disabled => deploy-to-staging.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploy-to-staging.yaml.disabled => deploy-to-staging.yaml} (100%) diff --git a/.github/workflows/deploy-to-staging.yaml.disabled b/.github/workflows/deploy-to-staging.yaml similarity index 100% rename from .github/workflows/deploy-to-staging.yaml.disabled rename to .github/workflows/deploy-to-staging.yaml From fca742eb2d82304f71655f4daa9c663f3aba729c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 11:08:50 -0700 Subject: [PATCH 061/190] disabling workflow --- .../{deploy-to-staging.yaml => deploy-to-staging.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploy-to-staging.yaml => deploy-to-staging.yaml.disabled} (100%) diff --git a/.github/workflows/deploy-to-staging.yaml b/.github/workflows/deploy-to-staging.yaml.disabled similarity index 100% rename from .github/workflows/deploy-to-staging.yaml rename to .github/workflows/deploy-to-staging.yaml.disabled From 22f403021510ae2e9d6d008962d2cc5a3c70c151 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 11:36:33 -0700 Subject: [PATCH 062/190] macs2 for MCELLBI201B --- deployments/biology/image/environment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deployments/biology/image/environment.yml b/deployments/biology/image/environment.yml index 43f807b0c..cc4bee30c 100644 --- a/deployments/biology/image/environment.yml +++ b/deployments/biology/image/environment.yml @@ -59,6 +59,9 @@ dependencies: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets=5.9.* +# for MCELLBI201B (FA24) https://github.com/berkeley-dsep-infra/datahub/issues/5988 +- macs2==2.2.9.1 + # For https://github.com/berkeley-dsep-infra/datahub/issues/1846 # Conda does not have these - pip: From 4f103e4be5c0ea2b4b2db047410dbe24cabe8f99 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 16:07:19 -0700 Subject: [PATCH 063/190] hub/images deployment for all hubs --- .../deploy-jupyterhub-base-images.yaml | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/deploy-jupyterhub-base-images.yaml diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml new file mode 100644 index 000000000..ca00521a0 --- /dev/null +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -0,0 +1,89 @@ +name: Deploy base hub images to staging +on: + workflow_dispatch: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + echo "Deploying base hub images to all deployments" + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin,kubectl' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy base hub images to staging + if: ${{ env.DEPLOY }} + run: | + while read deployment; do + echo "Pretending to deploy base hub image to ${deployment} :P" + echo "hubploy deploy --debug ${deployment} hub staging" + done < <(kubectl get namespaces -A | grep -e staging | awk -F"-" '{print$1}') From c77d3c4fa16039a9b11156a252ed15b70f120168 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 16:19:02 -0700 Subject: [PATCH 064/190] kubectl auth --- .github/workflows/deploy-jupyterhub-base-images.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index ca00521a0..4e2c74e29 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -56,6 +56,13 @@ jobs: with: install_components: 'gke-gcloud-auth-plugin,kubectl' + - name: Get GKE credentials for the cluster + uses: 'google-github-actions/get-gke-credentials@v2' + with: + cluster_name: ${{ secrets.GKE_CLUSTER_NAME }} + location: ${{ secrets.GKE_CLUSTER_LOCATION }} + + - name: Install SOPS if: ${{ env.DEPLOY }} run: | From d1ae27bc5f0f8cd65875a5c27fdd3f57f2a7c54e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 16:22:59 -0700 Subject: [PATCH 065/190] add project id --- .github/workflows/deploy-jupyterhub-base-images.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index 4e2c74e29..b38d1f696 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -61,6 +61,7 @@ jobs: with: cluster_name: ${{ secrets.GKE_CLUSTER_NAME }} location: ${{ secrets.GKE_CLUSTER_LOCATION }} + project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Install SOPS From 5830cf7dbd06284a3d7e78621618042699e4a17a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 16:25:30 -0700 Subject: [PATCH 066/190] add proper logic for deploying or not --- .github/workflows/deploy-jupyterhub-base-images.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index b38d1f696..9b8acb35c 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -28,7 +28,11 @@ jobs: echo "DEPLOY=1" >> $GITHUB_ENV fi done - echo "Deploying base hub images to all deployments" + if [[ -n "${DEPLOY}" ]]; then + echo "Deploying base hub images to all deployments" + else + echo "No hub images to deploy" + fi - name: Setup python if: ${{ env.DEPLOY }} @@ -57,13 +61,13 @@ jobs: install_components: 'gke-gcloud-auth-plugin,kubectl' - name: Get GKE credentials for the cluster + if: ${{ env.DEPLOY }} uses: 'google-github-actions/get-gke-credentials@v2' with: cluster_name: ${{ secrets.GKE_CLUSTER_NAME }} location: ${{ secrets.GKE_CLUSTER_LOCATION }} project_id: ${{ secrets.GCP_PROJECT_ID }} - - name: Install SOPS if: ${{ env.DEPLOY }} run: | From b3c414eeec544f0536f770ef9306bbccd7a30a93 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 16:33:43 -0700 Subject: [PATCH 067/190] remove project_id --- .github/workflows/deploy-jupyterhub-base-images.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index 9b8acb35c..c7650dae7 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -66,7 +66,6 @@ jobs: with: cluster_name: ${{ secrets.GKE_CLUSTER_NAME }} location: ${{ secrets.GKE_CLUSTER_LOCATION }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Install SOPS if: ${{ env.DEPLOY }} From d8f3d09bfa0fac9273532ca8e42a4d70f2cf0c18 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 22 Aug 2024 16:38:04 -0700 Subject: [PATCH 068/190] temp disable this broken workflow --- ...se-images.yaml => deploy-jupyterhub-base-images.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploy-jupyterhub-base-images.yaml => deploy-jupyterhub-base-images.yaml.disabled} (100%) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml.disabled similarity index 100% rename from .github/workflows/deploy-jupyterhub-base-images.yaml rename to .github/workflows/deploy-jupyterhub-base-images.yaml.disabled From 15dd86b00a62fe492f48658f5f839be2fa738a83 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 09:48:53 -0700 Subject: [PATCH 069/190] use bash instead of kubectl to get deployments from said directory --- ...abled => deploy-jupyterhub-base-images.yaml} | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) rename .github/workflows/{deploy-jupyterhub-base-images.yaml.disabled => deploy-jupyterhub-base-images.yaml} (87%) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml.disabled b/.github/workflows/deploy-jupyterhub-base-images.yaml similarity index 87% rename from .github/workflows/deploy-jupyterhub-base-images.yaml.disabled rename to .github/workflows/deploy-jupyterhub-base-images.yaml index c7650dae7..d40283e11 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml.disabled +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -58,14 +58,7 @@ jobs: if: ${{ env.DEPLOY }} uses: google-github-actions/setup-gcloud@v2 with: - install_components: 'gke-gcloud-auth-plugin,kubectl' - - - name: Get GKE credentials for the cluster - if: ${{ env.DEPLOY }} - uses: 'google-github-actions/get-gke-credentials@v2' - with: - cluster_name: ${{ secrets.GKE_CLUSTER_NAME }} - location: ${{ secrets.GKE_CLUSTER_LOCATION }} + install_components: 'gke-gcloud-auth-plugin' - name: Install SOPS if: ${{ env.DEPLOY }} @@ -94,7 +87,13 @@ jobs: - name: Deploy base hub images to staging if: ${{ env.DEPLOY }} run: | + ignored_directories=("template") while read deployment; do + for ignored in "${ignored_directories[@]}"; do + if [[ "${deployment}" == "${ignored}" ]]; then + continue 2 # skip to the next "read deployment" + fi + done echo "Pretending to deploy base hub image to ${deployment} :P" echo "hubploy deploy --debug ${deployment} hub staging" - done < <(kubectl get namespaces -A | grep -e staging | awk -F"-" '{print$1}') + done < <(find deployments/ -maxdepth 1 -type d -printf '%f\n' | grep -v template) From 848c3a7e14cb7558c93d19387f21759eb30afa2a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 09:50:31 -0700 Subject: [PATCH 070/190] make comment more explainy --- .github/workflows/deploy-jupyterhub-base-images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index d40283e11..97bc73f97 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -91,7 +91,7 @@ jobs: while read deployment; do for ignored in "${ignored_directories[@]}"; do if [[ "${deployment}" == "${ignored}" ]]; then - continue 2 # skip to the next "read deployment" + continue 2 # skip to the next "while read deployment" fi done echo "Pretending to deploy base hub image to ${deployment} :P" From 5bf5d780860d9d67d59424d988d7c983bfbd0fbf Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 10:08:35 -0700 Subject: [PATCH 071/190] clean up and fix logic --- .github/workflows/deploy-jupyterhub-base-images.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index 97bc73f97..fc533d639 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -87,13 +87,13 @@ jobs: - name: Deploy base hub images to staging if: ${{ env.DEPLOY }} run: | - ignored_directories=("template") + ignored_directories=("template") # these are directories that we never want to deploy to while read deployment; do for ignored in "${ignored_directories[@]}"; do if [[ "${deployment}" == "${ignored}" ]]; then - continue 2 # skip to the next "while read deployment" + continue 2 # skip to the next iteration of "while read deployment" fi done echo "Pretending to deploy base hub image to ${deployment} :P" echo "hubploy deploy --debug ${deployment} hub staging" - done < <(find deployments/ -maxdepth 1 -type d -printf '%f\n' | grep -v template) + done < <(ls deployments/ | sed -e 's,/,,g') From f48595d03923386b43700505eba2d2b0b1cbf208 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 23 Aug 2024 11:13:17 -0700 Subject: [PATCH 072/190] Add shared-read-write directories for COMPSS 214A coure --- deployments/datahub/config/common.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index 0cc9c5680..8ee1369fc 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -193,6 +193,24 @@ jupyterhub: subPath: _shared/course/mba-247 readOnly: true + # COMPSS 214A, Fall 2024, https://jira-secure.berkeley.edu/browse/DH-311 + course::1538139::enrollment_type::teacher: + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/compss-214a-readwrite + subPath: _shared/course/compss-214a + course::1538139::enrollment_type::ta: + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/compss-214a-readwrite + subPath: _shared/course/compss-214a + course::1538139::enrollment_type::student: + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/compss-214a + subPath: _shared/course/compss-214a + readOnly: true + course::1534506: # Demog Data Event, April 1 - Sep 30, https://github.com/berkeley-dsep-infra/datahub/issues/5643 mem_limit: 8192M mem_guarantee: 8192M From 697c2acc56dbd5697259254ed8152276399730b3 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 12:26:01 -0700 Subject: [PATCH 073/190] add deploy hub images to prod --- .../deploy-jupyterhub-base-images.yaml | 97 ++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index fc533d639..6596e5ed6 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -4,9 +4,104 @@ on: push: branches: - staging + - prod jobs: - deploy: + deploy-hub-images-staging: + if: github.event_name == 'push' && github.ref == 'refs/heads/staging' + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + if [[ -n "${DEPLOY}" ]]; then + echo "Deploying base hub images to all deployments" + else + echo "No hub images to deploy" + fi + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy base hub images to staging + if: ${{ env.DEPLOY }} + run: | + ignored_directories=("template") # these are directories that we never want to deploy to + while read deployment; do + for ignored in "${ignored_directories[@]}"; do + if [[ "${deployment}" == "${ignored}" ]]; then + continue 2 # skip to the next iteration of "while read deployment" + fi + done + echo "Pretending to deploy base hub image to ${deployment} :P" + echo "hubploy deploy --debug ${deployment} hub staging" + done < <(ls deployments/ | sed -e 's,/,,g') + + deploy-hub-images-prod: + if: github.event_name == 'push' && github.ref == 'refs/heads/prod' runs-on: ubuntu-latest steps: - name: Get PR labels From f48ef8ad64bd684eac0d1d4f21a73541f0c93f2a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 12:28:21 -0700 Subject: [PATCH 074/190] s/staging/prod --- .github/workflows/deploy-jupyterhub-base-images.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index 6596e5ed6..ff84d0845 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -115,7 +115,7 @@ jobs: with: fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + - name: Pull out any hubs that need deploying from the labels on the merge commit to prod run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do @@ -179,7 +179,7 @@ jobs: helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ helm repo update - - name: Deploy base hub images to staging + - name: Deploy base hub images to prod if: ${{ env.DEPLOY }} run: | ignored_directories=("template") # these are directories that we never want to deploy to @@ -190,5 +190,5 @@ jobs: fi done echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --debug ${deployment} hub staging" + echo "hubploy deploy --debug ${deployment} hub prod" done < <(ls deployments/ | sed -e 's,/,,g') From 826cc80acd479207c7e17361925ee7af0450a16f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:24:49 -0700 Subject: [PATCH 075/190] testing out node placeholder deploy --- .circleci/config.yml | 58 ------------------- ....disabled => deploy-node-placeholder.yaml} | 0 node-placeholder/values.yaml | 2 +- 3 files changed, 1 insertion(+), 59 deletions(-) rename .github/workflows/{deploy-node-placeholder.yaml.disabled => deploy-node-placeholder.yaml} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a4582583..bc124d61f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -337,64 +337,6 @@ jobs: hubploy deploy --timeout 30m workshop hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - deploy-node-placeholder: - docker: - - image: buildpack-deps:bionic-scm - working_directory: ~/repo - steps: - - checkout - - - run: - name: install google-cloud-sdk - command: | - export GCLOUD_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-457.0.0-linux-x86_64.tar.gz - curl -sSL $GCLOUD_URL | tar -xzf - -C ${HOME} - # Be careful with quote ordering here. ${PATH} must not be expanded - # Don't use ~ here - bash can interpret PATHs containing ~, but most other things can't. - # Always use full PATHs in PATH! - echo 'export PATH="${HOME}/google-cloud-sdk/bin:${PATH}"' >> ${BASH_ENV} - - - run: - name: Setup helm3 - command: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | \ - tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - - - run: - name: Install sops - command: | - echo $SOPS_ACCOUNT_KEY > ${HOME}/repo/sops.key - echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/repo/sops.key' >> ${BASH_ENV} - mkdir -p ${HOME}/repo/bin - curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops - chmod 755 ${HOME}/repo/bin/sops - echo 'export PATH="${HOME}/repo/bin:${PATH}"' >> ${BASH_ENV} - - - run: - name: Activate credentials for datahub cluster (spring-2024) - command: | - gcloud components install gke-gcloud-auth-plugin - sops -d -i deployments/datahub/secrets/gke-key.json - gcloud auth \ - activate-service-account \ - --key-file deployments/datahub/secrets/gke-key.json - - gcloud container clusters \ - --region=us-central1 --project=ucb-datahub-2018 \ - get-credentials spring-2024 - - - run: - name: Deploy node placeholder chart - command: | - sops -d -i node-placeholder/secrets.yaml - helm upgrade \ - --install --wait \ - --namespace=node-placeholder node-placeholder node-placeholder \ - -f node-placeholder/secrets.yaml --debug - deploy-support: docker: - image: buildpack-deps:bionic-scm diff --git a/.github/workflows/deploy-node-placeholder.yaml.disabled b/.github/workflows/deploy-node-placeholder.yaml similarity index 100% rename from .github/workflows/deploy-node-placeholder.yaml.disabled rename to .github/workflows/deploy-node-placeholder.yaml diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 2c3932b5b..8d485f037 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -143,7 +143,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 29247442944 - replicas: 0 + replicas: 1 dlab: nodeSelector: hub.jupyter.org/pool-name: dlab-pool From f87bc08c4b60adf5de6ad163dbc3a80d790de762 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:31:38 -0700 Subject: [PATCH 076/190] fixing workflow to test again --- .github/workflows/deploy-node-placeholder.yaml | 2 +- node-placeholder/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-node-placeholder.yaml b/.github/workflows/deploy-node-placeholder.yaml index 1099d5ca7..31ab884e7 100644 --- a/.github/workflows/deploy-node-placeholder.yaml +++ b/.github/workflows/deploy-node-placeholder.yaml @@ -74,7 +74,7 @@ jobs: sops -d -i deployments/datahub/secrets/gke-key.json gcloud auth \ activate-service-account \ - --key-file deployments/datahub/secrets/gke-key.json \ + --key-file deployments/datahub/secrets/gke-key.json gcloud container clusters \ --region=us-central1 --project=ucb-datahub-2018 \ get-credentials spring-2024 diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 8d485f037..2c3932b5b 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -143,7 +143,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 29247442944 - replicas: 1 + replicas: 0 dlab: nodeSelector: hub.jupyter.org/pool-name: dlab-pool From 6b23ddfe5f45d2eed2dbf006608903a10ce8d44b Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:33:47 -0700 Subject: [PATCH 077/190] remove node-placeholder workflow entirely this time --- .circleci/config.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc124d61f..eebc87736 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -799,14 +799,6 @@ workflows: branches: only: - prod - deploy-node-placeholder: - jobs: - - deploy-node-placeholder: - filters: - branches: - # We don't have staging / prod for our node placeholder pods - # So we deploy only when deploying staging - only: staging deploy-support: jobs: From f6fad643c836a59bc46f3aa577c9106774d0849e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:38:27 -0700 Subject: [PATCH 078/190] removing debug from helm commands --- .github/workflows/deploy-node-placeholder.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-node-placeholder.yaml b/.github/workflows/deploy-node-placeholder.yaml index 31ab884e7..a227b2c7e 100644 --- a/.github/workflows/deploy-node-placeholder.yaml +++ b/.github/workflows/deploy-node-placeholder.yaml @@ -86,4 +86,4 @@ jobs: helm upgrade \ --install --wait \ --namespace=node-placeholder node-placeholder node-placeholder \ - -f node-placeholder/secrets.yaml --debug \ No newline at end of file + -f node-placeholder/secrets.yaml From 48204dbc6d90e7ba9ffdfccc2222f1c6e9ce57be Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:39:01 -0700 Subject: [PATCH 079/190] removing additional debug --- .github/workflows/deploy-support.yaml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-support.yaml.disabled b/.github/workflows/deploy-support.yaml.disabled index 88442fc9d..3b4ab60e1 100644 --- a/.github/workflows/deploy-support.yaml.disabled +++ b/.github/workflows/deploy-support.yaml.disabled @@ -89,4 +89,4 @@ jobs: --namespace=support \ support support/ \ -f support/secrets.yaml \ - --set installCRDs=true --debug + --set installCRDs=true From 630e8dac71aeb579afca835e602190b75fc0cc34 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:47:09 -0700 Subject: [PATCH 080/190] use github actions to deploy support chart --- .circleci/config.yml | 69 ------------------- ...port.yaml.disabled => deploy-support.yaml} | 0 2 files changed, 69 deletions(-) rename .github/workflows/{deploy-support.yaml.disabled => deploy-support.yaml} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index eebc87736..6a0754c58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -337,66 +337,6 @@ jobs: hubploy deploy --timeout 30m workshop hub ${CIRCLE_BRANCH} no_output_timeout: 30m - deploy-support: - docker: - - image: buildpack-deps:bionic-scm - working_directory: ~/repo - steps: - - checkout - - - run: - name: install google-cloud-sdk - command: | - export GCLOUD_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-457.0.0-linux-x86_64.tar.gz - curl -sSL $GCLOUD_URL | tar -xzf - -C ${HOME} - # Be careful with quote ordering here. ${PATH} must not be expanded - # Don't use ~ here - bash can interpret PATHs containing ~, but most other things can't. - # Always use full PATHs in PATH! - echo 'export PATH="${HOME}/google-cloud-sdk/bin:${PATH}"' >> ${BASH_ENV} - - - run: - name: Install sops - command: | - echo $SOPS_ACCOUNT_KEY > ${HOME}/repo/sops.key - echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/repo/sops.key' >> ${BASH_ENV} - mkdir -p ${HOME}/repo/bin - curl -sSL https://github.com/mozilla/sops/releases/download/v3.7.0/sops-v3.7.0.linux -o ${HOME}/repo/bin/sops - chmod 755 ${HOME}/repo/bin/sops - echo 'export PATH="${HOME}/repo/bin:${PATH}"' >> ${BASH_ENV} - - - run: - name: Setup helm3 - command: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | \ - tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - - - run: - name: Activate credentials for datahub cluster (spring-2024) - command: | - gcloud components install gke-gcloud-auth-plugin - sops -d -i deployments/datahub/secrets/gke-key.json - gcloud auth \ - activate-service-account \ - --key-file deployments/datahub/secrets/gke-key.json - - gcloud container clusters \ - --region=us-central1 --project=ucb-datahub-2018 \ - get-credentials spring-2024 - - - run: - name: Deploy support helm chart - command: | - sops -d -i support/secrets.yaml - helm dep up support - helm upgrade \ - --install --wait \ - --namespace=support \ - support support/ \ - -f support/secrets.yaml \ - --set installCRDs=true --debug - workflows: version: 2 test-build-images: @@ -799,12 +739,3 @@ workflows: branches: only: - prod - - deploy-support: - jobs: - - deploy-support: - filters: - branches: - # We don't have staging / prod for our support cluster - # So we deploy only when deploying staging - only: staging diff --git a/.github/workflows/deploy-support.yaml.disabled b/.github/workflows/deploy-support.yaml similarity index 100% rename from .github/workflows/deploy-support.yaml.disabled rename to .github/workflows/deploy-support.yaml From b4940208edfc47cd12f1ac9a85d552396affffa6 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 23 Aug 2024 14:53:09 -0700 Subject: [PATCH 081/190] removing commented out line --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4eacc64bd..399167dc6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,5 @@ jupyter-repo2docker==2024.3.0 myst-parser chardet requests==2.31.0 -# requests==2.32.3 urllib3<2.0.0 yamllint==1.35.1 From 18380ff2a22eecf5f24be257d8076db267765407 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Mon, 26 Aug 2024 09:58:32 -0700 Subject: [PATCH 082/190] Add stanza to provide elevated access for Prob 140 and Data 8 course staff --- deployments/data8/config/common.yaml | 14 +++++++------- deployments/prob140/config/common.yaml | 25 ++++++++++++------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/deployments/data8/config/common.yaml b/deployments/data8/config/common.yaml index 471ffda36..01175151d 100644 --- a/deployments/data8/config/common.yaml +++ b/deployments/data8/config/common.yaml @@ -61,19 +61,19 @@ jupyterhub: # - course::N::enrollment_type::teacher # - course::N::enrollment_type::ta - # Data 8, Summer 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5834 - course-staff-1535365: + # Data 8, Fall 2024, https://github.com/berkeley-dsep-infra/datahub/issues/6014 + course-staff-1538208: # description: Enable course staff to view and access servers. # # this role provides permissions to... scopes: - admin-ui - - list:users!group=course::1535365 - - admin:servers!group=course::1535365 - - access:servers!group=course::1535365 + - list:users!group=course::1538208 + - admin:servers!group=course::1538208 + - access:servers!group=course::1538208 # # this role will be assigned to... groups: - - course::1535365::enrollment_type::teacher - - course::1535365::enrollment_type::ta + - course::1538208::enrollment_type::teacher + - course::1538208::enrollment_type::ta singleuser: extraFiles: diff --git a/deployments/prob140/config/common.yaml b/deployments/prob140/config/common.yaml index 1727c8711..a1e9e8563 100644 --- a/deployments/prob140/config/common.yaml +++ b/deployments/prob140/config/common.yaml @@ -49,20 +49,19 @@ jupyterhub: # - course::N::enrollment_type::teacher # - course::N::enrollment_type::ta - ## Data C140, Spring 2024, #https://github.com/berkeley-dsep-infra/datahub/issues/5466 - #course-staff-1533557: + # Data C140, Fall 2024, #https://github.com/berkeley-dsep-infra/datahub/issues/6011 + course-staff-1537389: # description: Enable course staff to view and access servers. - # this role provides permissions to... - # scopes: - # - admin-ui - # - list:users!group=course::1533557 - # - admin:servers!group=course::1533557 - # - access:servers!group=course::1533557 - # this role will be assigned to... - # groups: - # - course::1533557::enrollment_type::teacher - # - course::1533557::enrollment_type::ta - + # # this role provides permissions to... + scopes: + - admin-ui + - list:users!group=course::1537389 + - admin:servers!group=course::1537389 + - access:servers!group=course::1537389 + # # this role will be assigned to... + groups: + - course::1537389::enrollment_type::teacher + - course::1537389::enrollment_type::ta nodeSelector: hub.jupyter.org/pool-name: core-pool-2024-05-08 From bc00743eb67c1b0dfb94297ad6f9c9bdf5d35131 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 11:19:49 -0700 Subject: [PATCH 083/190] combine staging/prod deploy to one workflow, skip if we need to deploy all hubs --- .github/workflows/deploy-hubs.yaml | 202 ++++++++++++++++++ .../deploy-jupyterhub-base-images.yaml | 41 ++-- .../workflows/deploy-to-prod.yaml.disabled | 94 -------- .../workflows/deploy-to-staging.yaml.disabled | 94 -------- 4 files changed, 225 insertions(+), 206 deletions(-) create mode 100644 .github/workflows/deploy-hubs.yaml delete mode 100644 .github/workflows/deploy-to-prod.yaml.disabled delete mode 100644 .github/workflows/deploy-to-staging.yaml.disabled diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml new file mode 100644 index 000000000..77d1fe468 --- /dev/null +++ b/.github/workflows/deploy-hubs.yaml @@ -0,0 +1,202 @@ +# this workflow re-deploys SPECIFIC hubs to staging or prod if the single-user +# server image or config has changed based on the PR labels "hub: ". +# +# however, this workflow will be not run if the PR labels of "hub-images" or +# "jupyterhub-deployment" are present, as these labels will trigger the +# "deploy-jupyterhub-base-images.yaml" workflow which re-deploys every hub. +# +name: Deploy staging and prod hubs +on: + workflow_dispatch: + push: + branches: + - staging + - prod + +jobs: + deploy-hubs-to-staging: + if: github.event_name == 'push' && github.ref == 'refs/heads/staging' + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + HUBS=() + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == hub-* ]]; then + label=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$label" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this + # means the base hub image has changed, and all hubs (staging or prod) need to + # be redeployed. This workflow will not run in that case. + if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then + echo "Base hub image has changed, not deploying individual hubs to staging" + echo "DEPLOY=0" >> $GITHUB_ENV + fi + echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV + + - name: Check out the image repo + if: ${{ env.DEPLOY }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy hubs to staging + if: ${{ env.DEPLOY }} + run: | + for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do + echo "Deploying $hub to staging" + echo "hubploy --verbose deploy $hub hub staging" + done + + deploy-hubs-to-prod: + if: github.event_name == 'push' && github.ref == 'refs/heads/prod' + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Pull out any hubs that need deploying from the labels on the merge commit to prod + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + HUBS=() + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == hub-* ]]; then + label=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$label" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this + # means the base hub image has changed, and all hubs (staging or prod) need to + # be redeployed. This workflow will not run in that case. + if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then + echo "Base hub image has changed, not deploying individual hubs to prod" + echo "DEPLOY=0" >> $GITHUB_ENV + fi + echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy hubs to prod + if: ${{ env.DEPLOY }} + run: | + for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do + echo "Deploying $hub to prod" + echo "hubploy --verbose deploy $hub hub prod" + done diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index ff84d0845..5323dc8ff 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -1,4 +1,8 @@ -name: Deploy base hub images to staging +# if the PR labels "hub-images" or "jupyterhub-deployment" are present, this +# means the base hub image has changed, and all hubs (staging or prod) need to +# be redeployed. +# +name: Deploy base hub images to all hubs in staging and prod on: workflow_dispatch: push: @@ -17,11 +21,6 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - name: Pull out any hubs that need deploying from the labels on the merge commit to staging run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" @@ -30,12 +29,18 @@ jobs: echo "DEPLOY=1" >> $GITHUB_ENV fi done - if [[ -n "${DEPLOY}" ]]; then - echo "Deploying base hub images to all deployments" + if [[ -n "${{ env.DEPLOY }}" ]]; then + echo "Deploying base hub images to all deployments on staging" else - echo "No hub images to deploy" + echo "No hub images to deploy to staging" fi + - name: Check out the image repo + if: ${{ env.DEPLOY }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + - name: Setup python if: ${{ env.DEPLOY }} uses: actions/setup-python@v5 @@ -89,7 +94,7 @@ jobs: - name: Deploy base hub images to staging if: ${{ env.DEPLOY }} run: | - ignored_directories=("template") # these are directories that we never want to deploy to + ignored_directories=("template") # these are directories that we never want to deploy while read deployment; do for ignored in "${ignored_directories[@]}"; do if [[ "${deployment}" == "${ignored}" ]]; then @@ -110,11 +115,6 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - name: Pull out any hubs that need deploying from the labels on the merge commit to prod run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" @@ -123,12 +123,17 @@ jobs: echo "DEPLOY=1" >> $GITHUB_ENV fi done - if [[ -n "${DEPLOY}" ]]; then - echo "Deploying base hub images to all deployments" + if [[ -n "${{ env.DEPLOY }}" ]]; then + echo "Deploying base hub images to all deployments on prod" else echo "No hub images to deploy" fi + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + - name: Setup python if: ${{ env.DEPLOY }} uses: actions/setup-python@v5 @@ -182,7 +187,7 @@ jobs: - name: Deploy base hub images to prod if: ${{ env.DEPLOY }} run: | - ignored_directories=("template") # these are directories that we never want to deploy to + ignored_directories=("template") # these are directories that we never want to deploy while read deployment; do for ignored in "${ignored_directories[@]}"; do if [[ "${deployment}" == "${ignored}" ]]; then diff --git a/.github/workflows/deploy-to-prod.yaml.disabled b/.github/workflows/deploy-to-prod.yaml.disabled deleted file mode 100644 index 0cd1a2ba0..000000000 --- a/.github/workflows/deploy-to-prod.yaml.disabled +++ /dev/null @@ -1,94 +0,0 @@ -name: Deploy images to prod hubs -# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it -on: - workflow_dispatch: - push: - branches: - - prod - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Get PR labels - id: pr-labels - uses: irby/get-labels-on-push@v1.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Pull out any hubs that need deploying from the labels on the merge commit to prod - run: | - echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - HUBS=() - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == hub-* ]]; then - label=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$label" - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done - echo "Hubs to deploy: $HUBS" - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV - - - name: Setup python - if: ${{ env.DEPLOY }} - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - if: ${{ env.DEPLOY }} - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - - name: Auth to gcloud - if: ${{ env.DEPLOY }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GKE_KEY }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Install Google Cloud SDK - if: ${{ env.DEPLOY }} - uses: google-github-actions/setup-gcloud@v2 - with: - install_components: 'gke-gcloud-auth-plugin' - - - name: Install SOPS - if: ${{ env.DEPLOY }} - run: | - mkdir -p ${HOME}/bin - curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops - chmod 755 ${HOME}/bin/sops - echo "${HOME}/bin" >> $GITHUB_PATH - - - name: Store SOPS secret in a file - if: ${{ env.DEPLOY }} - run: | - cat << EOF > ${HOME}/sops.key - ${{ secrets.SOPS_KEY }} - EOF - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV - - - name: Install Helm - if: ${{ env.DEPLOY }} - run: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ - helm repo update - - - name: Deploy hubs to prod - if: ${{ env.DEPLOY }} - run: | - for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do - echo "Deploying $hub to prod" - hubploy --verbose deploy $hub hub prod - done diff --git a/.github/workflows/deploy-to-staging.yaml.disabled b/.github/workflows/deploy-to-staging.yaml.disabled deleted file mode 100644 index c72f4db5b..000000000 --- a/.github/workflows/deploy-to-staging.yaml.disabled +++ /dev/null @@ -1,94 +0,0 @@ -name: Deploy images to staging hubs -# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it -on: - workflow_dispatch: - push: - branches: - - staging - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Get PR labels - id: pr-labels - uses: irby/get-labels-on-push@v1.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Pull out any hubs that need deploying from the labels on the merge commit to staging - run: | - echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - HUBS=() - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == hub-* ]]; then - label=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$label" - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done - echo "Hubs to deploy: $HUBS" - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV - - - name: Setup python - if: ${{ env.DEPLOY }} - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - if: ${{ env.DEPLOY }} - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - - name: Auth to gcloud - if: ${{ env.DEPLOY }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GKE_KEY }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Install Google Cloud SDK - if: ${{ env.DEPLOY }} - uses: google-github-actions/setup-gcloud@v2 - with: - install_components: 'gke-gcloud-auth-plugin' - - - name: Install SOPS - if: ${{ env.DEPLOY }} - run: | - mkdir -p ${HOME}/bin - curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops - chmod 755 ${HOME}/bin/sops - echo "${HOME}/bin" >> $GITHUB_PATH - - - name: Store SOPS secret in a file - if: ${{ env.DEPLOY }} - run: | - cat << EOF > ${HOME}/sops.key - ${{ secrets.SOPS_KEY }} - EOF - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV - - - name: Install Helm - if: ${{ env.DEPLOY }} - run: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ - helm repo update - - - name: Deploy hubs to staging - if: ${{ env.DEPLOY }} - run: | - for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do - echo "Deploying $hub to staging" - hubploy --verbose deploy $hub hub staging - done From c563e20457bb96f7a04520751ba7ad72431bc780 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 11:25:46 -0700 Subject: [PATCH 084/190] s/triage/apply-labels --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 07b21837d..d886cd0dd 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,7 +3,7 @@ on: - pull_request_target jobs: - triage: + apply-labels: permissions: contents: read pull-requests: write From 69c4b09f50ec20578a48e30a9cd32c1d21364704 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 11:28:48 -0700 Subject: [PATCH 085/190] s/debug/verbose --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 77d1fe468..0613d792c 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -105,7 +105,7 @@ jobs: run: | for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" - echo "hubploy --verbose deploy $hub hub staging" + echo "hubploy --debug deploy $hub hub staging" done deploy-hubs-to-prod: @@ -198,5 +198,5 @@ jobs: run: | for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to prod" - echo "hubploy --verbose deploy $hub hub prod" + echo "hubploy --debug deploy $hub hub prod" done From ec636c3568cc7e09dbb3e1fc96b3115c4a438512 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 11:32:16 -0700 Subject: [PATCH 086/190] more wording --- .github/workflows/deploy-jupyterhub-base-images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml index 5323dc8ff..18736907e 100644 --- a/.github/workflows/deploy-jupyterhub-base-images.yaml +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -124,7 +124,7 @@ jobs: fi done if [[ -n "${{ env.DEPLOY }}" ]]; then - echo "Deploying base hub images to all deployments on prod" + echo "Deploying base hub images to all deployments to prod" else echo "No hub images to deploy" fi From ec0286ba906a96ece9ffdcee3c80713547edf296 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 15:16:19 -0700 Subject: [PATCH 087/190] more tweaking and better logic flow + file rename to something sane --- ...-base-images.yaml => deploy-all-hubs.yaml} | 0 .github/workflows/deploy-hubs.yaml | 40 ++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) rename .github/workflows/{deploy-jupyterhub-base-images.yaml => deploy-all-hubs.yaml} (100%) diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-all-hubs.yaml similarity index 100% rename from .github/workflows/deploy-jupyterhub-base-images.yaml rename to .github/workflows/deploy-all-hubs.yaml diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 0613d792c..39919229b 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -28,21 +28,23 @@ jobs: run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" HUBS=() - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == hub-* ]]; then - label=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$label" - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this # means the base hub image has changed, and all hubs (staging or prod) need to - # be redeployed. This workflow will not run in that case. + # be redeployed. The rest of this job will not run in that case. if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then echo "Base hub image has changed, not deploying individual hubs to staging" echo "DEPLOY=0" >> $GITHUB_ENV + else + # deploy any hubs that have been labeled for deployment + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == hub-* ]]; then + label=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$label" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV - name: Check out the image repo if: ${{ env.DEPLOY }} @@ -127,21 +129,23 @@ jobs: run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" HUBS=() - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == hub-* ]]; then - label=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$label" - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this # means the base hub image has changed, and all hubs (staging or prod) need to - # be redeployed. This workflow will not run in that case. + # be redeployed. The rest of this job will not run in that case. if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then echo "Base hub image has changed, not deploying individual hubs to prod" echo "DEPLOY=0" >> $GITHUB_ENV + else + # deploy any hubs that have been labeled for deployment + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == hub-* ]]; then + label=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$label" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV - name: Setup python if: ${{ env.DEPLOY }} From ee7dd887791a076158e9d40ec8ae76fc3f04b7e3 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 15:23:48 -0700 Subject: [PATCH 088/190] for seriously --- .github/workflows/deploy-hubs.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 39919229b..bea73df68 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -33,7 +33,6 @@ jobs: # be redeployed. The rest of this job will not run in that case. if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then echo "Base hub image has changed, not deploying individual hubs to staging" - echo "DEPLOY=0" >> $GITHUB_ENV else # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do @@ -134,7 +133,6 @@ jobs: # be redeployed. The rest of this job will not run in that case. if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then echo "Base hub image has changed, not deploying individual hubs to prod" - echo "DEPLOY=0" >> $GITHUB_ENV else # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do From b0f0cfa7f2ca543c60f32b5b67b651e6c0f23a42 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 15:44:09 -0700 Subject: [PATCH 089/190] removing some old debugging text --- .github/workflows/deploy-all-hubs.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml index 18736907e..fbc149187 100644 --- a/.github/workflows/deploy-all-hubs.yaml +++ b/.github/workflows/deploy-all-hubs.yaml @@ -29,11 +29,6 @@ jobs: echo "DEPLOY=1" >> $GITHUB_ENV fi done - if [[ -n "${{ env.DEPLOY }}" ]]; then - echo "Deploying base hub images to all deployments on staging" - else - echo "No hub images to deploy to staging" - fi - name: Check out the image repo if: ${{ env.DEPLOY }} @@ -123,11 +118,6 @@ jobs: echo "DEPLOY=1" >> $GITHUB_ENV fi done - if [[ -n "${{ env.DEPLOY }}" ]]; then - echo "Deploying base hub images to all deployments to prod" - else - echo "No hub images to deploy" - fi - name: Check out the image repo uses: actions/checkout@v4 From 3e41afb12617fc15dc19a319f043efdba8f2378a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 16:43:38 -0700 Subject: [PATCH 090/190] bash conditionals are confusing --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index bea73df68..a6fd9269e 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -31,7 +31,7 @@ jobs: # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this # means the base hub image has changed, and all hubs (staging or prod) need to # be redeployed. The rest of this job will not run in that case. - if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then + if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then echo "Base hub image has changed, not deploying individual hubs to staging" else # deploy any hubs that have been labeled for deployment @@ -131,7 +131,7 @@ jobs: # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this # means the base hub image has changed, and all hubs (staging or prod) need to # be redeployed. The rest of this job will not run in that case. - if [ -n $GITHUB_PR_LABEL_HUB_IMAGES ] || [ -n $GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]; then + if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then echo "Base hub image has changed, not deploying individual hubs to prod" else # deploy any hubs that have been labeled for deployment From 62ff498e053d4b024b3e8e0c6b73d26dcc0aaa95 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 26 Aug 2024 16:47:29 -0700 Subject: [PATCH 091/190] final little tweak for wording --- .github/workflows/deploy-all-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml index fbc149187..2018b1301 100644 --- a/.github/workflows/deploy-all-hubs.yaml +++ b/.github/workflows/deploy-all-hubs.yaml @@ -21,7 +21,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + - name: Check to see if the base jupyterhub image has changed, and deploy all hubs to prod if it has run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do @@ -110,7 +110,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Pull out any hubs that need deploying from the labels on the merge commit to prod + - name: Check to see if the base jupyterhub image has changed, and deploy all hubs to prod if it has run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do From 864140709b4179bcf2da62384992b8b26c1941ff Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Tue, 27 Aug 2024 12:29:11 -0700 Subject: [PATCH 092/190] Add elevated privileges for PH 142 and Data 101 course staff --- deployments/data101/config/common.yaml | 15 +++++++++++++++ deployments/publichealth/config/common.yaml | 14 +++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/deployments/data101/config/common.yaml b/deployments/data101/config/common.yaml index 8720901ee..7936ed10f 100644 --- a/deployments/data101/config/common.yaml +++ b/deployments/data101/config/common.yaml @@ -48,6 +48,21 @@ jupyterhub: # # this role will be assigned to... # groups: # - course::N::group::Admins + + # Data 101, Fall 2024, #https://jira-secure.berkeley.edu/browse/DH-364 + course-staff-1536858: + # description: Enable course staff to view and access servers. + # this role provides permissions to... + scopes: + - admin-ui + - list:users!group=course::1536858 + - admin:servers!group=course::1536858 + - access:servers!group=course::1536858 + # this role will be assigned to... + groups: + - course::1536858::enrollment_type::teacher + - course::1536858::enrollment_type::ta + extraConfig: data101-new-db: | from jupyterhub.utils import exponential_backoff diff --git a/deployments/publichealth/config/common.yaml b/deployments/publichealth/config/common.yaml index 44f92504f..e55e9723f 100644 --- a/deployments/publichealth/config/common.yaml +++ b/deployments/publichealth/config/common.yaml @@ -38,19 +38,19 @@ jupyterhub: groups: - course::1524699::group::all-admins - # PHW 142, Fall 2024, #5831 - course-staff-1535789: + # PHW 142, Fall 2024, #https://github.com/berkeley-dsep-infra/datahub/issues/6026 + course-staff-1537782: # description: Enable course staff to view and access servers. # this role provides permissions to... scopes: - admin-ui - - list:users!group=course::1535789 - - admin:servers!group=course::1535789 - - access:servers!group=course::1535789 + - list:users!group=course::1537782 + - admin:servers!group=course::1537782 + - access:servers!group=course::1537782 # this role will be assigned to... groups: - - course::1535789::enrollment_type::teacher - - course::1535789::enrollment_type::ta + - course::1537782::enrollment_type::teacher + - course::1537782::enrollment_type::ta extraConfig: # Use 1x- in `common.yaml` extraConfig values. From 368af17dd96362a8380c08be4245acc1181d4fbd Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 12:47:23 -0700 Subject: [PATCH 093/190] hopefully fixing bash arrays + parsing --- .github/workflows/deploy-hubs.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index a6fd9269e..3f7cc4510 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -37,8 +37,8 @@ jobs: # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then - label=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$label" + hub_name=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$hub_name " echo "DEPLOY=1" >> $GITHUB_ENV fi done @@ -119,11 +119,6 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - name: Pull out any hubs that need deploying from the labels on the merge commit to prod run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" @@ -137,14 +132,20 @@ jobs: # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then - label=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$label" + hub_name=$(echo $label | awk -F'-' '{print $2}') + HUBS+="$hub_name " echo "DEPLOY=1" >> $GITHUB_ENV fi done echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi + - name: Check out the image repo + if: ${{ env.DEPLOY }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + - name: Setup python if: ${{ env.DEPLOY }} uses: actions/setup-python@v5 From 594dfe5ed6d3cb0f39860e2e2606961cad68bfc1 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 14:42:06 -0700 Subject: [PATCH 094/190] changing debug output to verbose --- .github/workflows/deploy-all-hubs.yaml | 4 ++-- .github/workflows/deploy-hubs.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml index 2018b1301..cd032018c 100644 --- a/.github/workflows/deploy-all-hubs.yaml +++ b/.github/workflows/deploy-all-hubs.yaml @@ -97,7 +97,7 @@ jobs: fi done echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --debug ${deployment} hub staging" + echo "hubploy deploy --verbose ${deployment} hub staging" done < <(ls deployments/ | sed -e 's,/,,g') deploy-hub-images-prod: @@ -185,5 +185,5 @@ jobs: fi done echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --debug ${deployment} hub prod" + echo "hubploy deploy --verbose ${deployment} hub prod" done < <(ls deployments/ | sed -e 's,/,,g') diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 3f7cc4510..807240013 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -106,7 +106,7 @@ jobs: run: | for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to staging" - echo "hubploy --debug deploy $hub hub staging" + echo "hubploy --verbose deploy $hub hub staging" done deploy-hubs-to-prod: @@ -201,5 +201,5 @@ jobs: run: | for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do echo "Deploying $hub to prod" - echo "hubploy --debug deploy $hub hub prod" + echo "hubploy --verbose deploy $hub hub prod" done From 8dbc0c694a1defdcf4b069eb166315360e8591ef Mon Sep 17 00:00:00 2001 From: Balaji Alwar <balajialwar@berkeley.edu> Date: Tue, 27 Aug 2024 16:06:56 -0700 Subject: [PATCH 095/190] Add admins for workshop hub --- deployments/workshop/config/common.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deployments/workshop/config/common.yaml b/deployments/workshop/config/common.yaml index 1efc418de..02710f096 100644 --- a/deployments/workshop/config/common.yaml +++ b/deployments/workshop/config/common.yaml @@ -27,6 +27,14 @@ jupyterhub: JupyterHub: admin_access: false authenticator_class: dummy + Authenticator: + admin_users: + # infrastructure + - rylo + - sknapp + - felder + - balajialwar + - gmerritt proxy: chp: nodeSelector: From 6fd9cb1d8f34c64eae350cf610ee9a85d351e135 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 16:29:40 -0700 Subject: [PATCH 096/190] script and workflow to get deployments from deployments/ subdir --- .github/scripts/get-deployments.py | 29 ++++++++++++++++++++++++++ .github/workflows/deploy-all-hubs.yaml | 8 +------ 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100755 .github/scripts/get-deployments.py diff --git a/.github/scripts/get-deployments.py b/.github/scripts/get-deployments.py new file mode 100755 index 000000000..b96ad2767 --- /dev/null +++ b/.github/scripts/get-deployments.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import argparse +import os + +def main(args): + + for deployment in next(os.walk(args.deployments))[1]: + if deployment not in args.ignore: + print(deployment) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate a list of deployments from the deployments/ directory.') + parser.add_argument( + '--deployments', + '-d', + default='deployments', + help='The directory to search for deployments.' + ) + parser.add_argument( + '--ignore', + '-i', + nargs='*', + default=(), + help='Ignore one or more directories in the deployments/ subdir.' + ) + args = parser.parse_args() + + main(args) \ No newline at end of file diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml index cd032018c..1eacf59c8 100644 --- a/.github/workflows/deploy-all-hubs.yaml +++ b/.github/workflows/deploy-all-hubs.yaml @@ -177,13 +177,7 @@ jobs: - name: Deploy base hub images to prod if: ${{ env.DEPLOY }} run: | - ignored_directories=("template") # these are directories that we never want to deploy while read deployment; do - for ignored in "${ignored_directories[@]}"; do - if [[ "${deployment}" == "${ignored}" ]]; then - continue 2 # skip to the next iteration of "while read deployment" - fi - done echo "Pretending to deploy base hub image to ${deployment} :P" echo "hubploy deploy --verbose ${deployment} hub prod" - done < <(ls deployments/ | sed -e 's,/,,g') + done < <(.github/scripts/get-deployments.py --ignore template) From d931fa14ae522a05e5df4ceeb2a95441ebbed5bd Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 16:30:22 -0700 Subject: [PATCH 097/190] forgot about staging --- .github/workflows/deploy-all-hubs.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml index 1eacf59c8..475d19655 100644 --- a/.github/workflows/deploy-all-hubs.yaml +++ b/.github/workflows/deploy-all-hubs.yaml @@ -89,16 +89,10 @@ jobs: - name: Deploy base hub images to staging if: ${{ env.DEPLOY }} run: | - ignored_directories=("template") # these are directories that we never want to deploy while read deployment; do - for ignored in "${ignored_directories[@]}"; do - if [[ "${deployment}" == "${ignored}" ]]; then - continue 2 # skip to the next iteration of "while read deployment" - fi - done echo "Pretending to deploy base hub image to ${deployment} :P" echo "hubploy deploy --verbose ${deployment} hub staging" - done < <(ls deployments/ | sed -e 's,/,,g') + done < <(.github/scripts/get-deployments.py --ignore template) deploy-hub-images-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' From ac0f289a1d42697e475801b29d7c59ba07cf72be Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 16:38:06 -0700 Subject: [PATCH 098/190] newline at EOF --- .github/scripts/get-deployments.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/get-deployments.py b/.github/scripts/get-deployments.py index b96ad2767..81cc0161f 100755 --- a/.github/scripts/get-deployments.py +++ b/.github/scripts/get-deployments.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - import argparse import os @@ -26,4 +25,4 @@ def main(args): ) args = parser.parse_args() - main(args) \ No newline at end of file + main(args) From 84f6b64d2db59e3443df9eeee7bbdeae2335cbaa Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 16:39:05 -0700 Subject: [PATCH 099/190] docstring ftw --- .github/scripts/get-deployments.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/get-deployments.py b/.github/scripts/get-deployments.py index 81cc0161f..7dbf8cb6c 100755 --- a/.github/scripts/get-deployments.py +++ b/.github/scripts/get-deployments.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +''' +Get a list of deployments from the deployments/ directory, excluding any +directories specified with the --ignore flag. +''' import argparse import os def main(args): - for deployment in next(os.walk(args.deployments))[1]: if deployment not in args.ignore: print(deployment) From c888a45b2730808b89f40e61e60520befe248854 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 16:45:55 -0700 Subject: [PATCH 100/190] must remember: double quotes, not single. double quotes, not single. match the style of the rest of the python i write. match the style of the rest of the python i write --- .github/scripts/get-deployments.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/scripts/get-deployments.py b/.github/scripts/get-deployments.py index 7dbf8cb6c..1ffa5bf53 100755 --- a/.github/scripts/get-deployments.py +++ b/.github/scripts/get-deployments.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -''' +""" Get a list of deployments from the deployments/ directory, excluding any directories specified with the --ignore flag. -''' +""" import argparse import os @@ -11,20 +11,20 @@ def main(args): if deployment not in args.ignore: print(deployment) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate a list of deployments from the deployments/ directory.') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate a list of deployments from the deployments/ directory.") parser.add_argument( - '--deployments', - '-d', - default='deployments', - help='The directory to search for deployments.' + "--deployments", + "-d", + default="deployments", + help="The directory to search for deployments." ) parser.add_argument( - '--ignore', - '-i', - nargs='*', - default=(), - help='Ignore one or more directories in the deployments/ subdir.' + "--ignore", + "-i", + nargs="*", + default=[], + help="Ignore one or more directories in the deployments/ subdir." ) args = parser.parse_args() From d462e9a1009e495f0fde1cb07b7efa7326e27f36 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Tue, 27 Aug 2024 16:46:43 -0700 Subject: [PATCH 101/190] formatting --- .github/scripts/get-deployments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/get-deployments.py b/.github/scripts/get-deployments.py index 1ffa5bf53..56a6ec56b 100755 --- a/.github/scripts/get-deployments.py +++ b/.github/scripts/get-deployments.py @@ -12,7 +12,9 @@ def main(args): print(deployment) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate a list of deployments from the deployments/ directory.") + parser = argparse.ArgumentParser( + description="Generate a list of deployments from the deployments/ directory." + ) parser.add_argument( "--deployments", "-d", From 17dce9dd12b5abbd2d34d2e165d403b2dec264b4 Mon Sep 17 00:00:00 2001 From: Andrew Bray <andrewbray@berkeley.edu> Date: Tue, 27 Aug 2024 21:51:00 -0700 Subject: [PATCH 102/190] update version of stat20data package --- deployments/stat20/image/r-packages/2023-fall-stat-20.r | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/image/r-packages/2023-fall-stat-20.r b/deployments/stat20/image/r-packages/2023-fall-stat-20.r index e5217b064..83039e84d 100644 --- a/deployments/stat20/image/r-packages/2023-fall-stat-20.r +++ b/deployments/stat20/image/r-packages/2023-fall-stat-20.r @@ -43,7 +43,7 @@ devtools::install_github("mdbeckman/dcData", ref="56888a6") devtools::install_github("hadley/emo@3f03b11") devtools::install_github("andrewpbray/boxofdata@8afd934") devtools::install_github("tidymodels/infer@2806a69") -devtools::install_github("stat20/stat20data@25f1bae") +devtools::install_github("stat20/stat20data@11b4377") # file.symlink("/opt/shared/stat20/stat20data", "/usr/local/lib/R/site-library/stat20data") From 88fbf92d8454cb6c0e405258d25601376287e0f4 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 10:51:03 -0700 Subject: [PATCH 103/190] python really helps clean things up --- .github/scripts/determine-hub-deployments.py | 61 +++++++ .github/scripts/get-deployments.py | 33 ---- .github/workflows/deploy-all-hubs.yaml | 177 ------------------- .github/workflows/deploy-hubs.yaml | 60 +++---- 4 files changed, 87 insertions(+), 244 deletions(-) create mode 100755 .github/scripts/determine-hub-deployments.py delete mode 100755 .github/scripts/get-deployments.py delete mode 100644 .github/workflows/deploy-all-hubs.yaml diff --git a/.github/scripts/determine-hub-deployments.py b/.github/scripts/determine-hub-deployments.py new file mode 100755 index 000000000..f45328517 --- /dev/null +++ b/.github/scripts/determine-hub-deployments.py @@ -0,0 +1,61 @@ +#! /usr/bin/env python +""" +Check the Github environment variables for hub deployments and determine if we +will deploy all hubs or just a subset. + +All hubs will be deployed if the environment variable +GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT or GITHUB_PR_LABEL_HUB_IMAGES is set. + +Otherwise, the environment variables GITHUB_PR_LABEL_HUB_<HUB_NAME> will be +checked to determine which hubs to deploy. + +If no hubs need deploying, nothing will be emitted. +""" +import argparse +import os + +def main(args): + hubs = [] + + # Deploy all hubs + if ( + "GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT" or + "GITHUB_PR_LABEL_HUB_IMAGES" + ) in os.environ.keys(): + for deployment in next(os.walk(args.deployments))[1]: + if deployment not in args.ignore: + hubs.append(deployment) + + # Deploy specific hubs + else: + hub_labels = [ + k.lower() for k in os.environ.keys() + if k.startswith("GITHUB_PR_LABEL_HUB_") + ] + hubs = [x.split("_")[-1] for x in hub_labels] + hubs = [x for x in hubs if x not in args.ignore] + + hubs.sort() + for h in hubs: + print(h) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Get hubs that need to be deployed from environment variables." + ) + parser.add_argument( + "--deployments", + "-d", + default="deployments", + help="The directory to search for deployments." + ) + parser.add_argument( + "--ignore", + "-i", + nargs="*", + default=["template"], + help="Ignore one or more deployment targets." + ) + args = parser.parse_args() + + main(args) diff --git a/.github/scripts/get-deployments.py b/.github/scripts/get-deployments.py deleted file mode 100755 index 56a6ec56b..000000000 --- a/.github/scripts/get-deployments.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -""" -Get a list of deployments from the deployments/ directory, excluding any -directories specified with the --ignore flag. -""" -import argparse -import os - -def main(args): - for deployment in next(os.walk(args.deployments))[1]: - if deployment not in args.ignore: - print(deployment) - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Generate a list of deployments from the deployments/ directory." - ) - parser.add_argument( - "--deployments", - "-d", - default="deployments", - help="The directory to search for deployments." - ) - parser.add_argument( - "--ignore", - "-i", - nargs="*", - default=[], - help="Ignore one or more directories in the deployments/ subdir." - ) - args = parser.parse_args() - - main(args) diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml deleted file mode 100644 index 475d19655..000000000 --- a/.github/workflows/deploy-all-hubs.yaml +++ /dev/null @@ -1,177 +0,0 @@ -# if the PR labels "hub-images" or "jupyterhub-deployment" are present, this -# means the base hub image has changed, and all hubs (staging or prod) need to -# be redeployed. -# -name: Deploy base hub images to all hubs in staging and prod -on: - workflow_dispatch: - push: - branches: - - staging - - prod - -jobs: - deploy-hub-images-staging: - if: github.event_name == 'push' && github.ref == 'refs/heads/staging' - runs-on: ubuntu-latest - steps: - - name: Get PR labels - id: pr-labels - uses: irby/get-labels-on-push@v1.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check to see if the base jupyterhub image has changed, and deploy all hubs to prod if it has - run: | - echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done - - - name: Check out the image repo - if: ${{ env.DEPLOY }} - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Setup python - if: ${{ env.DEPLOY }} - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - if: ${{ env.DEPLOY }} - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - - name: Auth to gcloud - if: ${{ env.DEPLOY }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GKE_KEY }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Install Google Cloud SDK - if: ${{ env.DEPLOY }} - uses: google-github-actions/setup-gcloud@v2 - with: - install_components: 'gke-gcloud-auth-plugin' - - - name: Install SOPS - if: ${{ env.DEPLOY }} - run: | - mkdir -p ${HOME}/bin - curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops - chmod 755 ${HOME}/bin/sops - echo "${HOME}/bin" >> $GITHUB_PATH - - - name: Store SOPS secret in a file - if: ${{ env.DEPLOY }} - run: | - cat << EOF > ${HOME}/sops.key - ${{ secrets.SOPS_KEY }} - EOF - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV - - - name: Install Helm - if: ${{ env.DEPLOY }} - run: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ - helm repo update - - - name: Deploy base hub images to staging - if: ${{ env.DEPLOY }} - run: | - while read deployment; do - echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --verbose ${deployment} hub staging" - done < <(.github/scripts/get-deployments.py --ignore template) - - deploy-hub-images-prod: - if: github.event_name == 'push' && github.ref == 'refs/heads/prod' - runs-on: ubuntu-latest - steps: - - name: Get PR labels - id: pr-labels - uses: irby/get-labels-on-push@v1.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check to see if the base jupyterhub image has changed, and deploy all hubs to prod if it has - run: | - echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done - - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Setup python - if: ${{ env.DEPLOY }} - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - if: ${{ env.DEPLOY }} - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - - name: Auth to gcloud - if: ${{ env.DEPLOY }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GKE_KEY }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Install Google Cloud SDK - if: ${{ env.DEPLOY }} - uses: google-github-actions/setup-gcloud@v2 - with: - install_components: 'gke-gcloud-auth-plugin' - - - name: Install SOPS - if: ${{ env.DEPLOY }} - run: | - mkdir -p ${HOME}/bin - curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops - chmod 755 ${HOME}/bin/sops - echo "${HOME}/bin" >> $GITHUB_PATH - - - name: Store SOPS secret in a file - if: ${{ env.DEPLOY }} - run: | - cat << EOF > ${HOME}/sops.key - ${{ secrets.SOPS_KEY }} - EOF - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV - - - name: Install Helm - if: ${{ env.DEPLOY }} - run: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ - helm repo update - - - name: Deploy base hub images to prod - if: ${{ env.DEPLOY }} - run: | - while read deployment; do - echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --verbose ${deployment} hub prod" - done < <(.github/scripts/get-deployments.py --ignore template) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 807240013..317269a09 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -1,9 +1,5 @@ -# this workflow re-deploys SPECIFIC hubs to staging or prod if the single-user -# server image or config has changed based on the PR labels "hub: <hubname>". -# -# however, this workflow will be not run if the PR labels of "hub-images" or -# "jupyterhub-deployment" are present, as these labels will trigger the -# "deploy-jupyterhub-base-images.yaml" workflow which re-deploys every hub. +# This workflow will determine if the base hub image and/or single-user server +# image for any or all hubs has has changed, and if so, deploy accordingly. # name: Deploy staging and prod hubs on: @@ -24,25 +20,23 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + - name: Check for hubs that need deploying from the labels on the merge commit to staging run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - HUBS=() - # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this - # means the base hub image has changed, and all hubs (staging or prod) need to - # be redeployed. The rest of this job will not run in that case. + # If the PR labels "hub-images" or "jupyterhub-deployment" are + # present, this means the base hub image has changed, and all hubs + # (staging or prod) need to be redeployed. + # + # Otherwise, just check to see if the PR labels contain any hubs, + # and if so, deploy just those hubs to staging. if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then - echo "Base hub image has changed, not deploying individual hubs to staging" + echo "DEPLOY=1" >> $GITHUB_ENV else - # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then - hub_name=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$hub_name " echo "DEPLOY=1" >> $GITHUB_ENV fi done - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi - name: Check out the image repo @@ -104,10 +98,10 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | - for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do - echo "Deploying $hub to staging" - echo "hubploy --verbose deploy $hub hub staging" - done + while read deployment; do + echo "Pretending to deploy base hub image to ${deployment} :P" + echo "hubploy deploy --verbose ${deployment} hub staging" + done < <(.github/scripts/determine-hub-deployments.py) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -119,25 +113,23 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Pull out any hubs that need deploying from the labels on the merge commit to prod + - name: Check for hubs that need deploying from the labels on the merge commit to prod run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - HUBS=() - # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this - # means the base hub image has changed, and all hubs (staging or prod) need to - # be redeployed. The rest of this job will not run in that case. + # If the PR labels "hub-images" or "jupyterhub-deployment" are + # present, this means the base hub image has changed, and all hubs + # (staging or prod) need to be redeployed. + # + # Otherwise, just check to see if the PR labels contain any hubs, + # and if so, deploy just those hubs to prod. if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then - echo "Base hub image has changed, not deploying individual hubs to prod" + echo "DEPLOY=1" >> $GITHUB_ENV else - # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then - hub_name=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$hub_name " echo "DEPLOY=1" >> $GITHUB_ENV fi done - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi - name: Check out the image repo @@ -199,7 +191,7 @@ jobs: - name: Deploy hubs to prod if: ${{ env.DEPLOY }} run: | - for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do - echo "Deploying $hub to prod" - echo "hubploy --verbose deploy $hub hub prod" - done + while read deployment; do + echo "Pretending to deploy base hub image to ${deployment} :P" + echo "hubploy deploy --verbose ${deployment} hub prod" + done < <(.github/scripts/determine-hub-deployments.py) From f9ad125a17da338d89475ad277979369fa5109b9 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 11:19:46 -0700 Subject: [PATCH 104/190] add arg to allow us to limit hub deployments to only specific ones --- .github/scripts/determine-hub-deployments.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/scripts/determine-hub-deployments.py b/.github/scripts/determine-hub-deployments.py index f45328517..4598f370a 100755 --- a/.github/scripts/determine-hub-deployments.py +++ b/.github/scripts/determine-hub-deployments.py @@ -37,6 +37,8 @@ def main(args): hubs.sort() for h in hubs: + if args.only_deploy and h not in args.only_deploy: + continue print(h) if __name__ == "__main__": @@ -56,6 +58,12 @@ def main(args): default=["template"], help="Ignore one or more deployment targets." ) + parser.add_argument( + "--only-deploy", + "-o", + nargs="*", + help="Only deploy the specified hubs." + ) args = parser.parse_args() main(args) From ddacff134625698918fd808a80ec058e02e18b2b Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 12:42:14 -0700 Subject: [PATCH 105/190] seriously hating bash --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 317269a09..6c621b8bf 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -29,7 +29,7 @@ jobs: # # Otherwise, just check to see if the PR labels contain any hubs, # and if so, deploy just those hubs to staging. - if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then + if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV else for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do @@ -122,7 +122,7 @@ jobs: # # Otherwise, just check to see if the PR labels contain any hubs, # and if so, deploy just those hubs to prod. - if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then + if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV else for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do From 739c19ae8afab4eeb5bcbded0ae3ea76bb13e118 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 12:56:47 -0700 Subject: [PATCH 106/190] reformat comments to be a bit more clear --- .github/workflows/deploy-hubs.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 6c621b8bf..e68848288 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -27,10 +27,11 @@ jobs: # present, this means the base hub image has changed, and all hubs # (staging or prod) need to be redeployed. # - # Otherwise, just check to see if the PR labels contain any hubs, - # and if so, deploy just those hubs to staging. if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV + # Otherwise, just check to see if the PR labels contain any hubs, + # and if so, deploy just those hubs to staging. + # else for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then @@ -120,10 +121,11 @@ jobs: # present, this means the base hub image has changed, and all hubs # (staging or prod) need to be redeployed. # - # Otherwise, just check to see if the PR labels contain any hubs, - # and if so, deploy just those hubs to prod. if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV + # Otherwise, just check to see if the PR labels contain any hubs, + # and if so, deploy just those hubs to prod. + # else for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then From 8d31b03b1e0c66aca714d3093a93f757696467d8 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 13:15:27 -0700 Subject: [PATCH 107/190] more explainy comments --- .github/scripts/determine-hub-deployments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/determine-hub-deployments.py b/.github/scripts/determine-hub-deployments.py index 4598f370a..cc4010bb7 100755 --- a/.github/scripts/determine-hub-deployments.py +++ b/.github/scripts/determine-hub-deployments.py @@ -17,7 +17,7 @@ def main(args): hubs = [] - # Deploy all hubs + # Deploy all hubs by getting deployment names from the dirs in deployments/ if ( "GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT" or "GITHUB_PR_LABEL_HUB_IMAGES" @@ -26,7 +26,7 @@ def main(args): if deployment not in args.ignore: hubs.append(deployment) - # Deploy specific hubs + # Deploy only the modified/flagged hubs by PR labels else: hub_labels = [ k.lower() for k in os.environ.keys() From 132acfab0540bd78fb99b31df7d401189d24f31c Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 13:18:43 -0700 Subject: [PATCH 108/190] grammarizing the crap out of comments --- .github/workflows/deploy-hubs.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index e68848288..a9a9f98db 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -29,8 +29,8 @@ jobs: # if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV - # Otherwise, just check to see if the PR labels contain any hubs, - # and if so, deploy just those hubs to staging. + # Otherwise, check to see if the PR labels contain any hubs, and + # deploy just those hubs to staging. # else for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do @@ -123,8 +123,8 @@ jobs: # if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV - # Otherwise, just check to see if the PR labels contain any hubs, - # and if so, deploy just those hubs to prod. + # Otherwise, check to see if the PR labels contain any hubs, and + # deploy just those hubs to prod. # else for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do From 9f6491456751442bae6993ff58e248db3ead7744 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 14:30:06 -0700 Subject: [PATCH 109/190] might as well explicitly call python to execute these scripts... cant hurt --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index a9a9f98db..144f72728 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -102,7 +102,7 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" echo "hubploy deploy --verbose ${deployment} hub staging" - done < <(.github/scripts/determine-hub-deployments.py) + done < <(python .github/scripts/determine-hub-deployments.py) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -196,4 +196,4 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" echo "hubploy deploy --verbose ${deployment} hub prod" - done < <(.github/scripts/determine-hub-deployments.py) + done < <(python .github/scripts/determine-hub-deployments.py) From a7b591ae7fc6a2afe00fbb9fb54dcf64fdcfcc94 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 15:22:32 -0700 Subject: [PATCH 110/190] bashenannigans --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 144f72728..25d8c75ca 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -27,7 +27,7 @@ jobs: # present, this means the base hub image has changed, and all hubs # (staging or prod) need to be redeployed. # - if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then + if [[ -n GITHUB_PR_LABEL_HUB_IMAGES || -n GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]]; then echo "DEPLOY=1" >> $GITHUB_ENV # Otherwise, check to see if the PR labels contain any hubs, and # deploy just those hubs to staging. @@ -121,7 +121,7 @@ jobs: # present, this means the base hub image has changed, and all hubs # (staging or prod) need to be redeployed. # - if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then + if [[ -v GITHUB_PR_LABEL_HUB_IMAGES || -v GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]]; then echo "DEPLOY=1" >> $GITHUB_ENV # Otherwise, check to see if the PR labels contain any hubs, and # deploy just those hubs to prod. From 42be9afaf6755e48006808f492b23c57f6fe451d Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 15:56:45 -0700 Subject: [PATCH 111/190] fixing bash conditionals... again --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 25d8c75ca..144f72728 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -27,7 +27,7 @@ jobs: # present, this means the base hub image has changed, and all hubs # (staging or prod) need to be redeployed. # - if [[ -n GITHUB_PR_LABEL_HUB_IMAGES || -n GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]]; then + if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV # Otherwise, check to see if the PR labels contain any hubs, and # deploy just those hubs to staging. @@ -121,7 +121,7 @@ jobs: # present, this means the base hub image has changed, and all hubs # (staging or prod) need to be redeployed. # - if [[ -v GITHUB_PR_LABEL_HUB_IMAGES || -v GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT ]]; then + if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then echo "DEPLOY=1" >> $GITHUB_ENV # Otherwise, check to see if the PR labels contain any hubs, and # deploy just those hubs to prod. From 9451616f11c82a4eff7d3225474bdf914104e282 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 28 Aug 2024 16:51:46 -0700 Subject: [PATCH 112/190] bumping datahub placeholders to 2 --- node-placeholder/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 2c3932b5b..fb8d0ed88 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -135,7 +135,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 1 + replicas: 2 dev: nodeSelector: hub.jupyter.org/pool-name: dev-pool From 28de8e1a7d1df71662913688d78bfca551c3337b Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 29 Aug 2024 09:17:30 -0700 Subject: [PATCH 113/190] testing for logodev --- .github/workflows/deploy-hubs.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 144f72728..402e90dc8 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -101,8 +101,8 @@ jobs: run: | while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --verbose ${deployment} hub staging" - done < <(python .github/scripts/determine-hub-deployments.py) + hubploy deploy --verbose ${deployment} hub staging + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -195,5 +195,5 @@ jobs: run: | while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --verbose ${deployment} hub prod" - done < <(python .github/scripts/determine-hub-deployments.py) + hubploy deploy --verbose ${deployment} hub prod + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) From d3c6164e876795609232de9385277623a07392af Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:53:58 +0000 Subject: [PATCH 114/190] update logodev image tag to cb7832b0a131: deployments/logodev/hubploy.yaml --- deployments/logodev/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index d532f0a5b..893cc72ba 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:c44b36bfede9 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:cb7832b0a131 cluster: provider: gcloud From 5dea919e94c932f284232b90233657e293e070a5 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 29 Aug 2024 10:07:53 -0700 Subject: [PATCH 115/190] fix positional args for hubploy --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 402e90dc8..9b8637bb2 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -101,7 +101,7 @@ jobs: run: | while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" - hubploy deploy --verbose ${deployment} hub staging + hubploy --verbose deploy ${deployment} hub staging done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) deploy-hubs-to-prod: @@ -195,5 +195,5 @@ jobs: run: | while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" - hubploy deploy --verbose ${deployment} hub prod + hubploy --verbose deploy ${deployment} hub prod done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) From 123963d398096105eab0450b66fa72236d18810f Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 29 Aug 2024 10:25:13 -0700 Subject: [PATCH 116/190] adding timeout to hubploy --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 9b8637bb2..a072ba509 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -101,7 +101,7 @@ jobs: run: | while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" - hubploy --verbose deploy ${deployment} hub staging + hubploy --verbose deploy --timeout 30m ${deployment} hub staging done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) deploy-hubs-to-prod: @@ -195,5 +195,5 @@ jobs: run: | while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" - hubploy --verbose deploy ${deployment} hub prod + hubploy --verbose deploy --timeout 30m ${deployment} hub prod done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) From 90002bbcc34c747789d5b6e808792e5e0bde3a27 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 29 Aug 2024 13:54:54 -0700 Subject: [PATCH 117/190] moving stat159 to new image repo --- .circleci/config.yml | 25 --- deployments/stat159/hubploy.yaml | 10 +- deployments/stat159/image/apt.txt | 103 ----------- deployments/stat159/image/environment.yml | 173 ------------------ .../stat159/image/infra-requirements.txt | 29 --- deployments/stat159/image/overrides.json | 5 - deployments/stat159/image/postBuild | 8 - 7 files changed, 1 insertion(+), 352 deletions(-) delete mode 100644 deployments/stat159/image/apt.txt delete mode 100644 deployments/stat159/image/environment.yml delete mode 100644 deployments/stat159/image/infra-requirements.txt delete mode 100644 deployments/stat159/image/overrides.json delete mode 100644 deployments/stat159/image/postBuild diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a0754c58..541b8916e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -313,12 +313,6 @@ jobs: hubploy deploy --timeout 30m shiny hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy stat159 - command: | - hubploy deploy --timeout 30m stat159 hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy stat20 command: | @@ -496,15 +490,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: stat159 - name: stat159 image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: stat20 name: stat20 image build @@ -678,15 +663,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: stat159 - name: stat159 image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: stat20 name: stat20 image build @@ -727,7 +703,6 @@ workflows: # - logodev image build - publichealth image build - shiny image build - - stat159 image build - stat20 image build filters: diff --git a/deployments/stat159/hubploy.yaml b/deployments/stat159/hubploy.yaml index 87d309ad5..c56c228a0 100644 --- a/deployments/stat159/hubploy.yaml +++ b/deployments/stat159/hubploy.yaml @@ -1,14 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat159-user-image - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat159-user-image:2dc3f5a cluster: provider: gcloud diff --git a/deployments/stat159/image/apt.txt b/deployments/stat159/image/apt.txt deleted file mode 100644 index 96646f693..000000000 --- a/deployments/stat159/image/apt.txt +++ /dev/null @@ -1,103 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download/compression tools -curl -wget -zip - -# Core text editors on a *nix box: vim and emacs -vim -emacs-nox -emacs-goodies-el - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# powerful terminal-based file manager, better than the one in JLab -mc - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert and myst -# LaTeX tools -texlive-xetex -texlive-plain-generic -texlive-fonts-extra -texlive-fonts-recommended -texlive-lang-chinese -texlive-latex-extra -texlive-latex-recommended -texlive-humanities -texlive-music -texlive-science -latexmk -lmodern -latexdiff - -# Other useful document-related tools -pandoc -imagemagick -# /end nbconvert/myst tools - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -## This section adds tools for desktop environment usage -dbus-x11 -xorg -xubuntu-icon-theme -xfce4 -xfce4-goodies -xclip -xsel -firefox -chromium-browser - -# GUI text editors -emacs -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld diff --git a/deployments/stat159/image/environment.yml b/deployments/stat159/image/environment.yml deleted file mode 100644 index 0ce2a9864..000000000 --- a/deployments/stat159/image/environment.yml +++ /dev/null @@ -1,173 +0,0 @@ -name: stat159-s23 - -channels: - - conda-forge - -dependencies: - # Base language runtimes - - python==3.11.* - - nodejs=16 # VS code-server requires node < 17 - - # Other user-facing Python packages - - altair==4.2.2 - - bash_kernel==0.9.0 - - beautifulsoup4==4.11.1 - - black==22.12.0 - - bokeh==2.4.3 - - bqplot==0.12.36 - - cartopy==0.21.1 - - coverage==7.0.5 - - cython==0.29.33 - - dask==2023.1.1 - - dask-labextension==6.0.0 - - datascience==0.17.6 - - fortran-magic==0.7 - - gh-scoped-creds==4.1 - - h5netcdf==1.1.0 - - h5py==3.7.0 - - hdf4==4.2.15 - - hdf5==1.12.2 - - hypothesis==6.70.0 - - intake==0.6.6 - - intake-esm==2022.9.18 - - intake-xarray==0.6.1 - - ipycanvas==0.13.1 - - ipydatagrid==1.1.14 - - ipympl==0.9.2 - - ipyparallel==8.4.1 - - ipywidgets==8.0.7 - - jupyter-resource-usage==0.7.1 - # commented out by sknapp 2024.01.11 due to JL4 incompatibility - # - # we can revisit once this class is taught again - # - # - jupyter_bokeh==3.0.5 - - jupyter_server==2.7.0 - - jupyterlab==4.0.11 - # commented out by sknapp 2024.01.11 due to JL4 incompatibility - # - # we can revisit once this class is taught again - # - # - jupyterlab-drawio==0.9.0 - - jupyterlab-favorites==3.1.1 - - jupyterlab-geojson==3.3.1 - - jupyterlab-git==0.41.0 - - # LSP for JupyterLab/Python - enhanced completions/language support - - jupyterlab-lsp==5.0.0 - - python-lsp-server==1.7.0 - - - jupyterlab-variableinspector==3.0.9 - - jupyterlab-myst==2.1.0 - - jupyterlab_pygments==0.2.2 - - jupyterlab_server==2.19.0 - - jupyterlab-spellchecker==0.8.3 - - jupyter_server_ydoc==0.8.0 - - jupytext==1.14.4 - - mamba==1.2.0 - - matplotlib==3.8.* - - matplotlib-inline==0.1.6 - - mock==5.0.1 - - # nbdime 3.1.1 is incompatible with jupyter server 2.0 - # See https://github.com/jupyter/nbdime/pull/649. - # Use manual pip install from git (see below), revert once - # official compatible release is available. - #- nbdime==3.1.1 - - - nbgitpuller==1.2.1 - - networkx==3.0 - - numba==0.59.1 - - numpy==1.23.5 - - pandas==1.5.3 - - pep8==1.7.1 - - pillow==9.4.0 - - pip==22.3.1 - - plotly==5.12.0 - - pooch==1.6.0 - - prettytable==3.6.0 - - pyarrow==10.0.1 - - pypdf2==2.11.1 - - pytables==3.7.0 - - pytest==7.2.1 - - pytest-cov==4.0.0 - - pytest-notebook==0.6.1 - - requests==2.28.2 - - ruyaml==0.91.0 - - scikit-image==0.19.3 - - scikit-learn==1.2.0 - - scipy==1.10.0 - - seaborn==0.12.2 - - sqlparse==0.4.3 - - statsmodels==0.13.5 - - sympy==1.11.1 - - tk==8.6.12 - - tornado==6.2.0 - - tqdm==4.64.1 - - xarray==2023.1.0 - - xlrd==2.0.1 - - # Packages for Jupyter book. These are normal packages, but we put them in a - # separate section so they are easier to manually sync between the main - # environment.yml file and a pip requirements file that _only_ builds - # jupyterbook sites using Github Actions. - - jupyter-book==0.15.0 - - jupyter-sphinx==0.4.0 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - myst-nb==0.17.1 - - # Userland small TeX (similar to tinytex) - - tectonic==0.12.0 - - # Packages that could be installed via the system package manager - # (apt, homebrew, etc) but we bring them through here for better - # reproducibility - - git==2.39.1 - - pandoc==2.19.2 - - pandocfilters==1.5.0 - - # ----------------- Hub-only section ---------------------------- - - # Packages needed only on the hub (mostly linux) - # If installing this environment on a personal machine, - # comment these out (but not the pip section below) - - micro==2.0.8 - - python-pdfkit==1.0.0 - - syncthing==1.23.0 - - websockify==0.11.0 - - - jupyter-server-proxy==4.3.0 - # VS Code support - - jupyter-vscode-proxy==0.2 - - code-server==4.10.1 - - # Companions to tectonic, not yet available for desktop - - texlab==5.4.0 - - chktex==1.7.8 - -# Packages not available on conda-forge, installed through pip - - pip: - - jupysql==0.10.5 - - ipytest==0.13.0 - - jupyterhub==4.1.6 - - nbval==0.10.0 - - nbdime==3.2.0 - - # ----------------- Hub-only section ---------------------------- - - # Packages needed only on the hub (mostly linux) - # If installing this environment on a personal machine, - # comment these out - - jupyter-remote-desktop-proxy==2.0.0 - # syncthing for dropbox-like functionality - - jupyter-syncthing-proxy==1.0.3 - # Needed for RTC on NFS-backed hubs - - git+https://github.com/berkeley-dsep-infra/tmpystore.git@84765e1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - # for notebook exporting - - nbconvert==7.6.0 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 diff --git a/deployments/stat159/image/infra-requirements.txt b/deployments/stat159/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/stat159/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/stat159/image/overrides.json b/deployments/stat159/image/overrides.json deleted file mode 100644 index fbdcbfc47..000000000 --- a/deployments/stat159/image/overrides.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@jupyterlab/apputils-extension:notification": { - "fetchNews": "false" - } -} diff --git a/deployments/stat159/image/postBuild b/deployments/stat159/image/postBuild deleted file mode 100644 index bccc06bbc..000000000 --- a/deployments/stat159/image/postBuild +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -eux - -mkdir -p ${NB_PYTHON_PREFIX}/share/jupyter/lab/settings -cp -p overrides.json ${NB_PYTHON_PREFIX}/share/jupyter/lab/settings - -# Install the MyST CLI - https://github.com/executablebooks/mystjs -npm install -g myst-cli From e2a645ebcf5a65b8f4ddfe173943a9f1ebeea9f4 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 29 Aug 2024 14:10:14 -0700 Subject: [PATCH 118/190] allow stat159 to be deployed --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index a072ba509..4ff7da1d5 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -102,7 +102,7 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub staging - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -196,4 +196,4 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub prod - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159) From 2de811074e806685e8c51d148df99943b49dda27 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Thu, 29 Aug 2024 21:48:50 +0000 Subject: [PATCH 119/190] update stat159 image tag to 2d4c8f1488bd: deployments/stat159/hubploy.yaml --- deployments/stat159/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat159/hubploy.yaml b/deployments/stat159/hubploy.yaml index c56c228a0..88dbf1e7e 100644 --- a/deployments/stat159/hubploy.yaml +++ b/deployments/stat159/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat159-user-image:2dc3f5a + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat159-user-image:2d4c8f1488bd cluster: provider: gcloud From d747a476db3f639e2477ffcefdae9fbcff7ab5de Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Fri, 30 Aug 2024 10:56:54 -0700 Subject: [PATCH 120/190] adding README in old stat159/image dir to point people to new image repo --- deployments/stat159/images/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 deployments/stat159/images/README.md diff --git a/deployments/stat159/images/README.md b/deployments/stat159/images/README.md new file mode 100644 index 000000000..a38bf0a05 --- /dev/null +++ b/deployments/stat159/images/README.md @@ -0,0 +1,5 @@ +# Stat159 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/stat159-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat159-image/CONTRIBUTING.md) for instructions on how to propose changes to the image. From 63b6fe142fcaaeb6a4ddd3bde67793b989a0285c Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Fri, 30 Aug 2024 11:21:00 -0700 Subject: [PATCH 121/190] fix path to docs --- deployments/stat159/{images => image}/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename deployments/stat159/{images => image}/README.md (62%) diff --git a/deployments/stat159/images/README.md b/deployments/stat159/image/README.md similarity index 62% rename from deployments/stat159/images/README.md rename to deployments/stat159/image/README.md index a38bf0a05..6572078bd 100644 --- a/deployments/stat159/images/README.md +++ b/deployments/stat159/image/README.md @@ -2,4 +2,4 @@ This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/stat159-image). -Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat159-image/CONTRIBUTING.md) for instructions on how to propose changes to the image. +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat159-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. From accefb149787e9d2286894874e5f0776d057dc43 Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Fri, 30 Aug 2024 11:52:17 -0700 Subject: [PATCH 122/190] adding nature hub --- .circleci/config.yml | 25 +++++ .github/labeler.yml | 2 + deployments/nature/config/common.yaml | 93 +++++++++++++++++++ .../nature/config/filestore/squash-flags.json | 16 ++++ deployments/nature/config/prod.yaml | 18 ++++ deployments/nature/config/staging.yaml | 19 ++++ deployments/nature/hubploy.yaml | 19 ++++ deployments/nature/image/apt.txt | 52 +++++++++++ deployments/nature/image/environment.yml | 31 +++++++ deployments/nature/image/postBuild | 6 ++ deployments/nature/image/start | 5 + deployments/nature/secrets/gcr-key.json | 30 ++++++ deployments/nature/secrets/gke-key.json | 30 ++++++ deployments/nature/secrets/prod.yaml | 21 +++++ deployments/nature/secrets/staging.yaml | 21 +++++ docs/admins/howto/new-hub.qmd | 15 ++- node-placeholder/values.yaml | 8 ++ 17 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 deployments/nature/config/common.yaml create mode 100644 deployments/nature/config/filestore/squash-flags.json create mode 100644 deployments/nature/config/prod.yaml create mode 100644 deployments/nature/config/staging.yaml create mode 100644 deployments/nature/hubploy.yaml create mode 100644 deployments/nature/image/apt.txt create mode 100644 deployments/nature/image/environment.yml create mode 100644 deployments/nature/image/postBuild create mode 100644 deployments/nature/image/start create mode 100644 deployments/nature/secrets/gcr-key.json create mode 100644 deployments/nature/secrets/gke-key.json create mode 100644 deployments/nature/secrets/prod.yaml create mode 100644 deployments/nature/secrets/staging.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 541b8916e..6fa02f25d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -289,6 +289,12 @@ jobs: # hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} # no_output_timeout: 30m + - run: + name: Deploy nature + command: | + hubploy deploy --timeout 30m nature hub ${CIRCLE_BRANCH} + no_output_timeout: 30m + - run: name: Deploy prob140 command: | @@ -472,6 +478,15 @@ workflows: # ignore: # - staging # - prod + - hubploy/build-image: + deployment: nature + name: nature image build + # Filters can only be per-job? wtf + filters: + branches: + ignore: + - staging + - prod - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -645,6 +660,15 @@ workflows: # branches: # only: # - staging + - hubploy/build-image: + deployment: nature + name: nature image build + push: true + # Filters can only be per-job? wtf + filters: + branches: + only: + - staging - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -701,6 +725,7 @@ workflows: - ischool image build - julia hub image build # - logodev image build + - nature image build - publichealth image build - shiny image build - stat20 image build diff --git a/.github/labeler.yml b/.github/labeler.yml index c6a3ee5fb..a74875c95 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -115,6 +115,8 @@ node-placeholder-scaler: - 'deployments/julia/**' 'hub: logodev': - 'deployments/logodev/**' +'hub: nature': + - 'deployments/nature/**' 'hub: prob140': - 'deployments/prob140/**' 'hub: publichealth': diff --git a/deployments/nature/config/common.yaml b/deployments/nature/config/common.yaml new file mode 100644 index 000000000..4ffa072de --- /dev/null +++ b/deployments/nature/config/common.yaml @@ -0,0 +1,93 @@ +nfsPVC: + enabled: true + nfs: + serverIP: 10.75.147.186 + +jupyterhub: + scheduling: + userScheduler: + nodeSelector: + hub.jupyter.org/pool-name: core-pool-2024-05-08 + proxy: + chp: + nodeSelector: + hub.jupyter.org/pool-name: core-pool-2024-05-08 + + hub: + nodeSelector: + hub.jupyter.org/pool-name: core-pool-2024-05-08 + config: + loadRoles: + # datahub staff + datahub-staff: + description: Enable admin for datahub staff + # this role provides permissions to... + scopes: + - admin-ui + - admin:groups + - admin:users + - admin:servers + - read:roles + - read:hub + - access:servers + # this role will be assigned to... + groups: + - course::1524699::group::all-admins + singleuser: + extraFiles: + remove-exporters: + mountPath: /etc/jupyter/jupyter_notebook_config.py + stringData: | + c.QtPDFExporter.enabled = False + c.QtPNGExporter.enabled = False + c.WebPDFExporter.embed_images = True + extraEnv: + # Unset NotebookApp from hub/values. Necessary for recent lab versions. + JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" + nodeSelector: + hub.jupyter.org/pool-name: nature-pool + storage: + type: static + static: + pvcName: home-nfs-v3 + subPath: "{username}" + memory: + guarantee: 4G + limit: 4G + + #custom: + # group_profiles: + # + # # Example: increase memory for everyone affiliated with a course. + # + # # Name of Class 100, Fall '22; requested in #98765 + # course::123456: + # mem_limit: 4096M + # mem_guarantee: 2048M + # + # # Example: grant admin rights to course staff. + # # Enrollment types returned by the Canvas API are `teacher`, + # # `student`, `ta`, `observer`, and `designer`. + # # https://canvas.instructure.com/doc/api/enrollments.html + # + # # Some other class 200, Spring '23; requested in #98776 + # course::234567::enrollment_type::teacher: + # mem_limit: 2096M + # mem_guarantee: 2048M + # course::234567::enrollment_type::ta: + # mem_limit: 2096M + # mem_guarantee: 2048M + # + # + # # Example: a fully specified CanvasOAuthenticator group name. + # # This could be useful for temporary resource bumps where the + # # instructor could add people to groups in the bCourses UI. This + # # would benefit from the ability to read resource bumps from + # # jupyterhub's properties. (attributes in the ORM) + # + # # Name of Class 100, Fall '22; requested in #98770 + # course::123456::group::lab4-bigdata: + # - mountPath: /home/rstudio/.ssh + # name: home + # subPath: _some_directory/_ssh + # readOnly: true diff --git a/deployments/nature/config/filestore/squash-flags.json b/deployments/nature/config/filestore/squash-flags.json new file mode 100644 index 000000000..832c32e7f --- /dev/null +++ b/deployments/nature/config/filestore/squash-flags.json @@ -0,0 +1,16 @@ +{ +"--file-share": + { + "name": "shares", + "capacity": "1TiB", + "nfs-export-options": [ + { + "access-mode": "READ_WRITE", + "ip-ranges": ["10.0.0.0/8"], + "squash-mode": "ROOT_SQUASH", + "anon_uid": 1000, + "anon_gid": 1000 + } + ], + } +} diff --git a/deployments/nature/config/prod.yaml b/deployments/nature/config/prod.yaml new file mode 100644 index 000000000..d0c7b77fa --- /dev/null +++ b/deployments/nature/config/prod.yaml @@ -0,0 +1,18 @@ +nfsPVC: + nfs: + shareName: shares/nature/prod + +jupyterhub: + ingress: + enabled: true + hosts: + - nature.datahub.berkeley.edu + tls: + - secretName: tls-cert + hosts: + - nature.datahub.berkeley.edu + hub: + db: + pvc: + # This also holds logs + storage: 4Gi diff --git a/deployments/nature/config/staging.yaml b/deployments/nature/config/staging.yaml new file mode 100644 index 000000000..572ad127a --- /dev/null +++ b/deployments/nature/config/staging.yaml @@ -0,0 +1,19 @@ +nfsPVC: + nfs: + shareName: shares/nature/staging + +jupyterhub: + scheduling: + userScheduler: + replicas: 1 + prePuller: + continuous: + enabled: false + ingress: + enabled: true + hosts: + - nature-staging.datahub.berkeley.edu + tls: + - secretName: tls-cert + hosts: + - nature-staging.datahub.berkeley.edu diff --git a/deployments/nature/hubploy.yaml b/deployments/nature/hubploy.yaml new file mode 100644 index 000000000..1ae5a7106 --- /dev/null +++ b/deployments/nature/hubploy.yaml @@ -0,0 +1,19 @@ +images: + images: + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image + path: image/ + repo2docker: + base_image: docker.io/library/buildpack-deps:jammy + registry: + provider: gcloud + gcloud: + project: ucb-datahub-2018 + service_key: gcr-key.json + +cluster: + provider: gcloud + gcloud: + project: ucb-datahub-2018 + service_key: gke-key.json + cluster: spring-2024 + zone: us-central1 diff --git a/deployments/nature/image/apt.txt b/deployments/nature/image/apt.txt new file mode 100644 index 000000000..a990f28e0 --- /dev/null +++ b/deployments/nature/image/apt.txt @@ -0,0 +1,52 @@ +# Some linux packages for basic terminal work, particularly +# oriented at users new to Unix/cmd line environments. + +# installing less as more just isn't enough +less + +# Download tools +curl +wget +vim + +# for easily managing multiple repositories with one command (perl-doc +# is needed for its help pages to work) +mr +perl-doc + +# Regular build tools for compiling common stuff +build-essential + +# Dependencies for nbconvert +texlive-xetex +texlive-fonts-recommended +texlive-plain-generic +texlive-lang-chinese +lmodern + +# Other useful document-related tools +pandoc +latexdiff + +# Some useful git utilities use basic Ruby +ruby + +# Other niceties for command-line work and life +rsync + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-325 +libnss3 +libnspr4 +libdbus-1-3 +libatk1.0-0 +libatk-bridge2.0-0 +libcups2 +libdrm2 +libxkbcommon0 +libatspi2.0-0 +libxcomposite1 +libxdamage1 +libxfixes3 +libxrandr2 +libgbm1 +libasound2 diff --git a/deployments/nature/image/environment.yml b/deployments/nature/image/environment.yml new file mode 100644 index 000000000..7947296da --- /dev/null +++ b/deployments/nature/image/environment.yml @@ -0,0 +1,31 @@ +name: nature + +channels: +- conda-forge + +dependencies: + +# Items required for basic level functionality +- python==3.11.* +- git==2.46.0 +- jupyter-resource-usage=1.1.0 +- jupyterhub==4.1.6 +- jupyterlab==4.2.5 +- jupyter_server==2.14.2 +- notebook==7.2.2 +- nbgitpuller==1.2.1 + +# vscode +- code-server==4.23.1 +- jupyter-vscode-proxy==0.6 + +# other packages +- seaborn==0.13.2 +- altair==5.4.1 +- ibis-framework[pandas,duckdb]==9.3.0 +- leafmap==0.36.10 +- jupyterlab-myst==2.4.2 + +- pip==24.2 +- pip: + - nbconvert[webpdf]==7.16.4 diff --git a/deployments/nature/image/postBuild b/deployments/nature/image/postBuild new file mode 100644 index 000000000..f8d71f964 --- /dev/null +++ b/deployments/nature/image/postBuild @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +playwright install chromium diff --git a/deployments/nature/image/start b/deployments/nature/image/start new file mode 100644 index 000000000..c3a978b7f --- /dev/null +++ b/deployments/nature/image/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" diff --git a/deployments/nature/secrets/gcr-key.json b/deployments/nature/secrets/gcr-key.json new file mode 100644 index 000000000..909076d23 --- /dev/null +++ b/deployments/nature/secrets/gcr-key.json @@ -0,0 +1,30 @@ +{ + "type": "ENC[AES256_GCM,data:0I7dYFUSBF5oONBtUxkc,iv:Ws7MGUhHoDOGESUyEi+HdPY1CbBDB98FufA8d+cOw18=,tag:bkfWgySRKcIXnIzB5/9AJQ==,type:str]", + "project_id": "ENC[AES256_GCM,data:TwqH+3sdXcs3zOOSoCgkaA==,iv:vDjOAoumM6tnMNcVkzlqPk7ujGEwOI/Db0kVaO5AAqk=,tag:OT7c60NkyS1aw0L1HvYd/g==,type:str]", + "private_key_id": "ENC[AES256_GCM,data:EpePb4USzZBigubVyyo+sjdNcGXAeyXgmJS33MZtDqXs8VfBCvL16w==,iv:NLyJFb62z/Yj+4X0IUTN8WxTgdpxtc5udlJ89WH6c6I=,tag:y9bHxvqYyaJsxMLaHhkvsA==,type:str]", + "private_key": "ENC[AES256_GCM,data:7W6mEaJGtJj26n+DzHoNy1MMLWha5mIKH+D71Gt7vjVzM/oHKbQCpF550Q/rX9iTsxDpYwQqKETF82Rb91GwnNTfXUhcFneyAcEuKUMlnlqdDgEUIOKMJf5ZAyARDK/nA38sLIbt+nFK8wVEVETJqQd09m+Wy1ebD6XuoXEaUiLq3drzwcwUTF8UgcClx5IGYbbNuFvQy3B4pVx1N/Ow1BtgiDO/OEFwXQhyS5kCGq8MHAX3bm1NpyQ2+cQNhpQ/A+H3/VseafjxiSGUXgkBRrz16zyS1yJoI87tV15ky5YZ4jgNrNlYxXfydUK7OvMz+gCvc/eBknru3UW75DC3PXYmmA2JnVoFieVGhtiAmuzFdK3u+uq61FnMjaCAhwvcdPV/Ml/Wavkawj3eViZj4ZfJalLNpWksHrAE/HYg8CHVW6P0Whws34Ya1xgLqnurUNE0CsyDqy7Tyrr0NEwDSox2d5eOMM3mADRt4UGtlB+Nm/rOg8IB5Gdc6904bIeaHTkcf5PE2ftoeI8NiNevyK9N1uYyh9Ahn3zFsasGWs8kPoISqCTM82dMXRe/Ayz2vZG3YpSY1N9EVE47784Pvt2+Eaxb5tFEqRopkRg2eGSj37GgI+feqLOwpPbrtyb3/4Nm9XWFoC6xn9FKR1iUOzCxL54JHf0arwl3ZUOr989qdYymH2wSpI8QdCpdI1ltkz8tONZze8z49mVoC+dyZlG0f8a+DWmX7cu3zD/MufKw9bySlsZl4X5odGU7SUwjeSOLx2PrKIxdMCVHO4RfhU0HCGUfdVOBCrEb+Uf6LUHDvnYqZ2O5/WUj+HMf+lrltLuMdesNFUu4XN7tyyTSUNfvma0pU3ljiolY/Aet/WfIMk3g0O2nCe4R21YgUT/w03sBYztPwlb3MaI5x/vveCtGfNdo8aotQ2e50JU/Q7Nj2Nght/Bca/Q0yZzQ9QiiE9Bty9VJgfXDBUawzd+LJyRmLvNHOtpTv0gfb0TOFDRJc8ngv9+rXfc+0rg3gJYXrgnfqMFBMxbK/QLAV7iSL0ZDmo6PQhpDYHwJGRmWgz/WnGX5bJYX2OW5wuTHZrArtVKIh7w0enMZLWEqVB9Iihl+vnOWBw+VvCzzYRnqmGEF9qY2y+bvgVNTenD5B5iHrQReJ/F1pBrfx9LyMs1qJ4NrwNl16/zD/YY4xlMV/i3dXyL6C7CopMl9wceb1ZQayxhT1GJVAjLPXYNWihfOhotbelNW6hWBPG7nIgI0jsgsX8hs/uI0HTmltRlosyjbQDTLABXiL2xx3wpAPrFG+asduM/b8JIqOc25SY5EYn8Jn8vW2Nr4L8y9DHmdxKgehAVRSL5jUJeJ9+p86ZdGe888w/zEP6XDKyuZy6TEcHfHoyKQvCfVSx0XChz5qTdjh58moe15bCFqRxdDosz70+Mi9myx4ZcPPZPo4+wKcKyL47mrUDui+E7C2ehSw4b7Y1Iz0CvDpYvlYaO8mr0ER9prN0q6X364Kw5OSXH5/LwRHeKtP9U9NSvlb8p6ATW9xAXu7xLDtbNu6tGcYq+eCHlwMIODX/4WSrwWfyWd/C6EJfbAG1e1AhzIXziQIsT/cqbp08ZPSuMTpxNdKn+SdrWOat9Zxr6glVDsdyO2CAWAAmpw71rypgxwvxPD/qEZoyjyHr0B3D/lFLtM43nNSwhAVcK9e+rcPX5VBLRg7Bf6s2B3yvC8Jfn0a6JN7HvrMp6Tt19gk2Q83nDX4aGWr8H4b1OBMehmUUFtDupoDlXxzchTR42Infvvp0iUmWbvipVMplu0mUrXHrzA8okv9u45LNUt2BPQE+cFgU1Uuvg4H2vhATSpr3YU94hM+C7quOkm77F+njDEtpa1fC6kKdeE2emXpKBqgf2j8oVCUpw1QITp3ZLXwr1PejrZ6IRM9ZDsrVF3O6Px/yl2imq+SYGMA9Ee0yue2OUxxoKp1VM3dictwI2JrwKoQVo+W0Tsd1Zj5GMcbJZsLJW54J3Kp3R78t9HgaW4R2x1yBbKrzE3VbAfoanxId6deAIZtEhLxoetDSNmxekrIjSan/GyjiLLCRf3sq8hKYvlGPW8wmge9H2jvd8ghmDRnITRKfBHohNGPQd9i53Q3iomkmiqxlYIbZFN1E2u2tSEyM2a5gI17+C2+JlUXa8AgqYVPI0Cdm5dzdJAxFH+BlAvKLKx4jvLSUNC2Igf40GEnCCzHWOk4anl0pAToovxecHXGZEIvL2LG5UVLp0zgOqvBQlsKA/f/90yl/5A,iv:9qLRN/v7Q2Twl/5SaOOCN3ACOs8ZyLnZx+Qkbg6X2Ig=,tag:cNLrRzp6ZWnU9WgWc0kdqQ==,type:str]", + "client_email": "ENC[AES256_GCM,data:r1/9+NDcLQ5oh4MAaF0h2gPfUJJLoMZ6SIJnBgnpa/nCCoAjOn3BlMlVb+odkhGnW1W4qIDnRFuepw==,iv:ALhr5g8VLEOV3PPw8E7rbV/INU2qte0VEJVbtVF8kWk=,tag:l8a4Al6+9k7HiO9fnclcGA==,type:str]", + "client_id": "ENC[AES256_GCM,data:lKB1S3DiB430noJTuWEFsxzhqeu6,iv:f9wPi87mJekuNaO4kBOsMEe7aK0qaVUTRRht7skZQd4=,tag:DgXci6MDlY7Y7v/p6pOAzA==,type:str]", + "auth_uri": "ENC[AES256_GCM,data:sw+45WN8mou/cSjmcbdweGel3odEV7tTw3pNSywaw0MSWSJ0H9qZIhM=,iv:tmCzryizCEmwQ/vnVhF2xecwXtjkj9j74qYX2OXfY8E=,tag:X2JuV9/6Ir4LyE8E7uUqLQ==,type:str]", + "token_uri": "ENC[AES256_GCM,data:u8bETqFMsmGIhG74OvFh1guEawWasf/TPi9W6q5mr5xdR0U=,iv:SvY5NGGU2pstrT3bgk7tLy72VIZWIcrtb6n4fQJmaJI=,tag:q/+ZftThljF/NQkx4yyWng==,type:str]", + "auth_provider_x509_cert_url": "ENC[AES256_GCM,data:0Wyr5hZPP8EFxj09do28+hTNpRQ2W7kgCd8RKwgdgj+HjyngwFNMlX8t,iv:dtagZNXNeN4gAO0EQSCPe4aJ2jo64uVgy7KNpm0O554=,tag:BTKYg7hnSmxCVBcYUQOGkA==,type:str]", + "client_x509_cert_url": "ENC[AES256_GCM,data:nBL+9RFEq7Yak5F/rfMTLVmrTlwRqJcmU7XuTby+CTWYxLrvj45cJ4zGjrFJVprR4SX/atfJvd+oz3INJtj+R2xuFqeAEwOA83cQ8hc2QQTp/dZnIfx/mfhLYcee2pYSdw9UvUeVMwZnrMNU2Ds=,iv:4hrK/i4PMwffA0fPkhO1IhOMgEuXo1BGUVHuK0gFjMA=,tag:LxT+OWq5eV72w5YjoSvgJQ==,type:str]", + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops", + "created_at": "2021-04-09T20:37:52Z", + "enc": "CiQA67O9AMN7YazFkiINr1H6EseNuc5e1Uaev/rqdGmyNJN0U4QSSADmhpq83a6J0ki/Id/e8MhBAAKdaADW6SFNX8J8wlVaXX2s7u3P8wMYloROZUpRG+mrpXAVsLAEWRFKV9ITu+TVluTg3Ig1HQ==" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-04-09T20:37:52Z", + "mac": "ENC[AES256_GCM,data:B+LJnO3hpZQKgKtQaDdxJMVVW80uDAdFBzmEBXOlE5IR5Qj20lgSbdwo7ELo1+pZnsa3m8Efik1qUkYm8cRz0FPDxt1GjUMNQ3EnkZYBQBY5I1IdS/+GkYIlGnzbwnc2+K5rx2n8dXoCKBLZmbLLgaLWJHgVN600UrBtVXwtohI=,iv:/nSQ0o+Z4hNQCjcHOB4PMvyhZI4Nv+POxOL4W3G6NVU=,tag:f+o3HM7bGecoDcS0CU1sSQ==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.0" + } +} \ No newline at end of file diff --git a/deployments/nature/secrets/gke-key.json b/deployments/nature/secrets/gke-key.json new file mode 100644 index 000000000..b93212fa3 --- /dev/null +++ b/deployments/nature/secrets/gke-key.json @@ -0,0 +1,30 @@ +{ + "type": "ENC[AES256_GCM,data:pviDYpoudn2mPXuSTCau,iv:5gsuBqez83Rd1A/wbbkKd5e3qbMzEmD7pL7jXW0cPqU=,tag:0mwsPS5wQptG/DaSQgm8HQ==,type:str]", + "project_id": "ENC[AES256_GCM,data:mR7xINYn/ly6dkpKJTZycg==,iv:Nxu2uscmC80IVfcAdDh9COmtAGDdtCHQ5uAORvjdGKM=,tag:ZejnMpC4ilLXA4D3tnxxwg==,type:str]", + "private_key_id": "ENC[AES256_GCM,data:6bJxLE0n0QM1HLX+ennIeHf9H6SYylodt5+zsgH07fjc30RSpBRUKA==,iv:YUFoEj2fQ4pNHt7TVVosBi9k9w6aXIKsQU6DSFG8brc=,tag:RK6k0YlB991yxtf+95E0Vw==,type:str]", + "private_key": "ENC[AES256_GCM,data:oF+xAYXTSufe3xtFNU9qlBZDX0iNhf9oUre7H7h5AygW0o/pu7PZNkuJYUGbcQuLS4zWWzJUFlhUxs3bjXGELhcqPa3VJVBHGUWjvbwMHUMW775KZOJaXj3fg66mF50Fnj24rqfMYh8ZXdkrlt5/EtE50CdCEVay60cODk4QMG0VIT/6PVrzrQlKz0EvkRnfOC9Xf2bBolWmKcnkw5V8BXZGh+kcRdxCwdNBNdTDHeJustlHaPdo86k2OzVu8Cbpll5iIAlJ034aFESEZrmyruYZ2Vcbh7fI9++gIVk7GiuSXk9bKn7HQAs3RcjcyiJ/x77IF2t7YQGpgpCggyJN17ufMqD11TyjLLRZzzpCmcv+OOQdORCTE+x+9AfePTNVd/G6fRLjWMGPd9LSstsNDkxgeYGWL00LlzzzRqTztMrRZOfa3miM92+G+tPRqHCjlP1tkzkKgpWpCshAGBQlZmwGSUBtVOjwBCVC6V72c7c8n28FktFjRylYZHJpAPB6U1XHD0Nojb7ZHkA1lsl05j79Ht2PUHZdqCEWct8WSSxahcIOWV95kwb9Vk2oCc6lsFhIVq1L2+ZR4O65D/JgljFLpyUPYNMKyfW7ZxNWC0Eb2GUqAKqyOIKyH5dZA/vpwJwoD6GHX/oZM9H3eg0NSr+mBUL3FRxDLa2TU8emHH5goxTw6wUaIhVndsYGPnhn/gPPsGaO+naTAq3uR1MLf6ednVn2GUNJdm1gdvXq9Q4woWO52CBGFVFuaSHsaWeN9t3kAq27ba071PJ/4uEpYRWZb7Iyt3BFgJbe8TZyO2+PfLaK5+ydDMAwJQLwmIpAk7Q86i0jkoEIPF0ZegH871UJALT6Jqf03dkYgPTlIgbb/EmyB0VSmkj7EGPxdBaCpz7dvdKlhmnPqA8rdarS/VFAjTJw0BOnNDPD4a3klQujqDEzv83RwkPj2ZCA1axfjGjvMacAF1e76Vesp9cOLmo6e4TU1BcC0vZ89tfVsxkZiAs/E2RWmTpgfklmv0dPzOkd2Eht+9oc1/mg0kUC1ipofKNYz4VacLmVYAe/gY8bC/wwcaM2mYjue10voUQPrleUHf4F5rlplHwfnJa88snh25EGAC8OqBoY+nmx6qq1ZCcugJaFW7ewh2oMfROlzYljScUk31C15vuLNTw/XcfbC1UwiuAZNsHCM6H99MycVDyYG5RffqK/LqQ2l/9ORE7YTZU9bwT836bTW8SkUfLdH+cDScuUamK9JYoyiTXlj/9GovywPgQ6AaE6z2CUKe+BZejwT5kNM/C+D/EK8WYOTVzWtE9mIdrgPfKNx80a6rOYIhtfb3YfvjLS4BD3sDPEPdl6U6koeyS0cDaVfAuAUrHiofXlxijyZ5XBxAAjzUFoZuCkg3AL209bcXiuW08ePn+XzVae0mDFoB3bJGKovDaE8dw0U3annk2kG/mHxjxPYbeYRB+UgVsxG9j2Pc9i9SiExJLj/HHyt5gfBH2Q2lISEmKEEkGtoh5Z40hcYnzgl6RzMtJYenSXmeQ3UOB2TRUNwRgYZfc3+hTHoqINBE5wK7vD/19HIJGLRmYwn8hnGf6p/gXxSmWum7t1wxuoawqWp3To3eF/OjX2MH/REwHQJHCo9PDPIZjBRWz1D8TopWi+v00C04+T8hqpD1H8QuNpdNKAYEEh1BjjW00aqNCQ+9JPfglN8+PwY7YtgGP15hkDNXZxDfb6RhIo6gw6wgwgTXz7N1VuTk7OEtB9OGrPL6KKOFoM6wg44qbSUZXnTY6ZezynUaqtYnlGVgCQGH6WyDfFdBYcyy6XXiztRinBMIG+b87Z0ucI5sGVvniMhCqOeLybaOKY+lSHL/cfZGJNU0tQaN0kRx5R69NWDN7mNyJqND4Y88aJs2mY62XJiaaSYCtuKezCgnvfebr8e+kleniI0hemGI1D3cMOcLMezgEYtobKKLzxg9YzZUxwcgvB7riS8ljcYSTwY15gVOnVfGBxMpjYojnaY5THtC4hiT97A3P/tn6nGFcO0quWATQ28Ef1bgjD08rScIcWgnq40uN1bgtsvPZDaxr4sCGSrNQEO7JO85Ed+bEWiu1/1giaNhsMP56ezwF9VsL4E0zvwPXI3zidfPfKDdJc1QvZGZPTfDJx2XJWSRrtAtYtYPyLUVq//lqIgn+N4Gw7R1gl9BM9iHPt09Nni5Nn3BWHxc9ELp3K0oAVaLcfEVxLQdCI3OvXHIOHxh3U07XsJlp4TuT1tu9gYBsDy6gcrX7ybJAk,iv:rbE1gehD5XO8z8OZ2pzZgGKXb//IP9VkeqLWVYazENw=,tag:0g/VRRFexQDWSkKBoD+b9Q==,type:str]", + "client_email": "ENC[AES256_GCM,data:F7CrAz9fxWpAfIgxzUtHuw4eWMh08MpcKpoLVW6WygRQEbqO6WQIxBzyrU1goMhA,iv:9FXvjHt2hy6spkWfyOQr1m67cWd6vChV+8vg5Frd7XI=,tag:MjptTtfiWHKoiVhB3dC9dA==,type:str]", + "client_id": "ENC[AES256_GCM,data:NFrWfX2TNO9AKzPOYHyM1AIX50eo,iv:lLV/e9tj/ba0KvxOMAYLWv3RyYU9n97ku2GHCCkgED8=,tag:H7k/cQqLJBNOaXnPCv4JfA==,type:str]", + "auth_uri": "ENC[AES256_GCM,data:K85ELixk7QPpuqhjNTG4QIYUbz5yyKy3qXfarTxO650S77SJWXHsYR8=,iv:vt2GJ7cwKJyY/7XDiCZdX51ZBY3nb7g9miHmmfabriY=,tag:HB8xc5oeEc1JCnqIa1eZYg==,type:str]", + "token_uri": "ENC[AES256_GCM,data:kaKX9Zx/DU0rqFWeyoVEpghkCjqOmTjifSgg0NU4ESdI5Ss=,iv:Rp5NJKCRDdxbw8xmp0gcy+CnilVE6YeookI3criH/mE=,tag:729lX9HQpeyUdg4C1AQAPw==,type:str]", + "auth_provider_x509_cert_url": "ENC[AES256_GCM,data:2tTSvbO8RUjv1cUwIg3/JTZcHIVKt+Ki4tyPRxvMwj46YeRkeE+ZC0/p,iv:9ci+AYYmjPq7hfIFPuilgUtxks/9ypIgU4+PXvwfSzM=,tag:AsGU//l4WbQX85/NV1L5dA==,type:str]", + "client_x509_cert_url": "ENC[AES256_GCM,data:uAvNlbSLjImhE93+Z6NuIRdfVVGbmvwf9cFD5N7y9Ln79W7Oa6Mi3lUgyzbmuk5T3j9W+u2SONcKMRiyekyXXsBEOWlJP5OHYb3x0jfO0dIy4+eWXxkkkfFgz6CNX7Z6xZJ+kg==,iv:PUC1h4esq7Bf6594yhV/CMHxFlbXbh48Y2RIJOVwUaQ=,tag:YBJjzkvNe9I4xUGZEYCQ5w==,type:str]", + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops", + "created_at": "2021-04-09T20:37:56Z", + "enc": "CiQA67O9AJiRNfloqVj5a7vZOpd6u0guaOQw8jj6EQhNSo6WNY0SSQDmhpq8262T34fGwV/BFo5FV/8s5kyPq3+1UG4orX22ASsKhzXJiOUxc5BqF8oVkMXgkjsR5saTuBI1tqpVbwfin8vspFa2JDo=" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-04-09T20:37:57Z", + "mac": "ENC[AES256_GCM,data:RCfor+HF2+DT4GWZ3dooDb9hhR7FWMmYJ9e6SRdXQcG25eIU9Jo5H+OdvRJVj4d5slljcjeIGadgODgvdfLD+rqq2RGTqp/cZJ1qlpdIec2y+6AHt0mqM7lHLw/qLs/5Cx/MMmSGk4BrIvGlK22ciixP660JV34mdSJuG/4As3Q=,iv:nl6FLf43mKK6MSQKEkHxeHoTuqwYEm0ZNwFNj0b+pCA=,tag:ZJQvL+MAuM11C1oTS2hLpw==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.0" + } +} \ No newline at end of file diff --git a/deployments/nature/secrets/prod.yaml b/deployments/nature/secrets/prod.yaml new file mode 100644 index 000000000..f88216a6b --- /dev/null +++ b/deployments/nature/secrets/prod.yaml @@ -0,0 +1,21 @@ +jupyterhub: + hub: + config: + CanvasOAuthenticator: + client_id: ENC[AES256_GCM,data:/VedfPLyL+Rj3gciMxQ5H84=,iv:WZsC/06SEfEEeH4/NY+txGypgP/lShrJLc8DVXS7tco=,tag:kdZCzktRscuCYz4nRlkxTA==,type:str] + client_secret: ENC[AES256_GCM,data:EqzXDvAIMyGeTnGjUDOg6X6XqSwDmogtz/HXhc2LNYMhoSgg8PSUqvEFNUNG6w59tkRXPerBR6PyOuG31ftYXQ==,iv:D/rf3aWams6O2NcmdjFYxEVV5dQoWv3ubZ1kTN5KMO8=,tag:/LLaEHP/RB17Td0UW1I2bQ==,type:str] + oauth_callback_url: ENC[AES256_GCM,data:53XmEOsjx6DttwhKUhSJYXwym9I8kuJd3p6GTIiV9b+f11dfADdqD75L6tmrvtnglopDVh/g,iv:8qtAR/o0WuOMvLa4UKhr0JOwJv+bFExzJT0ikPFgP4g=,tag:1BmSfQ/o7H66ATjdIWHFvw==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops + created_at: "2021-05-05T10:57:58Z" + enc: CiQA67O9AK2027WGYGTzywa01Cz+Ez7sOTk/d9payovyK5pg8g4SSADmhpq89bbIWFjlGg79o/iupJ4anLU5Ab9VL+qNzhu6e83JtJ7wSv6sK+cDiEfVSaKQ1YIcadDXFt4WUKRt7MFvAa1sLqp2LA== + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-08-30T18:15:24Z" + mac: ENC[AES256_GCM,data:hRMon9bg/+czXTjqV5bsZGYddmFjkZRIVkHq1xGg3mD04Vam1E13wqyZcWyj6lK35bkSiEJ7mhvwZzL4rzX+Nxej/Hg5IhlBD8xrIrWtqWg+7cA9OFC/xHUuYixIC7wQgiVxshUoZmVOk7pEbfJvQ3ogEAHhSovhv2Q82CDtqIQ=,iv:rnQXOoQH9x/oh6/qvxdKvFBrn9fPLWVkIotDzeuf+Fc=,tag:M0dRaAqO8r9/d3rxDHzS9w==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.0 diff --git a/deployments/nature/secrets/staging.yaml b/deployments/nature/secrets/staging.yaml new file mode 100644 index 000000000..f50b0b804 --- /dev/null +++ b/deployments/nature/secrets/staging.yaml @@ -0,0 +1,21 @@ +jupyterhub: + hub: + config: + CanvasOAuthenticator: + client_id: ENC[AES256_GCM,data:G0+pIvyOsh+Zj4ZxddKbON4=,iv:iX8jMrXcJvjBi0GKRtuN886bXWFwBfCsaUoH+HRMn6U=,tag:BsyGAveKrd8TvWMuUlw3kw==,type:str] + client_secret: ENC[AES256_GCM,data:6mR3zs4jvkVuVwjut56tuW3HIOHcYWRSIJJZiOeC+H82tqgZeuVEe7+/zMxq6J0ba5RMas9npep4svuL+TppcA==,iv:K6/MFusdxnLkrEA5LaXLO8mFm1Xa4U2OmxUBOLqdpJk=,tag:xweE4+/ZCIcXj9CtbFpUSw==,type:str] + oauth_callback_url: ENC[AES256_GCM,data:WCMlCx1a3JLn7HvmOKSkQUoZwKvnJtvn31srm2/+uh3Bmj15aH9575l5ZjL7ZB9ncAiYQeZtFdjODMBAz9A=,iv:I04Gv5SLv4Z0lG2PfbXjNJ7m36wRMkDhgvoRiGmGewM=,tag:jLrSQif6oidL5nCKwGyNwA==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops + created_at: "2021-05-05T10:57:58Z" + enc: CiQA67O9ALEiz+lgnWQQgjT08Fx2+SUNdWEA2MqdIoEl0Ett3zASSQDmhpq85T+08Rtt/sqeMktjA6t8rCVH8soCR/sNJwDHgXabOipn/od+64D/L+aggCaXqJ433twByk0+YUJAe5z733oW/3J53eU= + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-08-30T18:14:58Z" + mac: ENC[AES256_GCM,data:MzwYjNtvSq0Xpab4J4Jv1ZzrODG+gIFJqfE85lYuTG7Kb2sDqppz2XIDoVRvo0OB0wWbcujwvNEbtMwjccVPtAhKvE6D8GVsByQ/W+2KgML/ctZiggvmuMU3WSiZoZ5YB+LCEZWEjLJo9OkiSacL/es8XGFSVLBi0NF7HaldGMI=,iv:J6SxdxqzEsmPB36KCeKNXOIjcGVbaXeh5MDxKNHHDV8=,tag:+DhOfJofu6R23SZPVWXNqA==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.0 diff --git a/docs/admins/howto/new-hub.qmd b/docs/admins/howto/new-hub.qmd index 3554547fd..727362a24 100644 --- a/docs/admins/howto/new-hub.qmd +++ b/docs/admins/howto/new-hub.qmd @@ -142,7 +142,7 @@ You can run the following command in gcloud terminal to log in to the NFS utility VM: ```bash -gcloud compute ssh nfsserver-01 --zone=us-central1-b +gcloud compute ssh nfsserver-01 --zone=us-central1-b --tunnel-through-iap ``` Alternatively, launch console.cloud.google.com > Select *ucb-datahub-2018* as @@ -232,7 +232,7 @@ secrets for these are provided by the cookiecutter template, however the new hubs need to be added to the authorized callback list maintained in bcourses. -1. Use `sops` to edit `secrets/dev.yaml` and `secrets/prod.yaml`, replacing the +1. Use `sops` to edit `secrets/staging.yaml` and `secrets/prod.yaml`, replacing the cookiecutter hub_name. `cookiecutter` can't do this for you since the values are encrypted. @@ -242,6 +242,8 @@ bcourses. 3. Add `<hub_name>.datahub.berkeley.edu/hub/oauth_callback` to the production hub client (id 10720000000000472) +4. Copy gcr-key.json and gke-key.json from any other hub's secrets to the hub's secrets/ + Please reach out to Jonathan Felder to set this up, or <bcourseshelp@berkeley.edu> if he is not available. @@ -288,10 +290,15 @@ sections of the CircleCI configuration file: ``` Review hubploy.yaml file inside your project directory and update the -image name to the latest image. Something like this, +images section. Example from a11y hub: ``` yaml -image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image +images: + images: + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image + path: image/ + repo2docker: + base_image: docker.io/library/buildpack-deps:jammy ``` ### Add hub to the github labeler workflow diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index fb8d0ed88..7c1e77daf 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -168,6 +168,14 @@ nodePools: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 replicas: 1 + nature: + nodeSelector: + hub.jupyter.org/pool-name: nature-pool + resources: + requests: + # Some value slightly lower than allocatable RAM on the nodepool + memory: 60929654784 + replicas: 1 publichealth: nodeSelector: hub.jupyter.org/pool-name: publichealth-pool From 7cbda7a86ef64f5bec5d8e9b0a70c72edd544d4f Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 30 Aug 2024 12:42:28 -0700 Subject: [PATCH 123/190] Remove ai-suggested fragment. The syntax wasn't permitted, and this isn't a strong need. --- .github/workflows/quarto-docs.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quarto-docs.yml b/.github/workflows/quarto-docs.yml index 8f903f041..9442844ea 100644 --- a/.github/workflows/quarto-docs.yml +++ b/.github/workflows/quarto-docs.yml @@ -11,9 +11,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - # Don't run in forks, unless the forker sets a secret variable. I chose to - # use a secret rather than a var because secrets are not copied to forks. - if: ${{ github.event.repository.fork == false || secrets.ALLOW_FORK_DEPLOY == 'true' }} + # Don't run in forks. + if: ${{ github.event.repository.fork == false }} steps: - name: Check out repository uses: actions/checkout@v4 From 3ba28cc018ed5a12989dd29b262b904cf93d9a08 Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Fri, 30 Aug 2024 15:17:57 -0700 Subject: [PATCH 124/190] [DH-361] adding gh-scoped-creds for nature hub --- deployments/nature/config/common.yaml | 2 ++ deployments/nature/image/environment.yml | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deployments/nature/config/common.yaml b/deployments/nature/config/common.yaml index 4ffa072de..a92e96c05 100644 --- a/deployments/nature/config/common.yaml +++ b/deployments/nature/config/common.yaml @@ -43,6 +43,8 @@ jupyterhub: c.WebPDFExporter.embed_images = True extraEnv: # Unset NotebookApp from hub/values. Necessary for recent lab versions. + GH_SCOPED_CREDS_CLIENT_ID: Iv23liGBW8jtMBP0inyw + GH_SCOPED_CREDS_APP_URL: https://github.com/apps/uc-berkeley-nature-hub-git-access JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" nodeSelector: hub.jupyter.org/pool-name: nature-pool diff --git a/deployments/nature/image/environment.yml b/deployments/nature/image/environment.yml index 7947296da..7e657a279 100644 --- a/deployments/nature/image/environment.yml +++ b/deployments/nature/image/environment.yml @@ -6,26 +6,29 @@ channels: dependencies: # Items required for basic level functionality -- python==3.11.* - git==2.46.0 - jupyter-resource-usage=1.1.0 - jupyterhub==4.1.6 - jupyterlab==4.2.5 - jupyter_server==2.14.2 -- notebook==7.2.2 - nbgitpuller==1.2.1 +- notebook==7.2.2 +- python==3.11.* # vscode - code-server==4.23.1 - jupyter-vscode-proxy==0.6 # other packages -- seaborn==0.13.2 - altair==5.4.1 +- gh-scoped-creds==4.1 - ibis-framework[pandas,duckdb]==9.3.0 -- leafmap==0.36.10 - jupyterlab-myst==2.4.2 +- leafmap==0.36.10 +- seaborn==0.13.2 +- streamlit==1.38.0 +# pip installed packages, conda is prefered - pip==24.2 - pip: - nbconvert[webpdf]==7.16.4 From 663ae8eceed99f27886b65393ff80219ace95e23 Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Fri, 30 Aug 2024 16:33:35 -0700 Subject: [PATCH 125/190] [DH-361] enabling nonempty directory delete, show hidden files is already enabled --- deployments/nature/config/common.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deployments/nature/config/common.yaml b/deployments/nature/config/common.yaml index a92e96c05..09c0a3a1b 100644 --- a/deployments/nature/config/common.yaml +++ b/deployments/nature/config/common.yaml @@ -35,6 +35,11 @@ jupyterhub: - course::1524699::group::all-admins singleuser: extraFiles: + jupyter_server_config.json: + mountPath: /usr/local/etc/jupyter/jupyter_server_config.json + data: + FileContentsManager: + always_delete_dir: true remove-exporters: mountPath: /etc/jupyter/jupyter_notebook_config.py stringData: | From 794cc286e53e0b1e51b880bc6649ce742760b88d Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Fri, 30 Aug 2024 17:18:12 -0700 Subject: [PATCH 126/190] [DH-361] adding jupytext and jupyterlab-git --- deployments/nature/image/environment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deployments/nature/image/environment.yml b/deployments/nature/image/environment.yml index 7e657a279..9e5f85aa6 100644 --- a/deployments/nature/image/environment.yml +++ b/deployments/nature/image/environment.yml @@ -6,11 +6,14 @@ channels: dependencies: # Items required for basic level functionality +- gh-scoped-creds==4.1 - git==2.46.0 - jupyter-resource-usage=1.1.0 - jupyterhub==4.1.6 - jupyterlab==4.2.5 - jupyter_server==2.14.2 +- jupyterlab-git==0.50.1 +- jupytext==1.16.4 - nbgitpuller==1.2.1 - notebook==7.2.2 - python==3.11.* @@ -21,7 +24,6 @@ dependencies: # other packages - altair==5.4.1 -- gh-scoped-creds==4.1 - ibis-framework[pandas,duckdb]==9.3.0 - jupyterlab-myst==2.4.2 - leafmap==0.36.10 From 703b37fdd7bd483b68644249e21ffea483aa9e02 Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Tue, 3 Sep 2024 12:46:47 -0700 Subject: [PATCH 127/190] [DH-361] adding earthaccess --- deployments/nature/image/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/nature/image/environment.yml b/deployments/nature/image/environment.yml index 9e5f85aa6..edf33ca3b 100644 --- a/deployments/nature/image/environment.yml +++ b/deployments/nature/image/environment.yml @@ -24,6 +24,7 @@ dependencies: # other packages - altair==5.4.1 +- earthaccess==0.10.0 - ibis-framework[pandas,duckdb]==9.3.0 - jupyterlab-myst==2.4.2 - leafmap==0.36.10 From 5bf11d5733ac935f54b6808f6f8190da30118033 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Tue, 3 Sep 2024 13:47:19 -0700 Subject: [PATCH 128/190] Split off stat20 user image. --- .circleci/config.yml | 25 ----- deployments/stat20/hubploy.yaml | 7 +- deployments/stat20/image/Dockerfile | 92 ------------------- deployments/stat20/image/README.md | 5 + deployments/stat20/image/class-libs.R | 20 ---- deployments/stat20/image/environment.yml | 25 ----- deployments/stat20/image/file-locks | 13 --- .../stat20/image/infra-requirements.txt | 29 ------ .../stat20/image/install-mambaforge.bash | 42 --------- .../image/r-packages/2023-fall-stat-20.r | 50 ---------- deployments/stat20/image/rstudio-prefs.json | 3 - 11 files changed, 6 insertions(+), 305 deletions(-) delete mode 100644 deployments/stat20/image/Dockerfile create mode 100644 deployments/stat20/image/README.md delete mode 100644 deployments/stat20/image/class-libs.R delete mode 100644 deployments/stat20/image/environment.yml delete mode 100644 deployments/stat20/image/file-locks delete mode 100644 deployments/stat20/image/infra-requirements.txt delete mode 100755 deployments/stat20/image/install-mambaforge.bash delete mode 100644 deployments/stat20/image/r-packages/2023-fall-stat-20.r delete mode 100644 deployments/stat20/image/rstudio-prefs.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fa02f25d..396b89b7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -319,12 +319,6 @@ jobs: hubploy deploy --timeout 30m shiny hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy stat20 - command: | - hubploy deploy --timeout 30m stat20 hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy ugr01 command: | @@ -505,15 +499,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: stat20 - name: stat20 image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: ugr01 name: ugr01 image build @@ -687,15 +672,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: stat20 - name: stat20 image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: ugr01 name: ugr01 image build @@ -728,7 +704,6 @@ workflows: - nature image build - publichealth image build - shiny image build - - stat20 image build filters: branches: diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index dd982a4a7..518b3560a 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,10 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/stat20/image/Dockerfile b/deployments/stat20/image/Dockerfile deleted file mode 100644 index e83591c4a..000000000 --- a/deployments/stat20/image/Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -FROM rocker/geospatial:4.4.1 -# https://github.com/rocker-org/rocker-versioned2/wiki/geospatial_e06f866673fa - -ENV NB_USER rstudio -ENV NB_UID 1000 -ENV CONDA_DIR /srv/conda - -# Set ENV for all programs... -ENV PATH ${CONDA_DIR}/bin:$PATH - -# Pick up rocker's default TZ -ENV TZ=Etc/UTC - -# And set ENV for R! It doesn't read from the environment... -RUN echo "TZ=${TZ}" >> /usr/local/lib/R/etc/Renviron.site -RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site - -# Add PATH to /etc/profile so it gets picked up by the terminal -RUN echo "PATH=${PATH}" >> /etc/profile -RUN echo "export PATH" >> /etc/profile - -ENV HOME /home/${NB_USER} - -WORKDIR ${HOME} - -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -# tini is necessary because it is our ENTRYPOINT below. -RUN apt-get update && \ - apt-get -qq install \ - less \ - tini \ - fonts-symbola \ - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-plain-generic \ - > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# While quarto is included with rocker/verse, we sometimes need different -# versions than the default. For example a newer version might fix bugs. -ENV _QUARTO_VERSION=1.4.549 -RUN curl -L -o /tmp/quarto.deb https://github.com/quarto-dev/quarto-cli/releases/download/v${_QUARTO_VERSION}/quarto-${_QUARTO_VERSION}-linux-amd64.deb -RUN apt-get update > /dev/null && \ - apt-get install /tmp/quarto.deb > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - rm -f /tmp/quarto.deb - -ENV SHINY_SERVER_URL https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-1.5.21.1012-amd64.deb -RUN curl --silent --location --fail ${SHINY_SERVER_URL} > /tmp/shiny-server.deb && \ - apt install --no-install-recommends --yes /tmp/shiny-server.deb && \ - rm /tmp/shiny-server.deb - -# google-chrome is for pagedown; chromium doesn't work nicely with it (snap?) -RUN wget --quiet -O /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - apt-get update > /dev/null && \ - apt-get -qq install /tmp/chrome.deb > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - rm -f /tmp/chrome.deb - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/environment.yml -COPY infra-requirements.txt /tmp/infra-requirements.txt -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -COPY class-libs.R /tmp/class-libs.R - -COPY r-packages/2023-fall-stat-20.r /tmp/r-packages/ -RUN r /tmp/r-packages/2023-fall-stat-20.r - -# Configure locking behavior -COPY file-locks /etc/rstudio/file-locks - -# Disable visual markdown editing by default -COPY rstudio-prefs.json /etc/rstudio/rstudio-prefs.json - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/stat20/image/README.md b/deployments/stat20/image/README.md new file mode 100644 index 000000000..1c5508f86 --- /dev/null +++ b/deployments/stat20/image/README.md @@ -0,0 +1,5 @@ +# Stat20 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/stat20-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat20-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/stat20/image/class-libs.R b/deployments/stat20/image/class-libs.R deleted file mode 100644 index 1c9343537..000000000 --- a/deployments/stat20/image/class-libs.R +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env Rscript - -class_libs_install_version <- function(class_name, class_libs) { - print(paste("Installing packages for", class_name)) - for (i in seq(1, length(class_libs), 2)) { - installed_packages <- rownames(installed.packages()) - package_name = class_libs[i] - version = class_libs[i+1] - # Only install packages if they haven't already been installed! - # devtools doesn't do that by default - if (!package_name %in% installed_packages) { - print(paste("Installing", package_name, version)) - devtools::install_version(package_name, version, quiet=TRUE) - } else { - # FIXME: This ignores version incompatibilities :'( - print(paste("Not installing", package_name, " as it is already installed")) - } - } - print(paste("Done installing packages for", class_name)) -} diff --git a/deployments/stat20/image/environment.yml b/deployments/stat20/image/environment.yml deleted file mode 100644 index f69fba4a5..000000000 --- a/deployments/stat20/image/environment.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: stat20 - -channels: -- conda-forge - -dependencies: -- syncthing==1.22.2 -- jupyter-server-proxy==4.2.0 -- jupyter-rsession-proxy==2.2.0 -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* -# for usage analysis dashboard (to be removed during sp 24 maintenance window) -- pandas==2.2.1 -- matplotlib==3.8.3 -- pip: - - -r /tmp/infra-requirements.txt - # For push authentication to GitHub - - gh-scoped-creds==4.1 - - jupyter-shiny-proxy==1.1 - # for notebook exporting - - nbconvert==7.6.0 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - - shiny==0.8.1 # for usage analysis dashboard (to be removed during sp 24 maintenance window) - diff --git a/deployments/stat20/image/file-locks b/deployments/stat20/image/file-locks deleted file mode 100644 index 7b1a3fcf4..000000000 --- a/deployments/stat20/image/file-locks +++ /dev/null @@ -1,13 +0,0 @@ -# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking - -# rocker sets this to advisory, but this might be causing NFS issues. -# lets set it to the default (default: linkbased) -lock-type=linkbased - -# we'll also reduce the frequency by 1/3 -refresh-rate=60 -timeout-interval=90 - -# log attempts -# enable-logging=1 -# log-file=/tmp/rstudio-locking.log diff --git a/deployments/stat20/image/infra-requirements.txt b/deployments/stat20/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/stat20/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/stat20/image/install-mambaforge.bash b/deployments/stat20/image/install-mambaforge.bash deleted file mode 100755 index 812319edc..000000000 --- a/deployments/stat20/image/install-mambaforge.bash +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION=22.9.0-2 - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -# make sure we don't do anything funky with user's $HOME -# since this is run as root -unset HOME - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/deployments/stat20/image/r-packages/2023-fall-stat-20.r b/deployments/stat20/image/r-packages/2023-fall-stat-20.r deleted file mode 100644 index 83039e84d..000000000 --- a/deployments/stat20/image/r-packages/2023-fall-stat-20.r +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env Rscript - -source("/tmp/class-libs.R") - -class_name = "2023 Fall Stat 20" - -class_libs = c( - "fivethirtyeight", "0.6.2", - "gapminder", "0.3.0", - "janitor", "2.2.0", - "openintro", "2.4.0", - "pagedown", "0.16", - "palmerpenguins", "0.1.1", - "patchwork", "1.1.2", - "showtext", "0.9-4", - "swirl", "2.4.5", - "tidycensus", "1.1", - "tidymodels", "0.1.4", - "tigris", "1.5", - "unvotes", "0.3.0", - "xaringanthemer", "0.4.1", - "rmarkdown", "2.22", - "plotly", "4.10.1", - "reshape2", "1.4.4", - "kableExtra", "1.3.4", - "infer", "1.0.4", - "countdown", "0.4.0", - "ggrepel", "0.9.3", - "ggthemes", "4.2.4", - "latex2exp", "0.9.6", - "markdown", "1.7", - "downlit", "0.4.3", - "xml2", "1.3.4", - "gt", "0.9.0", - "quarto", "1.2", - "fs", "1.6.3", - "rsample", "1.2.1" -) - -class_libs_install_version(class_name, class_libs) - -devtools::install_github("mdbeckman/dcData", ref="56888a6") -devtools::install_github("hadley/emo@3f03b11") -devtools::install_github("andrewpbray/boxofdata@8afd934") -devtools::install_github("tidymodels/infer@2806a69") -devtools::install_github("stat20/stat20data@11b4377") - -# file.symlink("/opt/shared/stat20/stat20data", "/usr/local/lib/R/site-library/stat20data") - -print(paste("Done installing packages for",class_name)) diff --git a/deployments/stat20/image/rstudio-prefs.json b/deployments/stat20/image/rstudio-prefs.json deleted file mode 100644 index 0647cffbf..000000000 --- a/deployments/stat20/image/rstudio-prefs.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "visual_markdown_editing_is_default": false -} From fef58698740b307c94fed85f74dc8dfc5c68d23b Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Tue, 3 Sep 2024 13:53:15 -0700 Subject: [PATCH 129/190] Add stat20 to workflow. --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 4ff7da1d5..969ae17a5 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -102,7 +102,7 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub staging - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -196,4 +196,4 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub prod - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20) From 0ffe11bff2cd1151119811d346c5d0aedad1d917 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Tue, 3 Sep 2024 20:55:27 +0000 Subject: [PATCH 130/190] update stat20 image tag to d3fd1f663330: deployments/stat20/hubploy.yaml --- deployments/stat20/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index 518b3560a..9fd637c16 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:PLACEHOLDER + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:d3fd1f663330 cluster: provider: gcloud From 3db70fd320ca72fd77f9809a718c43625bfa58f2 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Tue, 3 Sep 2024 23:14:57 +0000 Subject: [PATCH 131/190] update stat20 image tag to 5eed7cbfc7ff: deployments/stat20/hubploy.yaml --- deployments/stat20/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index 9fd637c16..e9a4038e0 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:d3fd1f663330 + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:5eed7cbfc7ff cluster: provider: gcloud From 3844877d6fba966af3233cc076a15288a9d74a08 Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Wed, 4 Sep 2024 11:12:36 -0700 Subject: [PATCH 132/190] moving nature to own repo --- .circleci/config.yml | 25 ------------ deployments/nature/hubploy.yaml | 10 +---- deployments/nature/image/README.md | 5 +++ deployments/nature/image/apt.txt | 52 ------------------------ deployments/nature/image/environment.yml | 37 ----------------- deployments/nature/image/postBuild | 6 --- deployments/nature/image/start | 5 --- 7 files changed, 6 insertions(+), 134 deletions(-) create mode 100644 deployments/nature/image/README.md delete mode 100644 deployments/nature/image/apt.txt delete mode 100644 deployments/nature/image/environment.yml delete mode 100644 deployments/nature/image/postBuild delete mode 100644 deployments/nature/image/start diff --git a/.circleci/config.yml b/.circleci/config.yml index 396b89b7b..1fbf679e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -289,12 +289,6 @@ jobs: # hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} # no_output_timeout: 30m - - run: - name: Deploy nature - command: | - hubploy deploy --timeout 30m nature hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy prob140 command: | @@ -472,15 +466,6 @@ workflows: # ignore: # - staging # - prod - - hubploy/build-image: - deployment: nature - name: nature image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -645,15 +630,6 @@ workflows: # branches: # only: # - staging - - hubploy/build-image: - deployment: nature - name: nature image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -701,7 +677,6 @@ workflows: - ischool image build - julia hub image build # - logodev image build - - nature image build - publichealth image build - shiny image build diff --git a/deployments/nature/hubploy.yaml b/deployments/nature/hubploy.yaml index 1ae5a7106..8d113552d 100644 --- a/deployments/nature/hubploy.yaml +++ b/deployments/nature/hubploy.yaml @@ -1,14 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:anything cluster: provider: gcloud diff --git a/deployments/nature/image/README.md b/deployments/nature/image/README.md new file mode 100644 index 000000000..ce36af3e8 --- /dev/null +++ b/deployments/nature/image/README.md @@ -0,0 +1,5 @@ +# nature Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/nature-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/nature-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/nature/image/apt.txt b/deployments/nature/image/apt.txt deleted file mode 100644 index a990f28e0..000000000 --- a/deployments/nature/image/apt.txt +++ /dev/null @@ -1,52 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Download tools -curl -wget -vim - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -rsync - -# playwright deps https://jira-secure.berkeley.edu/browse/DH-325 -libnss3 -libnspr4 -libdbus-1-3 -libatk1.0-0 -libatk-bridge2.0-0 -libcups2 -libdrm2 -libxkbcommon0 -libatspi2.0-0 -libxcomposite1 -libxdamage1 -libxfixes3 -libxrandr2 -libgbm1 -libasound2 diff --git a/deployments/nature/image/environment.yml b/deployments/nature/image/environment.yml deleted file mode 100644 index edf33ca3b..000000000 --- a/deployments/nature/image/environment.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: nature - -channels: -- conda-forge - -dependencies: - -# Items required for basic level functionality -- gh-scoped-creds==4.1 -- git==2.46.0 -- jupyter-resource-usage=1.1.0 -- jupyterhub==4.1.6 -- jupyterlab==4.2.5 -- jupyter_server==2.14.2 -- jupyterlab-git==0.50.1 -- jupytext==1.16.4 -- nbgitpuller==1.2.1 -- notebook==7.2.2 -- python==3.11.* - -# vscode -- code-server==4.23.1 -- jupyter-vscode-proxy==0.6 - -# other packages -- altair==5.4.1 -- earthaccess==0.10.0 -- ibis-framework[pandas,duckdb]==9.3.0 -- jupyterlab-myst==2.4.2 -- leafmap==0.36.10 -- seaborn==0.13.2 -- streamlit==1.38.0 - -# pip installed packages, conda is prefered -- pip==24.2 -- pip: - - nbconvert[webpdf]==7.16.4 diff --git a/deployments/nature/image/postBuild b/deployments/nature/image/postBuild deleted file mode 100644 index f8d71f964..000000000 --- a/deployments/nature/image/postBuild +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -playwright install chromium diff --git a/deployments/nature/image/start b/deployments/nature/image/start deleted file mode 100644 index c3a978b7f..000000000 --- a/deployments/nature/image/start +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# See https://jira-secure.berkeley.edu/browse/DH-305 -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -exec "$@" From affcc5ebb96dd7e8f4ccb6a81ebf3c8f2bfee020 Mon Sep 17 00:00:00 2001 From: Jonathan Felder <felder@berkeley.edu> Date: Wed, 4 Sep 2024 11:16:32 -0700 Subject: [PATCH 133/190] enabling nature hub deployment --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 969ae17a5..39f0424a2 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -102,7 +102,7 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub staging - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -196,4 +196,4 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub prod - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature) From 03156ab114c0de86e0b0ab420d1f44619d5c3340 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:29:25 +0000 Subject: [PATCH 134/190] update nature image tag to 6e285f9f5ab2: deployments/nature/hubploy.yaml --- deployments/nature/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/nature/hubploy.yaml b/deployments/nature/hubploy.yaml index 8d113552d..b2d9c6d35 100644 --- a/deployments/nature/hubploy.yaml +++ b/deployments/nature/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:anything + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:6e285f9f5ab2 cluster: provider: gcloud From ad954865ea3b7f36d00305730afe298ce6b634a7 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:08:26 +0000 Subject: [PATCH 135/190] update nature image tag to 9ad22965ad22: deployments/nature/hubploy.yaml --- deployments/nature/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/nature/hubploy.yaml b/deployments/nature/hubploy.yaml index b2d9c6d35..61a4e283e 100644 --- a/deployments/nature/hubploy.yaml +++ b/deployments/nature/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:6e285f9f5ab2 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:9ad22965ad22 cluster: provider: gcloud From e9e30bdaa57ccc09c5d7325e8e81db18119a85b5 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 4 Sep 2024 13:07:01 -0700 Subject: [PATCH 136/190] updates for a11y to move to new repo --- .circleci/config.yml | 25 ---------- deployments/a11y/hubploy.yaml | 10 +--- deployments/a11y/image/README.md | 5 ++ deployments/a11y/image/apt.txt | 40 ---------------- deployments/a11y/image/environment.yml | 47 ------------------- deployments/a11y/image/infra-requirements.txt | 29 ------------ deployments/a11y/image/postBuild | 7 --- 7 files changed, 6 insertions(+), 157 deletions(-) create mode 100644 deployments/a11y/image/README.md delete mode 100644 deployments/a11y/image/apt.txt delete mode 100644 deployments/a11y/image/environment.yml delete mode 100644 deployments/a11y/image/infra-requirements.txt delete mode 100644 deployments/a11y/image/postBuild diff --git a/.circleci/config.yml b/.circleci/config.yml index 396b89b7b..4b6fa12dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -181,12 +181,6 @@ jobs: command: | gcloud components install gke-gcloud-auth-plugin - - run: - name: Deploy a11y - command: | - hubploy deploy --timeout 30m a11y hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy astro command: | @@ -335,15 +329,6 @@ workflows: version: 2 test-build-images: jobs: - - hubploy/build-image: - deployment: a11y - name: a11y image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: astro name: astro image build @@ -511,15 +496,6 @@ workflows: deploy: jobs: - - hubploy/build-image: - deployment: a11y - name: a11y image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: astro name: astro image build @@ -686,7 +662,6 @@ workflows: # CI level, we also make prod deploys go faster! - deploy: requires: - - a11y image build - astro image build - biology image build - cee image build diff --git a/deployments/a11y/hubploy.yaml b/deployments/a11y/hubploy.yaml index 1815bec43..ce4a79cd1 100644 --- a/deployments/a11y/hubploy.yaml +++ b/deployments/a11y/hubploy.yaml @@ -1,14 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:XXXX cluster: provider: gcloud diff --git a/deployments/a11y/image/README.md b/deployments/a11y/image/README.md new file mode 100644 index 000000000..1c5508f86 --- /dev/null +++ b/deployments/a11y/image/README.md @@ -0,0 +1,5 @@ +# Stat20 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/stat20-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat20-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/a11y/image/apt.txt b/deployments/a11y/image/apt.txt deleted file mode 100644 index 24dad6eac..000000000 --- a/deployments/a11y/image/apt.txt +++ /dev/null @@ -1,40 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Download tools -curl -wget -vim - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -rsync - -# Add chromium to the a11y image for testing with pa11y -chromium-browser - diff --git a/deployments/a11y/image/environment.yml b/deployments/a11y/image/environment.yml deleted file mode 100644 index 8723ab955..000000000 --- a/deployments/a11y/image/environment.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: a11y - -channels: -- conda-forge - -dependencies: -- python==3.11.* -- git==2.40.0 -- coverage==7.2.2 -- jupyter-archive==3.4.0 -- jupyter_server==2.7.0 -- jupyter-book==0.15.1 -- jupyter-resource-usage==0.7.1 -- jupyterhub==4.1.6 -- matplotlib==3.7.1 -- nbgitpuller==1.2.1 -- nbclassic==1.0.0 -- nbconvert==7.14.2 -- numpy==1.24.2 -- plotly==5.13.1 -- pyopenssl==23.1.0 -- requests==2.32.3 -- urllib3==2.2.2 -- websockify==0.11.0 -- pip==23.0.1 -- jupyterthemes==0.20.0 -- jupyterlab-accessible-themes==0.1.1 -- jupyterlab_pygments==0.2.2 -- pip: - #- -r infra-requirements.txt - # Upgrade separate from what everyone else uses for now - # https://github.com/berkeley-dsep-infra/datahub/issues/3693 - - notebook==7.1.0a1 - - jupyterlab==4.1.0b0 - - nbconvert-a11y==2023.12.6 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - - jupyterlab-a11y-checker==0.1.2 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 - # ### - # The items below are from infra-requirements, however lab conflicts with the - # alpha notebook. - # We disable infre-requirements.txt above and manually enable some of what it - # provides below. - # ### - # Matches version in images/hub/Dockerfile diff --git a/deployments/a11y/image/infra-requirements.txt b/deployments/a11y/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/a11y/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/a11y/image/postBuild b/deployments/a11y/image/postBuild deleted file mode 100644 index ec5f61cb1..000000000 --- a/deployments/a11y/image/postBuild +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -eux - -# install pa11y tools for DH-169 - -npm install -g pa11y-ci@3.1.0 -npm install -g pa11y@6.2.3 \ No newline at end of file From 05bff7b2362f7b84551c39e408ca6d7f910ede8e Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 4 Sep 2024 13:08:13 -0700 Subject: [PATCH 137/190] update readme verbiage --- deployments/a11y/image/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/a11y/image/README.md b/deployments/a11y/image/README.md index 1c5508f86..9ac9c2c81 100644 --- a/deployments/a11y/image/README.md +++ b/deployments/a11y/image/README.md @@ -1,5 +1,5 @@ -# Stat20 Image +# a11y Image -This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/stat20-user-image). +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/a11y-user-image). -Please see [the contribution guide](https://github.com/berkeley-dsep-infra/stat20-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/a11y-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. From b10684b259ae92352a4c56178c3551b0dddb9885 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 4 Sep 2024 13:10:51 -0700 Subject: [PATCH 138/190] enable deployment of a11y --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 39f0424a2..886db8078 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -102,7 +102,7 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub staging - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -196,4 +196,4 @@ jobs: while read deployment; do echo "Pretending to deploy base hub image to ${deployment} :P" hubploy --verbose deploy --timeout 30m ${deployment} hub prod - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y) From 2730b977bea9b0c415d50f08e40cdd4d4444cfa6 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 4 Sep 2024 13:12:01 -0700 Subject: [PATCH 139/190] twiddle the workflow --- .github/workflows/deploy-hubs.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 886db8078..436bc3250 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -100,8 +100,9 @@ jobs: if: ${{ env.DEPLOY }} run: | while read deployment; do - echo "Pretending to deploy base hub image to ${deployment} :P" + echo "Depoying base hub image to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging + echo done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y) deploy-hubs-to-prod: @@ -194,6 +195,7 @@ jobs: if: ${{ env.DEPLOY }} run: | while read deployment; do - echo "Pretending to deploy base hub image to ${deployment} :P" + echo "Deploying base hub image to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod + echo done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y) From 23909df44d7faec1fa75f101c93d8f5520c2b4b1 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Wed, 4 Sep 2024 20:19:02 +0000 Subject: [PATCH 140/190] update a11y image tag to e33a1decb7d0: deployments/a11y/hubploy.yaml --- deployments/a11y/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/a11y/hubploy.yaml b/deployments/a11y/hubploy.yaml index ce4a79cd1..1c715ee01 100644 --- a/deployments/a11y/hubploy.yaml +++ b/deployments/a11y/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:XXXX + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:e33a1decb7d0 cluster: provider: gcloud From fc3916f6c6be9854e1742b4ee34b37af538beef2 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Wed, 4 Sep 2024 20:35:12 +0000 Subject: [PATCH 141/190] update a11y image tag to 1967ba9bc53f: deployments/a11y/hubploy.yaml --- deployments/a11y/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/a11y/hubploy.yaml b/deployments/a11y/hubploy.yaml index 1c715ee01..995406af6 100644 --- a/deployments/a11y/hubploy.yaml +++ b/deployments/a11y/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:e33a1decb7d0 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:1967ba9bc53f cluster: provider: gcloud From d868115103efde65e710651ce89264c116c907b3 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Wed, 4 Sep 2024 21:54:58 +0000 Subject: [PATCH 142/190] update a11y image tag to 7c4cd8df980c: deployments/a11y/hubploy.yaml --- deployments/a11y/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/a11y/hubploy.yaml b/deployments/a11y/hubploy.yaml index 995406af6..429c55cde 100644 --- a/deployments/a11y/hubploy.yaml +++ b/deployments/a11y/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:1967ba9bc53f + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:7c4cd8df980c cluster: provider: gcloud From efd74b6c36322fd861c31fe70ae411298c263e0c Mon Sep 17 00:00:00 2001 From: Balaji Alwar <balajialwar@berkeley.edu> Date: Wed, 4 Sep 2024 17:55:14 -0700 Subject: [PATCH 143/190] Adding more tools for dashboaring in python in dashboard image --- deployments/dev/images/secondary/environment.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml index 2da3a6263..97971b0c4 100644 --- a/deployments/dev/images/secondary/environment.yml +++ b/deployments/dev/images/secondary/environment.yml @@ -39,3 +39,10 @@ dependencies: # Install voila and voici for generating voila dashboards - voila==0.5.7 - voici==0.6.1 +# Adding more data tools to make this image comprehensive +- dash==2.17.1 +- bokeh==3.4.0 +- panel==1.4.5 +- datashader==0.16.3 +- streamlit=1.38.0 +#- jupyter-containds=0.2.2 From 2b37362aa8dd7ba84ec49bbcab5eb81f609c292c Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 09:33:27 -0700 Subject: [PATCH 144/190] adding placeholder for image tag, remove path to image stuff --- .../template/{{cookiecutter.hub_name}}/hubploy.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/deployments/template/{{cookiecutter.hub_name}}/hubploy.yaml b/deployments/template/{{cookiecutter.hub_name}}/hubploy.yaml index ad392d602..2209f7e0e 100644 --- a/deployments/template/{{cookiecutter.hub_name}}/hubploy.yaml +++ b/deployments/template/{{cookiecutter.hub_name}}/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/primary-user-image - path: ../datahub/images/default - registry: - provider: gcloud - gcloud: - project: {{cookiecutter.project_name}} - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/primary-user-image:PLACEHOLDER cluster: provider: gcloud From d14bcc4fff51788cc8d085bff0da09d86a72e3a8 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 09:50:50 -0700 Subject: [PATCH 145/190] moving ugr01 to its own image repo --- .circleci/config.yml | 23 ----- deployments/ugr01/hubploy.yaml | 10 +- deployments/ugr01/image/README.md | 5 + deployments/ugr01/image/apt.txt | 97 ------------------- deployments/ugr01/image/environment.yml | 63 ------------ .../ugr01/image/infra-requirements.txt | 29 ------ deployments/ugr01/image/postBuild | 5 - deployments/ugr01/image/start | 5 - 8 files changed, 6 insertions(+), 231 deletions(-) create mode 100644 deployments/ugr01/image/README.md delete mode 100644 deployments/ugr01/image/apt.txt delete mode 100644 deployments/ugr01/image/environment.yml delete mode 100644 deployments/ugr01/image/infra-requirements.txt delete mode 100644 deployments/ugr01/image/postBuild delete mode 100644 deployments/ugr01/image/start diff --git a/.circleci/config.yml b/.circleci/config.yml index 78b4ba048..d8ffe9839 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -307,12 +307,6 @@ jobs: hubploy deploy --timeout 30m shiny hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy ugr01 - command: | - hubploy deploy --timeout 30m ugr01 hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy workshop command: | @@ -469,15 +463,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: ugr01 - name: ugr01 image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod deploy: jobs: @@ -624,14 +609,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: ugr01 - name: ugr01 image build - push: true - filters: - branches: - only: - - staging # Build images only during the staging deploy. All merges # to prod need to go via staging, so prod should *never* # use images not built for staging. By enforcing this at the diff --git a/deployments/ugr01/hubploy.yaml b/deployments/ugr01/hubploy.yaml index 2ff8daaea..7d961ecfb 100644 --- a/deployments/ugr01/hubploy.yaml +++ b/deployments/ugr01/hubploy.yaml @@ -1,14 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ugr01-user-image - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ugr01-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/ugr01/image/README.md b/deployments/ugr01/image/README.md new file mode 100644 index 000000000..2ee2276b9 --- /dev/null +++ b/deployments/ugr01/image/README.md @@ -0,0 +1,5 @@ +# Ugr01 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/ugr01-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/ugr01-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/ugr01/image/apt.txt b/deployments/ugr01/image/apt.txt deleted file mode 100644 index 10e53b036..000000000 --- a/deployments/ugr01/image/apt.txt +++ /dev/null @@ -1,97 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download tools -curl -wget - -# Core text editors on a *nix box: vim -vim - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# powerful terminal-based file manager, better than the one in JLab -mc - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -## This section adds tools for desktop environment usage -dbus-x11 -xorg -xubuntu-icon-theme -xfce4 -xfce4-goodies -xclip -xsel -firefox -chromium-browser - -# GUI text editors -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip - -# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 -libnss3 -libnspr4 diff --git a/deployments/ugr01/image/environment.yml b/deployments/ugr01/image/environment.yml deleted file mode 100644 index 058130f05..000000000 --- a/deployments/ugr01/image/environment.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: ugr01-FA24 - -channels: -- conda-forge -- pytorch - -dependencies: -- python==3.11.* -- git==2.39.1 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 -- folium==0.14.0 -- h5netcdf==1.0.2 -- ipywidgets==8.0.7 -- jupysql==0.8.0 -- jupyter-archive==3.4.0 -- matplotlib==3.7.1 -- mdit-py-plugins==0.4.0 -- numpy==1.24.2 -- pandas==2.0.2 -- plotly==5.13.1 -- requests==2.28.2 -- scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 -- seaborn==0.12.2 -- statsmodels==0.14.0 -- tensorflow-cpu==2.12.1 -- sqlalchemy==2.0.16 -- mlxtend==0.23.0 -# Spring 2024 data 100 -- pytorch==2.1.2 -- cpuonly==2.0 -- transformers==4.37.1 -# Spring 2024 table demos -- lxml==5.1.0 -# Spring 2024 Econ 148 Packages -- geopandas==0.14.2 -- geopy==2.4.1 -- lifelines==0.27.8 -- pycountry==22.3.5 -- pip -- pip: - # - -r infra-requirements.txt - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.6 - - nbconvert[webpdf] - # - pyppeteer==2.0.0 - - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 - - otter-grader==5.4.0 - - duckdb==0.10.1 - - duckdb_engine==0.11.2 diff --git a/deployments/ugr01/image/infra-requirements.txt b/deployments/ugr01/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/ugr01/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/ugr01/image/postBuild b/deployments/ugr01/image/postBuild deleted file mode 100644 index 1b6bcc9d1..000000000 --- a/deployments/ugr01/image/postBuild +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -playwright install chromium diff --git a/deployments/ugr01/image/start b/deployments/ugr01/image/start deleted file mode 100644 index c3a978b7f..000000000 --- a/deployments/ugr01/image/start +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# See https://jira-secure.berkeley.edu/browse/DH-305 -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -exec "$@" From 12c105c1e7bfd8491ac3fe114366b86533174a78 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 09:52:55 -0700 Subject: [PATCH 146/190] enabling deployability of ugr01 --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 436bc3250..698466d51 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Depoying base hub image to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying base hub image to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01) From ff3a6119940606d96f51f0f4890b8372e87187f1 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:02:25 +0000 Subject: [PATCH 147/190] update ugr01 image tag to d093fb974521: deployments/ugr01/hubploy.yaml --- deployments/ugr01/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/ugr01/hubploy.yaml b/deployments/ugr01/hubploy.yaml index 7d961ecfb..308ce3442 100644 --- a/deployments/ugr01/hubploy.yaml +++ b/deployments/ugr01/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ugr01-user-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ugr01-user-image:d093fb974521 cluster: provider: gcloud From e7f7cb8327e6e04e6a1c91b5d3a34ae6d62a1e21 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 12:03:47 -0700 Subject: [PATCH 148/190] moving data101 to its own repo --- .circleci/config.yml | 25 ---- deployments/data101/hubploy.yaml | 9 -- deployments/data101/image/README.md | 5 + deployments/data101/image/apt.txt | 100 ---------------- deployments/data101/image/environment.yml | 111 ------------------ .../data101/image/infra-requirements.txt | 29 ----- deployments/data101/image/postBuild | 19 --- deployments/data101/image/start | 5 - 8 files changed, 5 insertions(+), 298 deletions(-) create mode 100644 deployments/data101/image/README.md delete mode 100644 deployments/data101/image/apt.txt delete mode 100644 deployments/data101/image/environment.yml delete mode 100644 deployments/data101/image/infra-requirements.txt delete mode 100644 deployments/data101/image/postBuild delete mode 100644 deployments/data101/image/start diff --git a/.circleci/config.yml b/.circleci/config.yml index d8ffe9839..bcfc1b225 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -211,12 +211,6 @@ jobs: hubploy deploy --timeout 30m data100 hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy data101 - command: | - hubploy deploy --timeout 30m data101 hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy data102 command: | @@ -362,15 +356,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: data101 - name: data101 image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: data102 name: data102 image build @@ -510,15 +495,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: data101 - name: data101 image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: data102 name: data102 image build @@ -620,7 +596,6 @@ workflows: - cee image build - data8 image build - data100 image build - - data101 image build - data102 image build - datahub image build - dev image build diff --git a/deployments/data101/hubploy.yaml b/deployments/data101/hubploy.yaml index f117bbf87..962c7e362 100644 --- a/deployments/data101/hubploy.yaml +++ b/deployments/data101/hubploy.yaml @@ -1,15 +1,6 @@ images: images: - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy - - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json cluster: provider: gcloud diff --git a/deployments/data101/image/README.md b/deployments/data101/image/README.md new file mode 100644 index 000000000..533c7ba9a --- /dev/null +++ b/deployments/data101/image/README.md @@ -0,0 +1,5 @@ +# Data101 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/data101-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/data101-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/data101/image/apt.txt b/deployments/data101/image/apt.txt deleted file mode 100644 index 9b4955ded..000000000 --- a/deployments/data101/image/apt.txt +++ /dev/null @@ -1,100 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download tools -curl -wget - -# Core text editors on a *nix box: vim -vim - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# powerful terminal-based file manager, better than the one in JLab -mc - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -## This section adds tools for desktop environment usage -dbus-x11 -xorg -xubuntu-icon-theme -xfce4 -xfce4-goodies -xclip -xsel -firefox -chromium-browser - -# GUI text editors -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip - -# DH-93, DH94 -postgresql-client - -# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 -libnss3 -libnspr4 diff --git a/deployments/data101/image/environment.yml b/deployments/data101/image/environment.yml deleted file mode 100644 index 5c255e877..000000000 --- a/deployments/data101/image/environment.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: data101 - -channels: -- conda-forge - -dependencies: -- python==3.11.* -- syncthing==1.20.4 -- git==2.39.1 -- altair==5.0.1 -- beautifulsoup4==4.11.1 -- black==22.6.0 -- bokeh==2.4.3 -- bqplot==0.12.34 -- cartopy==0.21.0 -- coverage==7.2.2 -- cython==0.29.32 -- distributed==2023.6.0 -- fortran-magic==0.7 -- graphviz==8.0.3 -- python-graphviz==0.20.1 -- h5netcdf==1.0.2 -- h5py==3.7.0 -- hdf4==4.2.15 -- hdf5==1.12.2 -- intake==0.6.5 -- intake-esm==2021.8.17 -- intake-xarray==0.6.0 -- ipycanvas==0.12.1 -- ipydatagrid==1.1.12 -- ipympl==0.9.2 -- ipyparallel==8.4.1 -- jsonschema==4.17.3 -- jupyter-archive==3.4.0 -- jupyter-book==0.15.1 -- jupyter-resource-usage==1.0.0 -- jupyter-server-proxy==4.2.0 -- jupyter_bokeh -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab-geojson==3.2.0 -- jupyterlab-variableinspector==3.0.9 -- jupyterlab_pygments==0.2.2 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- matplotlib==3.7.1 -- matplotlib-inline==0.1.6 -- mock==4.0.3 -# https://jira-secure.berkeley.edu/browse/DH-165 -- mongo-tools==4.0.4 -- nbclassic==1.0.0 -- nbdime==3.1.1 -- nbgitpuller==1.2.1 -- networkx==2.8.6 -- notebook==7.0.7 -- numba==0.57.0 -- numpy==1.24.2 -- pandas==2.0.2 -- pandoc==2.12 -- pandocfilters==1.5.0 -- pep8==1.7.1 -- pgspecial==1.13.1 -- pillow==9.2.0 -- plotly==5.13.1 -- pooch==1.6.0 -- prettytable==3.4.1 -- pyarrow==9.0.0 -- pypdf2==2.10.4 -- pytables==3.7.0 -- pytest==7.1.2 -- pytest-cov==3.0.0 -- python-pdfkit==1.0.0 -- requests==2.28.2 -- scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 -- seaborn==0.12.2 -- sphinx-jupyterbook-latex==0.5.2 -- sqlparse==0.4.3 -- statsmodels==0.14.0 -- sympy==1.10.1 -- tornado==6.2.0 -- tqdm==4.64.0 -- xarray==2023.5.0 -- xlrd==2.0.1 -- micro==2.0.8 -- websockify==0.11.0 -- folium==0.14.0 -- sqlalchemy==2.0.16 -- pip -- pip: - # - -r infra-requirements.txt - - jupyter-desktop-server - - otter-grader==5.1.3 - - jupysql==0.10.0 - - geopandas==0.12.1 - - iwut==0.0.4 - - tensorflow-cpu==2.12.0 - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.6 - - nbconvert[webpdf]==7.16.4 - - pytest-notebook==0.8.1 - - pymongo==4.4.1 - - dbt-postgres==1.6.0 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 - - jupyterlab-a11y-checker==0.1.2 # Adding a11y-checker for Data 101 staff's exploration diff --git a/deployments/data101/image/infra-requirements.txt b/deployments/data101/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/data101/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/data101/image/postBuild b/deployments/data101/image/postBuild deleted file mode 100644 index 8b16a7df1..000000000 --- a/deployments/data101/image/postBuild +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -eux - -# install mongosh for DH-160 - -mkdir -p /tmp/mongosh -cd /tmp/mongosh -wget -q https://downloads.mongodb.com/compass/mongosh-2.0.2-linux-x64.tgz -tar xvfz mongosh-2.0.2-linux-x64.tgz -cp -v mongosh-2.0.2-linux-x64/bin/mongosh ${CONDA_DIR}/bin/ -cp -v mongosh-2.0.2-linux-x64/bin/mongosh_crypt_v1.so ${CONDA_DIR}/lib/ -chmod 775 ${CONDA_DIR}/bin/mongosh -chmod 775 ${CONDA_DIR}/lib/mongosh_crypt_v1.so -cd / -rm -rf /tmp/mongosh - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -playwright install chromium diff --git a/deployments/data101/image/start b/deployments/data101/image/start deleted file mode 100644 index c3a978b7f..000000000 --- a/deployments/data101/image/start +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# See https://jira-secure.berkeley.edu/browse/DH-305 -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -exec "$@" From 3d4ca9a707f33c84ed38d3ea56946e8142c38bdf Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 12:05:35 -0700 Subject: [PATCH 149/190] enable data101 deployment --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 698466d51..82d89c1c8 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Depoying base hub image to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying base hub image to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) From d833e287e5c53445f1242aa10333d7e2d5704a54 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 12:06:29 -0700 Subject: [PATCH 150/190] forgot the placeholder tag --- deployments/data101/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/data101/hubploy.yaml b/deployments/data101/hubploy.yaml index 962c7e362..cdd56a2cd 100644 --- a/deployments/data101/hubploy.yaml +++ b/deployments/data101/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:placeholder cluster: provider: gcloud From 2aca5992038e54d5dd87a14dc9b33535595fdaf3 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Thu, 5 Sep 2024 22:39:54 +0000 Subject: [PATCH 151/190] update data101 image tag to f702848059e1: deployments/data101/hubploy.yaml --- deployments/data101/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/data101/hubploy.yaml b/deployments/data101/hubploy.yaml index cdd56a2cd..c5ab243f7 100644 --- a/deployments/data101/hubploy.yaml +++ b/deployments/data101/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:placeholder + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:f702848059e1 cluster: provider: gcloud From 7e1ae46ac7dfeca130bc664da6acafa9a5ae3557 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Thu, 5 Sep 2024 15:49:35 -0700 Subject: [PATCH 152/190] fix typo and better wording --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 82d89c1c8..84cf3b9a2 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -100,7 +100,7 @@ jobs: if: ${{ env.DEPLOY }} run: | while read deployment; do - echo "Depoying base hub image to ${deployment}" + echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) @@ -195,7 +195,7 @@ jobs: if: ${{ env.DEPLOY }} run: | while read deployment; do - echo "Deploying base hub image to ${deployment}" + echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) From e33d98b4cc3141f9852763954f947055c8052bd0 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 02:08:32 +0000 Subject: [PATCH 153/190] update a11y image tag to 65e4cb99b4aa: deployments/a11y/hubploy.yaml --- deployments/a11y/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/a11y/hubploy.yaml b/deployments/a11y/hubploy.yaml index 429c55cde..45e8464e8 100644 --- a/deployments/a11y/hubploy.yaml +++ b/deployments/a11y/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:7c4cd8df980c + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/a11y-user-image:65e4cb99b4aa cluster: provider: gcloud From d815e612d8c4fce868f3bb4d22ba354aafaa3e77 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 02:13:30 +0000 Subject: [PATCH 154/190] update data101 image tag to 0e1d41cf9936: deployments/data101/hubploy.yaml --- deployments/data101/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/data101/hubploy.yaml b/deployments/data101/hubploy.yaml index c5ab243f7..cee47a052 100644 --- a/deployments/data101/hubploy.yaml +++ b/deployments/data101/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:f702848059e1 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:0e1d41cf9936 cluster: provider: gcloud From 54688b1ea40cd660326026d2b1726a7a55fe9d4d Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 02:14:52 +0000 Subject: [PATCH 155/190] update stat20 image tag to feaa2df801d9: deployments/stat20/hubploy.yaml --- deployments/stat20/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index e9a4038e0..fdf2ad5ca 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:5eed7cbfc7ff + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:feaa2df801d9 cluster: provider: gcloud From ac8c9cc556acdc8458ada5e05f9a79c28d632746 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 02:17:50 +0000 Subject: [PATCH 156/190] update stat159 image tag to da599718ae9f: deployments/stat159/hubploy.yaml --- deployments/stat159/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat159/hubploy.yaml b/deployments/stat159/hubploy.yaml index 88dbf1e7e..a8936b80c 100644 --- a/deployments/stat159/hubploy.yaml +++ b/deployments/stat159/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat159-user-image:2d4c8f1488bd + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat159-user-image:da599718ae9f cluster: provider: gcloud From ec6400e05ec65e04c156dbf19f9259ad7c0d14ad Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 02:50:53 +0000 Subject: [PATCH 157/190] update nature image tag to 00512349345a: deployments/nature/hubploy.yaml --- deployments/nature/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/nature/hubploy.yaml b/deployments/nature/hubploy.yaml index 61a4e283e..2836566d2 100644 --- a/deployments/nature/hubploy.yaml +++ b/deployments/nature/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:9ad22965ad22 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/nature-user-image:00512349345a cluster: provider: gcloud From a4b09516cccbaceb77b1ce0426a8ed3a806b6a98 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Fri, 6 Sep 2024 09:59:37 -0700 Subject: [PATCH 158/190] moving bio, cee and astro to new CI/CD workflow --- .circleci/config.yml | 75 -- .github/workflows/deploy-hubs.yaml | 4 +- deployments/astro/hubploy.yaml | 7 +- deployments/astro/image/README.md | 5 + deployments/astro/image/apt.txt | 30 - deployments/astro/image/environment.yml | 109 --- .../astro/image/infra-requirements.txt | 29 - deployments/astro/image/postBuild | 10 - deployments/astro/image/start | 5 - deployments/biology/hubploy.yaml | 7 +- deployments/biology/image/Dockerfile | 206 ----- deployments/biology/image/README.md | 5 + deployments/biology/image/Rprofile.site | 10 - deployments/biology/image/bio1b-packages.bash | 11 - .../biology/image/ccb293-packages.bash | 5 - deployments/biology/image/environment.yml | 82 -- deployments/biology/image/file-locks | 13 - deployments/biology/image/ib134-packages.bash | 32 - .../biology/image/infra-requirements.txt | 29 - .../biology/image/install-mambaforge.bash | 38 - deployments/biology/image/install.R | 48 - deployments/biology/image/mitozEnv_config.py | 3 - .../biology/image/patches/ncbiquery.py | 831 ------------------ deployments/biology/image/rsession.conf | 2 - deployments/cee/hubploy.yaml | 8 +- deployments/cee/image/README.md | 5 + deployments/cee/image/apt.txt | 36 - deployments/cee/image/environment.yml | 31 - deployments/cee/image/infra-requirements.txt | 29 - deployments/cee/image/postBuild | 8 - deployments/cee/image/qgis.desktop | 6 - deployments/cee/image/start | 8 - 32 files changed, 20 insertions(+), 1707 deletions(-) create mode 100644 deployments/astro/image/README.md delete mode 100644 deployments/astro/image/apt.txt delete mode 100644 deployments/astro/image/environment.yml delete mode 100644 deployments/astro/image/infra-requirements.txt delete mode 100644 deployments/astro/image/postBuild delete mode 100644 deployments/astro/image/start delete mode 100644 deployments/biology/image/Dockerfile create mode 100644 deployments/biology/image/README.md delete mode 100644 deployments/biology/image/Rprofile.site delete mode 100644 deployments/biology/image/bio1b-packages.bash delete mode 100644 deployments/biology/image/ccb293-packages.bash delete mode 100644 deployments/biology/image/environment.yml delete mode 100644 deployments/biology/image/file-locks delete mode 100644 deployments/biology/image/ib134-packages.bash delete mode 100644 deployments/biology/image/infra-requirements.txt delete mode 100644 deployments/biology/image/install-mambaforge.bash delete mode 100644 deployments/biology/image/install.R delete mode 100644 deployments/biology/image/mitozEnv_config.py delete mode 100644 deployments/biology/image/patches/ncbiquery.py delete mode 100644 deployments/biology/image/rsession.conf create mode 100644 deployments/cee/image/README.md delete mode 100644 deployments/cee/image/apt.txt delete mode 100644 deployments/cee/image/environment.yml delete mode 100644 deployments/cee/image/infra-requirements.txt delete mode 100755 deployments/cee/image/postBuild delete mode 100755 deployments/cee/image/qgis.desktop delete mode 100755 deployments/cee/image/start diff --git a/.circleci/config.yml b/.circleci/config.yml index bcfc1b225..ae9cdeca5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -181,24 +181,6 @@ jobs: command: | gcloud components install gke-gcloud-auth-plugin - - run: - name: Deploy astro - command: | - hubploy deploy --timeout 30m astro hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - - run: - name: Deploy biology - command: | - hubploy deploy --timeout 30m biology hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - - run: - name: Deploy cee - command: | - hubploy deploy --timeout 30m cee hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy data8 command: | @@ -311,33 +293,6 @@ workflows: version: 2 test-build-images: jobs: - - hubploy/build-image: - deployment: astro - name: astro image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - - hubploy/build-image: - deployment: biology - name: biology image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - - hubploy/build-image: - deployment: cee - name: cee image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: data8 name: data8 image build @@ -451,33 +406,6 @@ workflows: deploy: jobs: - - hubploy/build-image: - deployment: astro - name: astro image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - - hubploy/build-image: - deployment: biology - name: biology image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - - hubploy/build-image: - deployment: cee - name: cee image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: data8 name: data8 image build @@ -591,9 +519,6 @@ workflows: # CI level, we also make prod deploys go faster! - deploy: requires: - - astro image build - - biology image build - - cee image build - data8 image build - data100 image build - data102 image build diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 84cf3b9a2..d34a68037 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) diff --git a/deployments/astro/hubploy.yaml b/deployments/astro/hubploy.yaml index 43bb0b893..a82b84c7b 100644 --- a/deployments/astro/hubploy.yaml +++ b/deployments/astro/hubploy.yaml @@ -1,10 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/astro-user-image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/astro-user-image:placeholder cluster: provider: gcloud diff --git a/deployments/astro/image/README.md b/deployments/astro/image/README.md new file mode 100644 index 000000000..1184a6a58 --- /dev/null +++ b/deployments/astro/image/README.md @@ -0,0 +1,5 @@ +# Astro Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/astro-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/astro-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/astro/image/apt.txt b/deployments/astro/image/apt.txt deleted file mode 100644 index 422e2ccbc..000000000 --- a/deployments/astro/image/apt.txt +++ /dev/null @@ -1,30 +0,0 @@ -# installing less as more just isn't enough -less - -# For desktop environment -dbus-x11 -xfce4 -xfce4-panel -xfce4-session -xfce4-settings -xorg -xubuntu-icon-theme -# You definitely need a browser -firefox -# And a text editor -gedit -# And a terminal -xfce4-terminal - -# https://github.com/berkeley-dsep-infra/datahub/issues/2535 -emacs - -# vim4ever! -vim - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip - -# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 -libnss3 -libnspr4 diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml deleted file mode 100644 index af9024c5a..000000000 --- a/deployments/astro/image/environment.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: astrods - -channels: -- conda-forge -- pytorch - -dependencies: -- python=3.11.* -- jupyter-server-proxy>=4.3 -# A linux desktop environment -- websockify -- numpy==1.26.4 -- numexpr>=2.8.7 -- matplotlib==3.9.* -- graphviz>=2.50.0 -- psycopg2==2.9.* -- pytorch -- torchvision -- pytorch-lightning -- cartopy -- pandas>=2.2.2 -- protobuf -- jupysql -- mpich -- mpi4py -- mkl -- mkl-service -- netcdf4>=1.6 -- scikit-learn==1.5.* -- scikit-image==0.24.* -- scipy==1.14.* -- tensorflow-cpu==2.17.0 -- tensorflow==2.17.0 -- cudatoolkit -- jax>=0.4.31 -- jaxlib>=0.4.31 -- cython==3.0.11 -- sqlite==3.46.0 -- pip - -- pip: - - -r infra-requirements.txt - - opencv-python>=4.5.5.62 - - nose2>=0.10.0 - - sympy - - beautifulsoup4 - - tqdm - - corner==2.2.* - - dask>=2024.8.1 - - dask-ml>=2024.4.4 - - distributed>=2024.8.1 - - SQLAlchemy>=2.0.32 - - xarray>=2024.7.0 - - nltk>=3.9.1 - - seaborn>=0.13.2 - - bqplot>=0.12.43 - - astroquery>=0.4.7 - - astropy>=6.1.2 - - dustmaps>=1.0.13 - - pyvo>=1.5.2 - - joblib==1.4.2 - - pymc>=5.16.2 - - requests>=2.32.3 - - ipycanvas - - altair>=5.4.0 - - vega3>=0.13.0 - - pytz>=2024.1 - - lightkurve - - ipyparallel - - line_profiler>=4.1.3 - - snakeviz>=2.2.0 - - memory_profiler>=0.61.0 - - flask>=3.0.3 - - stsci.tools>=4.1.0 - - gensim>=4.3.3 - - tweet-preprocessor - - pyLDAvis==3.4.1 - - umap-learn>=0.5.6 - - pydot>=3.0.1 - - TPOT>=0.12.2 - - tables>=3.10.1 - - aiohttp>=3.10.4 - - watermark - - autopep8 - - vega_datasets - - vega - - pandas-bokeh - - pythreejs - - ipywidgets - - ipyvolume - - urllib3 - - six - - ipython - - notebook - - click - - fire - - pycodestyle - - flake8 - - tdtax>=0.1.6 -# - nb_black - - pycodestyle_magic - - twine - - otter-grader>=3.1.4 - # for notebook exporting - - nbconvert[webpdf] - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - - pytest-notebook==0.8.1 - - jupyter-tensorboard>=0.2.0 diff --git a/deployments/astro/image/infra-requirements.txt b/deployments/astro/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/astro/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/astro/image/postBuild b/deployments/astro/image/postBuild deleted file mode 100644 index b1b290345..000000000 --- a/deployments/astro/image/postBuild +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# 2024-01-13 sknapp: incompatible due to notebook 7 -# jupyter contrib nbextensions install --sys-prefix --symlink -# jupyter nbextensions_configurator enable --sys-prefix - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -playwright install chromium diff --git a/deployments/astro/image/start b/deployments/astro/image/start deleted file mode 100644 index c3a978b7f..000000000 --- a/deployments/astro/image/start +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# See https://jira-secure.berkeley.edu/browse/DH-305 -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -exec "$@" diff --git a/deployments/biology/hubploy.yaml b/deployments/biology/hubploy.yaml index 26ef5b965..8ac0aeabd 100644 --- a/deployments/biology/hubploy.yaml +++ b/deployments/biology/hubploy.yaml @@ -1,10 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:placeholder cluster: provider: gcloud diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile deleted file mode 100644 index 70680021a..000000000 --- a/deployments/biology/image/Dockerfile +++ /dev/null @@ -1,206 +0,0 @@ -FROM buildpack-deps:jammy-scm - -ENV TZ=America/Los_Angeles -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEBIAN_FRONTEND=noninteractive -ENV NB_USER jovyan -ENV NB_UID 1000 - -ENV CONDA_DIR /srv/conda -ENV R_LIBS_USER /srv/r - -# Explicitly add littler to PATH -# See https://github.com/conda-forge/r-littler-feedstock/issues/6 -ENV PATH ${CONDA_DIR}/lib/R/library/littler/bin:${CONDA_DIR}/bin:$PATH - -RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER} - -# Create user owned R libs dir -# This lets users temporarily install packages -RUN mkdir -p ${R_LIBS_USER} && chown ${NB_USER}:${NB_USER} ${R_LIBS_USER} - -# Required for PAUP* -# Note that this doesn't actually install python2, thankfully -RUN apt-get update -qq --yes > /dev/null && \ - apt-get install --yes -qq \ - libpython2.7 > /dev/null - -## library required for fast-PCA & https://github.com/DReichLab/EIG -RUN apt-get update -qq --yes && \ - apt-get install --yes --no-install-recommends -qq \ - libgsl-dev >/dev/null - -# Install these without 'recommended' packages to keep image smaller. -# Useful utils that folks sort of take for granted -RUN apt-get update -qq --yes && \ - apt-get install --yes --no-install-recommends -qq \ - less \ - htop \ - man \ - nano \ - screen \ - tar \ - tmux \ - wget \ - vim \ - tini \ - locales > /dev/null - -RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ - locale-gen - -# Needed by RStudio -RUN apt-get update -qq --yes && \ - apt-get install --yes --no-install-recommends -qq \ - psmisc \ - sudo \ - libapparmor1 \ - lsb-release \ - libclang-dev \ - libpq5 > /dev/null - -# Needed by many R libraries -# Picked up from https://github.com/rocker-org/rocker/blob/9dc3e458d4e92a8f41ccd75687cd7e316e657cc0/r-rspm/focal/Dockerfile -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - libgdal-dev \ - libgeos3.10.2 \ - libproj22 \ - libudunits2-0 \ - libxml2 > /dev/null - -# Install R. -# These packages must be installed into the base stage since they are in system -# paths rather than /srv. -# Pre-built R packages from rspm are built against system libs in jammy. -ENV R_VERSION=4.4.1-1.2204.0 -ENV LITTLER_VERSION=0.3.19-1.2204.0 -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 -RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" > /etc/apt/sources.list.d/cran.list -RUN curl --silent --location --fail https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc > /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc -RUN apt-get update -qq --yes > /dev/null && \ - apt-get install --yes -qq \ - r-base-core=${R_VERSION} \ - r-base-dev=${R_VERSION} \ - littler=${LITTLER_VERSION} \ - libglpk-dev \ - libzmq5 \ - nodejs npm > /dev/null - -ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2024.04.2-764-amd64.deb -RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \ - apt install --no-install-recommends --yes /tmp/rstudio.deb && \ - rm /tmp/rstudio.deb - -# Install desktop packages -RUN apt-get update -qq --yes > /dev/null && \ - apt-get install --yes -qq \ - dbus-x11 \ - firefox \ - xfce4 \ - xfce4-panel \ - xfce4-terminal \ - xfce4-session \ - xfce4-settings \ - xorg \ - xubuntu-icon-theme > /dev/null - -# for nbconvert & notebook-to-pdf -RUN apt-get update -qq --yes && \ - apt-get install --yes -qq \ - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - libx11-xcb1 \ - libxtst6 \ - libxrandr2 \ - libasound2 \ - libpangocairo-1.0-0 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libgtk-3-0 \ - libnss3 \ - libxss1 \ - > /dev/null - -# Adding ncompress,pbzip2 for issue #1885 BioE-131, Fall 2020 -RUN apt-get update -qq --yes > /dev/null && \ - apt-get install --yes -qq \ - ncompress \ - pbzip2 > /dev/null - -WORKDIR /home/jovyan - -# R_LIBS_USER is set by default in /etc/R/Renviron, which RStudio loads. -# We uncomment the default, and set what we wanna - so it picks up -# the packages we install. Without this, RStudio doesn't see the packages -# that R does. -# Stolen from https://github.com/jupyterhub/repo2docker/blob/6a07a48b2df48168685bb0f993d2a12bd86e23bf/repo2docker/buildpacks/r.py -RUN sed -i -e '/^R_LIBS_USER=/s/^/#/' /etc/R/Renviron && \ - echo "R_LIBS_USER=${R_LIBS_USER}" >> /etc/R/Renviron - -# Needed by Rhtslib -RUN apt-get update -qq --yes && \ - apt-get install --yes -qq \ - libcurl4-openssl-dev > /dev/null - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN chmod 777 /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/ -COPY infra-requirements.txt /tmp/ - -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -USER root -ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} -RUN playwright install-deps -RUN chown -Rh jovyan:jovyan /srv/conda - -USER ${NB_USER} - -# DH-333 -ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} -RUN playwright install chromium - -# 2024-01-13 sknapp: incompatible due to notebook 7 -# RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ -# jupyter nbextensions_configurator enable --sys-prefix - -# Set CRAN mirror to rspm before we install anything -COPY Rprofile.site /usr/lib/R/etc/Rprofile.site -# RStudio needs its own config -COPY rsession.conf /etc/rstudio/rsession.conf -# Use simpler locking strategy -COPY file-locks /etc/rstudio/file-locks - -# Install IRKernel -RUN r -e "install.packages('IRkernel', version='1.2')" && \ - r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -# Install R packages, cleanup temp package download location -COPY install.R /tmp/install.R -RUN r /tmp/install.R && \ - rm -rf /tmp/downloaded_packages/ /tmp/*.rds - -# install bio1b packages -COPY bio1b-packages.bash /tmp/bio1b-packages.bash -RUN bash /tmp/bio1b-packages.bash - -# install ib134L packages -COPY ib134-packages.bash /tmp/ib134-packages.bash -RUN bash /tmp/ib134-packages.bash - -# install ccb293 packages -COPY ccb293-packages.bash /tmp/ccb293-packages.bash -RUN bash /tmp/ccb293-packages.bash - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/biology/image/README.md b/deployments/biology/image/README.md new file mode 100644 index 000000000..4588d74ef --- /dev/null +++ b/deployments/biology/image/README.md @@ -0,0 +1,5 @@ +# Biology Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/biology-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/biology-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/biology/image/Rprofile.site b/deployments/biology/image/Rprofile.site deleted file mode 100644 index 961f50b97..000000000 --- a/deployments/biology/image/Rprofile.site +++ /dev/null @@ -1,10 +0,0 @@ -# Use RStudio's CRAN mirror to get binary packages -# 'latest' just means it has all available versions. -# We can specify version numbers in devtools::install_version -options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/jammy/latest")) - -# RStudio's CRAN mirror needs this to figure out which binary package to serve. -# If not set properly, it will just serve up source packages -# Quite hilarious, IMO. -# See https://docs.rstudio.com/rspm/admin/binaries.html -options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version$platform, R.version$arch, R.version$os))) diff --git a/deployments/biology/image/bio1b-packages.bash b/deployments/biology/image/bio1b-packages.bash deleted file mode 100644 index d926eaddb..000000000 --- a/deployments/biology/image/bio1b-packages.bash +++ /dev/null @@ -1,11 +0,0 @@ -# Install PAUP* for BIO 1B -# https://github.com/berkeley-dsep-infra/datahub/issues/1699 - -# This package was requested in 2020 for the instructor to try out. -# The 168 version doesn't exist so I've bumped it to 169, but also disabled -# it in case the package is no longer needed. -return - -wget https://phylosolutions.com/paup-test/paup4a169_ubuntu64.gz -O ${CONDA_DIR}/bin/paup.gz -gunzip ${CONDA_DIR}/bin/paup.gz -chmod +x ${CONDA_DIR}/bin/paup diff --git a/deployments/biology/image/ccb293-packages.bash b/deployments/biology/image/ccb293-packages.bash deleted file mode 100644 index 8da8e511d..000000000 --- a/deployments/biology/image/ccb293-packages.bash +++ /dev/null @@ -1,5 +0,0 @@ -# Install QIIME2 for CCB293 -# https://github.com/berkeley-dsep-infra/datahub/issues/1699 -wget https://data.qiime2.org/distro/core/qiime2-2021.8-py38-linux-conda.yml -mamba env create -n qiime2 --file qiime2-2021.8-py38-linux-conda.yml && mamba clean -afy -rm qiime2-2021.8-py38-linux-conda.yml diff --git a/deployments/biology/image/environment.yml b/deployments/biology/image/environment.yml deleted file mode 100644 index cc4bee30c..000000000 --- a/deployments/biology/image/environment.yml +++ /dev/null @@ -1,82 +0,0 @@ -channels: -- bioconda -- conda-forge -dependencies: -- python=3.11.* -- pip=22.2.* - -# Package to allow Jupyter Notebook or JupyterLab applications in one conda env to access other kernels (e.g. qiime2) -- nb_conda_kernels=2.3.1 - -# proxy web applications -- jupyter-server-proxy==4.2.0 -- jupyter-rsession-proxy==2.0.1 - -# Packages from bioconda for IB134L -# - bwa=0.7.12 -- samtools=1.3.1 -- mafft=7.471 -- emboss=6.6.0 -- augustus=3.5.0 -- raxml-ng=1.0.1 -- structure=2.3.4 -- paml=4.9 -#- repeatmasker=4.0.9 -- trimmomatic=0.39 -- blast=2.12.0 -- fastqc=0.11.9 -- phyml=3.3.20200621 -- sra-tools=2.11 -# - hisat2=2.2.1 -# - subread=2.0.1 -- plink=1.90b6.21 - -- syncthing==1.18.6 - -# Packages for IB120/201/CCB210 -- sympy=1.12 - -# Packages from bioconda for BioE-131, Fall 2020, Issue #1885 -# - bowtie2=2.5.3 # commented out by sknapp 2024.05.07 doesn't support py3.11 -- spades=3.14.1 - -# Packages for MCB280A, Spring 2022 -- bedtools=2.30.0 - -# Packages from bioconda for BIO1B -- raxml=8.2.* -- muscle=3.8.* -- dendropy=4.4.* - -# pedagogy packages -- scipy=1.11. -- pandas=2.2.2 -- seaborn=0.11.2 - -# compbio BioE c146, Fall 22 and into the future, issue 3785 -- scikit-learn=1.2.2 - -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* - -# for MCELLBI201B (FA24) https://github.com/berkeley-dsep-infra/datahub/issues/5988 -- macs2==2.2.9.1 - -# For https://github.com/berkeley-dsep-infra/datahub/issues/1846 -# Conda does not have these -- pip: - - -r infra-requirements.txt -# MCB280A, Spring 2022 - - pybedtools==0.10.0 -# removed for PR https://github.com/berkeley-dsep-infra/datahub/pull/4406 - macs2==2.2.7.1 - - geonomics==1.4.1 - - nlmpy==1.2.0 - - datascience==0.17.6 - -# Packages for MCB-160L iss #3942 - - allensdk==2.16.2 - - otter-grader==3.1.4 -# for exporting notebooks to pdf - - nbconvert[webpdf]==7.16.4 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 diff --git a/deployments/biology/image/file-locks b/deployments/biology/image/file-locks deleted file mode 100644 index 7b1a3fcf4..000000000 --- a/deployments/biology/image/file-locks +++ /dev/null @@ -1,13 +0,0 @@ -# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking - -# rocker sets this to advisory, but this might be causing NFS issues. -# lets set it to the default (default: linkbased) -lock-type=linkbased - -# we'll also reduce the frequency by 1/3 -refresh-rate=60 -timeout-interval=90 - -# log attempts -# enable-logging=1 -# log-file=/tmp/rstudio-locking.log diff --git a/deployments/biology/image/ib134-packages.bash b/deployments/biology/image/ib134-packages.bash deleted file mode 100644 index be81fa2ce..000000000 --- a/deployments/biology/image/ib134-packages.bash +++ /dev/null @@ -1,32 +0,0 @@ -############################ -# Install packages for IB134L -############################ - -#LOCAL_BIN=${REPO_DIR}/.local/bin -#mkdir -p ${LOCAL_BIN} -# -## mitoZ installation -# -#wget https://raw.githubusercontent.com/linzhi2013/MitoZ/master/version_2.4-alpha/release_MitoZ_v2.4-alpha.tar.bz2 -O ${REPO_DIR}/release_MitoZ_v2.4-alpha.tar.bz2 -#pushd ${REPO_DIR} -#tar -jxvf release_MitoZ_v2.4-alpha.tar.bz2 -#rm release_MitoZ_v2.4-alpha.tar.bz2 -#cd release_MitoZ_v2.4-alpha -#wget https://raw.githubusercontent.com/linzhi2013/MitoZ/master/version_2.4-alpha/mitozEnv.yaml -#cd .. -# -### create mitoZ env -#conda env create -n mitozEnv -f release_MitoZ_v2.4-alpha/mitozEnv.yaml # worked after reinstallation of conda -# -### patch ncbiquery.py -#cp patches/ncbiquery.py /srv/conda/envs/mitozEnv/lib/python3.6/site-packages/ete3/ncbi_taxonomy/ncbiquery.py -# -### download annotations -##source activate mitozEnv -##python3 mitozEnv_config.py -##source deactivate - - -### - - diff --git a/deployments/biology/image/infra-requirements.txt b/deployments/biology/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/biology/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/biology/image/install-mambaforge.bash b/deployments/biology/image/install-mambaforge.bash deleted file mode 100644 index 47bf53f1d..000000000 --- a/deployments/biology/image/install-mambaforge.bash +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION='24.3.0-0' - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/deployments/biology/image/install.R b/deployments/biology/image/install.R deleted file mode 100644 index 8e7e0cb82..000000000 --- a/deployments/biology/image/install.R +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env r - -# Install devtools, so we can install versioned packages -install.packages("devtools") - -# Install a bunch of R packages -# This doesn't do any dependency resolution or anything, -# so refer to `installed.packages()` for authoritative list -cran_packages <- c( - "tidyverse", "1.3.0", - "adegenet", "2.1.3", - "pegas", "0.14", - "phytools", "0.7-70", - "ape","5.4-1", - "seqinr","4.2-4", - "hierfstat","0.5-7", - "poppr","2.8.6", - "PopGenome","2.7.5", - "detectRUNS","0.9.6", - "pwr","1.3" , - "plotly","4.9.3", - "mixtools","1.2.0", - "mclust","5.4.7", - "pheatmap","1.0.12", - "phangorn","2.7.1", - "qqman","0.1.8" -) - -for (i in seq(1, length(cran_packages), 2)) { - devtools::install_version( - cran_packages[i], - version = cran_packages[i + 1] - ) -} - -## Bioconductor packages -if (!requireNamespace("BiocManager", quietly = TRUE)) - install.packages("BiocManager") - -BiocManager::install("EBSeq") -BiocManager::install("Rhtslib") -BiocManager::install("dada2") -BiocManager::install("phyloseq") -BiocManager::install("Biostrings") -BiocManager::install("cummeRbund") -BiocManager::install("DESeq2") -BiocManager::install("apeglm") -BiocManager::install('EnhancedVolcano') diff --git a/deployments/biology/image/mitozEnv_config.py b/deployments/biology/image/mitozEnv_config.py deleted file mode 100644 index a1c4da0a6..000000000 --- a/deployments/biology/image/mitozEnv_config.py +++ /dev/null @@ -1,3 +0,0 @@ -from ete3 import NCBITaxa -ncbi = NCBITaxa() -ncbi.update_taxonomy_database() diff --git a/deployments/biology/image/patches/ncbiquery.py b/deployments/biology/image/patches/ncbiquery.py deleted file mode 100644 index 5a2192d04..000000000 --- a/deployments/biology/image/patches/ncbiquery.py +++ /dev/null @@ -1,831 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# #START_LICENSE########################################################### -# -# -# This file is part of the Environment for Tree Exploration program -# (ETE). http://etetoolkit.org -# -# ETE is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ETE is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public -# License for more details. -# -# You should have received a copy of the GNU General Public License -# along with ETE. If not, see <http://www.gnu.org/licenses/>. -# -# -# ABOUT THE ETE PACKAGE -# ===================== -# -# ETE is distributed under the GPL copyleft license (2008-2015). -# -# If you make use of ETE in published work, please cite: -# -# Jaime Huerta-Cepas, Joaquin Dopazo and Toni Gabaldon. -# ETE: a python Environment for Tree Exploration. Jaime BMC -# Bioinformatics 2010,:24doi:10.1186/1471-2105-11-24 -# -# Note that extra references to the specific methods implemented in -# the toolkit may be available in the documentation. -# -# More info at http://etetoolkit.org. Contact: huerta@embl.de -# -# -# #END_LICENSE############################################################# - - -from __future__ import absolute_import -from __future__ import print_function -import sys -import os -try: - import cPickle as pickle -except ImportError: - # python 3 support - import pickle - -from collections import defaultdict, Counter - -import sqlite3 -import math -import tarfile -import six -from six.moves import map -import warnings - - -__all__ = ["NCBITaxa", "is_taxadb_up_to_date"] - -DB_VERSION = 2 -DEFAULT_TAXADB = os.path.join(os.environ.get('HOME', '/'), '.etetoolkit', 'taxa.sqlite') - - -def is_taxadb_up_to_date(dbfile=DEFAULT_TAXADB): - """Check if a valid and up-to-date taxa.sqlite database exists - - If dbfile= is not specified, DEFAULT_TAXADB is assumed - """ - db = sqlite3.connect(dbfile) - - try: - r = db.execute('SELECT version FROM stats;') - version = r.fetchone()[0] - except (sqlite3.OperationalError, ValueError, IndexError, TypeError): - version = None - - db.close() - - if version != DB_VERSION: - return False - - return True - - -class NCBITaxa(object): - """ - versionadded: 2.3 - - Provides a local transparent connector to the NCBI taxonomy database. - """ - - def __init__(self, dbfile=None, taxdump_file=None): - - if not dbfile: - self.dbfile = DEFAULT_TAXADB - else: - self.dbfile = dbfile - - if taxdump_file: - self.update_taxonomy_database(taxdump_file) - - if dbfile is None and not os.path.exists(self.dbfile): - print('NCBI database not present yet (first time used?)', file=sys.stderr) - self.update_taxonomy_database(taxdump_file) - - if not os.path.exists(self.dbfile): - raise ValueError("Cannot open taxonomy database: %s" % self.dbfile) - - self.db = None - self._connect() - - if not is_taxadb_up_to_date(self.dbfile): - print('NCBI database format is outdated. Upgrading', file=sys.stderr) - self.update_taxonomy_database(taxdump_file) - - def update_taxonomy_database(self, taxdump_file=None): - """Updates the ncbi taxonomy database by downloading and parsing the latest - taxdump.tar.gz file from the NCBI FTP site (via HTTP). - - :param None taxdump_file: an alternative location of the taxdump.tax.gz file. - """ - if not taxdump_file: - update_db(self.dbfile) - else: - update_db(self.dbfile, taxdump_file) - - def _connect(self): - self.db = sqlite3.connect(self.dbfile) - - def _translate_merged(self, all_taxids): - conv_all_taxids = set((list(map(int, all_taxids)))) - cmd = 'select taxid_old, taxid_new FROM merged WHERE taxid_old IN (%s)' %','.join(map(str, all_taxids)) - - result = self.db.execute(cmd) - conversion = {} - for old, new in result.fetchall(): - conv_all_taxids.discard(int(old)) - conv_all_taxids.add(int(new)) - conversion[int(old)] = int(new) - return conv_all_taxids, conversion - - - def get_fuzzy_name_translation(self, name, sim=0.9): - ''' - Given an inexact species name, returns the best match in the NCBI database of taxa names. - - :argument 0.9 sim: Min word similarity to report a match (from 0 to 1). - - :return: taxid, species-name-match, match-score - ''' - - - import sqlite3.dbapi2 as dbapi2 - _db = dbapi2.connect(self.dbfile) - _db.enable_load_extension(True) - module_path = os.path.split(os.path.realpath(__file__))[0] - _db.execute("select load_extension('%s')" % os.path.join(module_path, - "SQLite-Levenshtein/levenshtein.sqlext")) - - print("Trying fuzzy search for %s" % name) - maxdiffs = math.ceil(len(name) * (1-sim)) - cmd = 'SELECT taxid, spname, LEVENSHTEIN(spname, "%s") AS sim FROM species WHERE sim<=%s ORDER BY sim LIMIT 1;' % (name, maxdiffs) - taxid, spname, score = None, None, len(name) - result = _db.execute(cmd) - try: - taxid, spname, score = result.fetchone() - except TypeError: - cmd = 'SELECT taxid, spname, LEVENSHTEIN(spname, "%s") AS sim FROM synonym WHERE sim<=%s ORDER BY sim LIMIT 1;' % (name, maxdiffs) - result = _db.execute(cmd) - try: - taxid, spname, score = result.fetchone() - except Exception: - pass - else: - taxid = int(taxid) - else: - taxid = int(taxid) - - norm_score = 1 - (float(score)/len(name)) - if taxid: - print("FOUND! %s taxid:%s score:%s (%s)" %(spname, taxid, score, norm_score)) - - return taxid, spname, norm_score - - def get_rank(self, taxids): - 'return a dictionary converting a list of taxids into their corresponding NCBI taxonomy rank' - - all_ids = set(taxids) - all_ids.discard(None) - all_ids.discard("") - query = ','.join(['"%s"' %v for v in all_ids]) - cmd = "select taxid, rank FROM species WHERE taxid IN (%s);" %query - result = self.db.execute(cmd) - id2rank = {} - for tax, spname in result.fetchall(): - id2rank[tax] = spname - return id2rank - - def get_lineage_translator(self, taxids): - """Given a valid taxid number, return its corresponding lineage track as a - hierarchically sorted list of parent taxids. - """ - all_ids = set(taxids) - all_ids.discard(None) - all_ids.discard("") - query = ','.join(['"%s"' %v for v in all_ids]) - result = self.db.execute('SELECT taxid, track FROM species WHERE taxid IN (%s);' %query) - id2lineages = {} - for tax, track in result.fetchall(): - id2lineages[tax] = list(map(int, reversed(track.split(",")))) - - return id2lineages - - - def get_lineage(self, taxid): - """Given a valid taxid number, return its corresponding lineage track as a - hierarchically sorted list of parent taxids. - """ - if not taxid: - return None - result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %taxid) - raw_track = result.fetchone() - if not raw_track: - #perhaps is an obsolete taxid - _, merged_conversion = self._translate_merged([taxid]) - if taxid in merged_conversion: - result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %merged_conversion[taxid]) - raw_track = result.fetchone() - # if not raise error - if not raw_track: - #raw_track = ["1"] - raise ValueError("%s taxid not found" %taxid) - else: - warnings.warn("taxid %s was translated into %s" %(taxid, merged_conversion[taxid])) - - track = list(map(int, raw_track[0].split(","))) - return list(reversed(track)) - - def get_common_names(self, taxids): - query = ','.join(['"%s"' %v for v in taxids]) - cmd = "select taxid, common FROM species WHERE taxid IN (%s);" %query - result = self.db.execute(cmd) - id2name = {} - for tax, common_name in result.fetchall(): - if common_name: - id2name[tax] = common_name - return id2name - - def get_taxid_translator(self, taxids, try_synonyms=True): - """Given a list of taxids, returns a dictionary with their corresponding - scientific names. - """ - - all_ids = set(map(int, taxids)) - all_ids.discard(None) - all_ids.discard("") - query = ','.join(['"%s"' %v for v in all_ids]) - cmd = "select taxid, spname FROM species WHERE taxid IN (%s);" %query - result = self.db.execute(cmd) - id2name = {} - for tax, spname in result.fetchall(): - id2name[tax] = spname - - # any taxid without translation? lets tray in the merged table - if len(all_ids) != len(id2name) and try_synonyms: - not_found_taxids = all_ids - set(id2name.keys()) - taxids, old2new = self._translate_merged(not_found_taxids) - new2old = {v: k for k,v in six.iteritems(old2new)} - - if old2new: - query = ','.join(['"%s"' %v for v in new2old]) - cmd = "select taxid, spname FROM species WHERE taxid IN (%s);" %query - result = self.db.execute(cmd) - for tax, spname in result.fetchall(): - id2name[new2old[tax]] = spname - - return id2name - - def get_name_translator(self, names): - """ - Given a list of taxid scientific names, returns a dictionary translating them into their corresponding taxids. - - Exact name match is required for translation. - """ - - name2id = {} - #name2realname = {} - name2origname = {} - for n in names: - name2origname[n.lower()] = n - - names = set(name2origname.keys()) - - query = ','.join(['"%s"' %n for n in six.iterkeys(name2origname)]) - cmd = 'select spname, taxid from species where spname IN (%s)' %query - result = self.db.execute(cmd) - for sp, taxid in result.fetchall(): - oname = name2origname[sp.lower()] - name2id.setdefault(oname, []).append(taxid) - #name2realname[oname] = sp - missing = names - set([n.lower() for n in name2id.keys()]) - if missing: - query = ','.join(['"%s"' %n for n in missing]) - result = self.db.execute('select spname, taxid from synonym where spname IN (%s)' %query) - for sp, taxid in result.fetchall(): - oname = name2origname[sp.lower()] - name2id.setdefault(oname, []).append(taxid) - #name2realname[oname] = sp - return name2id - - def translate_to_names(self, taxids): - """ - Given a list of taxid numbers, returns another list with their corresponding scientific names. - """ - id2name = self.get_taxid_translator(taxids) - names = [] - for sp in taxids: - names.append(id2name.get(sp, sp)) - return names - - - def get_descendant_taxa(self, parent, intermediate_nodes=False, rank_limit=None, collapse_subspecies=False, return_tree=False): - """ - given a parent taxid or scientific species name, returns a list of all its descendants taxids. - If intermediate_nodes is set to True, internal nodes will also be dumped. - - """ - try: - taxid = int(parent) - except ValueError: - try: - taxid = self.get_name_translator([parent])[parent][0] - except KeyError: - raise ValueError('%s not found!' %parent) - - with open(self.dbfile+".traverse.pkl", "rb") as CACHED_TRAVERSE: - prepostorder = pickle.load(CACHED_TRAVERSE) - descendants = {} - found = 0 - for tid in prepostorder: - if tid == taxid: - found += 1 - elif found == 1: - descendants[tid] = descendants.get(tid, 0) + 1 - elif found == 2: - break - - if not found: - raise ValueError("taxid not found:%s" %taxid) - elif found == 1: - return [taxid] - - if rank_limit or collapse_subspecies or return_tree: - tree = self.get_topology(list(descendants.keys()), intermediate_nodes=intermediate_nodes, collapse_subspecies=collapse_subspecies, rank_limit=rank_limit) - if return_tree: - return tree - elif intermediate_nodes: - return list(map(int, [n.name for n in tree.get_descendants()])) - else: - return map(int, [n.name for n in tree]) - - elif intermediate_nodes: - return [tid for tid, count in six.iteritems(descendants)] - else: - return [tid for tid, count in six.iteritems(descendants) if count == 1] - - def get_topology(self, taxids, intermediate_nodes=False, rank_limit=None, collapse_subspecies=False, annotate=True): - """Given a list of taxid numbers, return the minimal pruned NCBI taxonomy tree - containing all of them. - - :param False intermediate_nodes: If True, single child nodes - representing the complete lineage of leaf nodes are kept. - Otherwise, the tree is pruned to contain the first common - ancestor of each group. - - :param None rank_limit: If valid NCBI rank name is provided, - the tree is pruned at that given level. For instance, use - rank="species" to get rid of sub-species or strain leaf - nodes. - - :param False collapse_subspecies: If True, any item under the - species rank will be collapsed into the species upper - node. - - """ - from .. import PhyloTree - taxids, merged_conversion = self._translate_merged(taxids) - if len(taxids) == 1: - root_taxid = int(list(taxids)[0]) - with open(self.dbfile+".traverse.pkl", "rb") as CACHED_TRAVERSE: - prepostorder = pickle.load(CACHED_TRAVERSE) - nodes = {} - visited = set() - start = prepostorder.index(root_taxid) - try: - end = prepostorder.index(root_taxid, start+1) - subtree = prepostorder[start:end+1] - except ValueError: - # If root taxid is not found in postorder, must be a tip node - subtree = [root_taxid] - - leaves = set([v for v, count in Counter(subtree).items() if count == 1]) - nodes[root_taxid] = PhyloTree(name=str(root_taxid)) - current_parent = nodes[root_taxid] - for tid in subtree: - if tid in visited: - current_parent = nodes[tid].up - else: - visited.add(tid) - nodes[tid] = PhyloTree(name=str(tid)) - current_parent.add_child(nodes[tid]) - if tid not in leaves: - current_parent = nodes[tid] - root = nodes[root_taxid] - else: - taxids = set(map(int, taxids)) - sp2track = {} - elem2node = {} - id2lineage = self.get_lineage_translator(taxids) - all_taxids = set() - for lineage in id2lineage.values(): - all_taxids.update(lineage) - id2rank = self.get_rank(all_taxids) - for sp in taxids: - track = [] - lineage = id2lineage[sp] - - for elem in lineage: - if elem not in elem2node: - node = elem2node.setdefault(elem, PhyloTree()) - node.name = str(elem) - node.taxid = elem - node.add_feature("rank", str(id2rank.get(int(elem), "no rank"))) - else: - node = elem2node[elem] - track.append(node) - sp2track[sp] = track - # generate parent child relationships - for sp, track in six.iteritems(sp2track): - parent = None - for elem in track: - if parent and elem not in parent.children: - parent.add_child(elem) - if rank_limit and elem.rank == rank_limit: - break - parent = elem - root = elem2node[1] - - #remove onechild-nodes - if not intermediate_nodes: - for n in root.get_descendants(): - if len(n.children) == 1 and int(n.name) not in taxids: - n.delete(prevent_nondicotomic=False) - - if len(root.children) == 1: - tree = root.children[0].detach() - else: - tree = root - - if collapse_subspecies: - to_detach = [] - for node in tree.traverse(): - if node.rank == "species": - to_detach.extend(node.children) - for n in to_detach: - n.detach() - - if annotate: - self.annotate_tree(tree) - - return tree - - - def annotate_tree(self, t, taxid_attr="name", tax2name=None, tax2track=None, tax2rank=None): - """Annotate a tree containing taxids as leaf names by adding the 'taxid', - 'sci_name', 'lineage', 'named_lineage' and 'rank' additional attributes. - - :param t: a Tree (or Tree derived) instance. - - :param name taxid_attr: Allows to set a custom node attribute - containing the taxid number associated to each node (i.e. - species in PhyloTree instances). - - :param tax2name,tax2track,tax2rank: Use these arguments to - provide pre-calculated dictionaries providing translation - from taxid number and names,track lineages and ranks. - """ - - taxids = set() - for n in t.traverse(): - try: - tid = int(getattr(n, taxid_attr)) - except (ValueError,AttributeError): - pass - else: - taxids.add(tid) - merged_conversion = {} - - taxids, merged_conversion = self._translate_merged(taxids) - - if not tax2name or taxids - set(map(int, list(tax2name.keys()))): - tax2name = self.get_taxid_translator(taxids) - if not tax2track or taxids - set(map(int, list(tax2track.keys()))): - tax2track = self.get_lineage_translator(taxids) - - all_taxid_codes = set([_tax for _lin in list(tax2track.values()) for _tax in _lin]) - extra_tax2name = self.get_taxid_translator(list(all_taxid_codes - set(tax2name.keys()))) - tax2name.update(extra_tax2name) - - tax2common_name = self.get_common_names(tax2name.keys()) - - if not tax2rank: - tax2rank = self.get_rank(list(tax2name.keys())) - - n2leaves = t.get_cached_content() - - for n in t.traverse('postorder'): - try: - node_taxid = int(getattr(n, taxid_attr)) - except (ValueError, AttributeError): - node_taxid = None - - n.add_features(taxid = node_taxid) - if node_taxid: - if node_taxid in merged_conversion: - node_taxid = merged_conversion[node_taxid] - n.add_features(sci_name = tax2name.get(node_taxid, getattr(n, taxid_attr, '')), - common_name = tax2common_name.get(node_taxid, ''), - lineage = tax2track[node_taxid], - rank = tax2rank.get(node_taxid, 'Unknown'), - named_lineage = [tax2name.get(tax, str(tax)) for tax in tax2track[node_taxid]]) - elif n.is_leaf(): - n.add_features(sci_name = getattr(n, taxid_attr, 'NA'), - common_name = '', - lineage = [], - rank = 'Unknown', - named_lineage = []) - else: - lineage = self._common_lineage([lf.lineage for lf in n2leaves[n]]) - ancestor = lineage[-1] - n.add_features(sci_name = tax2name.get(ancestor, str(ancestor)), - common_name = tax2common_name.get(ancestor, ''), - taxid = ancestor, - lineage = lineage, - rank = tax2rank.get(ancestor, 'Unknown'), - named_lineage = [tax2name.get(tax, str(tax)) for tax in lineage]) - - return tax2name, tax2track, tax2rank - - def _common_lineage(self, vectors): - occurrence = defaultdict(int) - pos = defaultdict(set) - for v in vectors: - for i, taxid in enumerate(v): - occurrence[taxid] += 1 - pos[taxid].add(i) - - common = [taxid for taxid, ocu in six.iteritems(occurrence) if ocu == len(vectors)] - if not common: - return [""] - else: - sorted_lineage = sorted(common, key=lambda x: min(pos[x])) - return sorted_lineage - - # OLD APPROACH: - - # visited = defaultdict(int) - # for index, name in [(ei, e) for v in vectors for ei, e in enumerate(v)]: - # visited[(name, index)] += 1 - - # def _sort(a, b): - # if a[1] > b[1]: - # return 1 - # elif a[1] < b[1]: - # return -1 - # else: - # if a[0][1] > b[0][1]: - # return 1 - # elif a[0][1] < b[0][1]: - # return -1 - # return 0 - - # matches = sorted(visited.items(), _sort) - - # if matches: - # best_match = matches[-1] - # else: - # return "", set() - - # if best_match[1] != len(vectors): - # return "", set() - # else: - # return best_match[0][0], [m[0][0] for m in matches if m[1] == len(vectors)] - - - def get_broken_branches(self, t, taxa_lineages, n2content=None): - """Returns a list of NCBI lineage names that are not monophyletic in the - provided tree, as well as the list of affected branches and their size. - - CURRENTLY EXPERIMENTAL - - """ - if not n2content: - n2content = t.get_cached_content() - - tax2node = defaultdict(set) - - unknown = set() - for leaf in t.iter_leaves(): - if leaf.sci_name.lower() != "unknown": - lineage = taxa_lineages[leaf.taxid] - for index, tax in enumerate(lineage): - tax2node[tax].add(leaf) - else: - unknown.add(leaf) - - broken_branches = defaultdict(set) - broken_clades = set() - for tax, leaves in six.iteritems(tax2node): - if len(leaves) > 1: - common = t.get_common_ancestor(leaves) - else: - common = list(leaves)[0] - if (leaves ^ set(n2content[common])) - unknown: - broken_branches[common].add(tax) - broken_clades.add(tax) - - broken_clade_sizes = [len(tax2node[tax]) for tax in broken_clades] - return broken_branches, broken_clades, broken_clade_sizes - - - # def annotate_tree_with_taxa(self, t, name2taxa_file, tax2name=None, tax2track=None, attr_name="name"): - # if name2taxa_file: - # names2taxid = dict([map(strip, line.split("\t")) - # for line in open(name2taxa_file)]) - # else: - # names2taxid = dict([(n.name, getattr(n, attr_name)) for n in t.iter_leaves()]) - - # not_found = 0 - # for n in t.iter_leaves(): - # n.add_features(taxid=names2taxid.get(n.name, 0)) - # n.add_features(species=n.taxid) - # if n.taxid == 0: - # not_found += 1 - # if not_found: - # print >>sys.stderr, "WARNING: %s nodes where not found within NCBI taxonomy!!" %not_found - - # return self.annotate_tree(t, tax2name, tax2track, attr_name="taxid") - - -def load_ncbi_tree_from_dump(tar): - from .. import Tree - # Download: http://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz - parent2child = {} - name2node = {} - node2taxname = {} - synonyms = set() - node2common = {} - print("Loading node names...") - for line in tar.extractfile("names.dmp"): - line = str(line.decode()) - fields = [_f.strip() for _f in line.split("|")] - nodename = fields[0] - name_type = fields[3].lower() - taxname = fields[1] - if name_type == "scientific name": - node2taxname[nodename] = taxname - if name_type == "genbank common name": - node2common[nodename] = taxname - elif name_type in set(["synonym", "equivalent name", "genbank equivalent name", - "anamorph", "genbank synonym", "genbank anamorph", "teleomorph"]): - synonyms.add( (nodename, taxname) ) - print(len(node2taxname), "names loaded.") - print(len(synonyms), "synonyms loaded.") - - print("Loading nodes...") - for line in tar.extractfile("nodes.dmp"): - line = str(line.decode()) - fields = line.split("|") - nodename = fields[0].strip() - parentname = fields[1].strip() - n = Tree() - n.name = nodename - n.taxname = node2taxname[nodename] - if nodename in node2common: - n.common_name = node2common[nodename] - n.rank = fields[2].strip() - parent2child[nodename] = parentname - name2node[nodename] = n - print(len(name2node), "nodes loaded.") - - print("Linking nodes...") - for node in name2node: - if node == "1": - t = name2node[node] - else: - parent = parent2child[node] - parent_node = name2node[parent] - parent_node.add_child(name2node[node]) - print("Tree is loaded.") - return t, synonyms - -def generate_table(t): - OUT = open("taxa.tab", "w") - for j, n in enumerate(t.traverse()): - if j%1000 == 0: - print("\r",j,"generating entries...", end=' ') - temp_node = n - track = [] - while temp_node: - track.append(temp_node.name) - temp_node = temp_node.up - if n.up: - print('\t'.join([n.name, n.up.name, n.taxname, getattr(n, "common_name", ""), n.rank, ','.join(track)]), file=OUT) - else: - print('\t'.join([n.name, "", n.taxname, getattr(n, "common_name", ""), n.rank, ','.join(track)]), file=OUT) - OUT.close() - -def update_db(dbfile, targz_file=None): - basepath = os.path.split(dbfile)[0] - if basepath and not os.path.exists(basepath): - os.mkdir(basepath) - - if not targz_file: - try: - from urllib import urlretrieve - except ImportError: - from urllib.request import urlretrieve - - print('Downloading taxdump.tar.gz from NCBI FTP site (via HTTP)...', file=sys.stderr) - urlretrieve("http://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz", "taxdump.tar.gz") - print('Done. Parsing...', file=sys.stderr) - targz_file = "taxdump.tar.gz" - - tar = tarfile.open(targz_file, 'r') - t, synonyms = load_ncbi_tree_from_dump(tar) - prepostorder = [int(node.name) for post, node in t.iter_prepostorder()] - pickle.dump(prepostorder, open(dbfile+'.traverse.pkl', "wb"), 2) - - print("Updating database: %s ..." %dbfile) - generate_table(t) - - open("syn.tab", "w").write('\n'.join(["%s\t%s" %(v[0],v[1]) for v in synonyms])) - - with open("merged.tab", "w") as merged: - for line in tar.extractfile("merged.dmp"): - line = str(line.decode()) - out_line = '\t'.join([_f.strip() for _f in line.split('|')[:2]]) - merged.write(out_line+'\n') - try: - upload_data(dbfile) - except: - raise - else: - os.system("rm syn.tab merged.tab taxa.tab") - # remove only downloaded taxdump file - if not targz_file: - os.system("rm taxdump.tar.gz") - -def upload_data(dbfile): - print() - print('Uploading to', dbfile) - basepath = os.path.split(dbfile)[0] - if basepath and not os.path.exists(basepath): - os.mkdir(basepath) - - db = sqlite3.connect(dbfile) - - create_cmd = """ - DROP TABLE IF EXISTS stats; - DROP TABLE IF EXISTS species; - DROP TABLE IF EXISTS synonym; - DROP TABLE IF EXISTS merged; - CREATE TABLE stats (version INT PRIMARY KEY); - CREATE TABLE species (taxid INT PRIMARY KEY, parent INT, spname VARCHAR(50) COLLATE NOCASE, common VARCHAR(50) COLLATE NOCASE, rank VARCHAR(50), track TEXT); - CREATE TABLE synonym (taxid INT,spname VARCHAR(50), PRIMARY KEY (spname, taxid)); - CREATE TABLE merged (taxid_old INT, taxid_new INT); - CREATE INDEX spname1 ON species (spname COLLATE NOCASE); - CREATE INDEX spname2 ON synonym (spname COLLATE NOCASE); - """ - for cmd in create_cmd.split(';'): - db.execute(cmd) - print() - - db.execute("INSERT INTO stats (version) VALUES (%d);" %DB_VERSION) - db.commit() - - for i, line in enumerate(open("syn.tab")): - if i%5000 == 0 : - print('\rInserting synonyms: % 6d' %i, end=' ', file=sys.stderr) - sys.stderr.flush() - taxid, spname = line.strip('\n').split('\t') - db.execute("INSERT INTO synonym (taxid, spname) VALUES (?, ?);", (taxid, spname)) - print() - db.commit() - for i, line in enumerate(open("merged.tab")): - if i%5000 == 0 : - print('\rInserting taxid merges: % 6d' %i, end=' ', file=sys.stderr) - sys.stderr.flush() - taxid_old, taxid_new = line.strip('\n').split('\t') - db.execute("INSERT INTO merged (taxid_old, taxid_new) VALUES (?, ?);", (taxid_old, taxid_new)) - print() - db.commit() - for i, line in enumerate(open("taxa.tab")): - if i%5000 == 0 : - print('\rInserting taxids: % 6d' %i, end=' ', file=sys.stderr) - sys.stderr.flush() - taxid, parentid, spname, common, rank, lineage = line.strip('\n').split('\t') - db.execute("INSERT INTO species (taxid, parent, spname, common, rank, track) VALUES (?, ?, ?, ?, ?, ?);", (taxid, parentid, spname, common, rank, lineage)) - print() - db.commit() - -if __name__ == "__main__": - ncbi = NCBITaxa() - - a = ncbi.get_descendant_taxa("hominidae") - print(a) - print(ncbi.get_common_names(a)) - print(ncbi.get_topology(a)) - b = ncbi.get_descendant_taxa("homo", intermediate_nodes=True, collapse_subspecies=True) - print(ncbi.get_taxid_translator(b)) - - print(ncbi.get_common_names(b)) - #ncbi.update_taxonomy_database() - diff --git a/deployments/biology/image/rsession.conf b/deployments/biology/image/rsession.conf deleted file mode 100644 index 1f82b6b54..000000000 --- a/deployments/biology/image/rsession.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Use binary packages! -r-cran-repos=https://packagemanager.rstudio.com/all/__linux__/jammy/latest diff --git a/deployments/cee/hubploy.yaml b/deployments/cee/hubploy.yaml index ae88ce005..7ad9e5d0a 100644 --- a/deployments/cee/hubploy.yaml +++ b/deployments/cee/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/cee-image - path: image/ - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/cee-image:placeholder cluster: provider: gcloud diff --git a/deployments/cee/image/README.md b/deployments/cee/image/README.md new file mode 100644 index 000000000..22da95223 --- /dev/null +++ b/deployments/cee/image/README.md @@ -0,0 +1,5 @@ +# Cee Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/cee-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/cee-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/cee/image/apt.txt b/deployments/cee/image/apt.txt deleted file mode 100644 index fdb6e7bd3..000000000 --- a/deployments/cee/image/apt.txt +++ /dev/null @@ -1,36 +0,0 @@ -# installing less as more just isn't enough -less - -# For desktop environment -dbus-x11 -xfce4 -xfce4-panel -xfce4-session -xfce4-settings -xubuntu-icon-theme -xauth -# You definitely need a browser -firefox -# And a text editor -gedit -# And a terminal -xfce4-terminal - -# https://github.com/berkeley-dsep-infra/datahub/issues/2535 -emacs - -# vim4ever! -vim - -# https://github.com/berkeley-dsep-infra/datahub/issues/3679 -texlive-xetex -# https://github.com/berkeley-dsep-infra/datahub/issues/3721 -texlive-fonts-recommended -texlive-plain-generic -texlive-lang-chinese - -# Needed for qgis maybe? https://github.com/conda-forge/pygridgen-feedstock/issues/10 -libgl1-mesa-glx - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip diff --git a/deployments/cee/image/environment.yml b/deployments/cee/image/environment.yml deleted file mode 100644 index 7ce6957e0..000000000 --- a/deployments/cee/image/environment.yml +++ /dev/null @@ -1,31 +0,0 @@ -channels: -- conda-forge - -# Almost all libraries should be added in requirements.txt -# Only libraries *not* available in PyPI should be here -dependencies: -- python=3.11.* -- jupyter-server-proxy==4.2.0 -#adding math functionality -- matplotlib=3.7.* -- scipy=1.10.* -- mpld3=0.5.* - -# Needed for linux desktop environment -- websockify=0.11.0 - -- qgis=3.22.* - -# Maybe needed for qgis? https://github.com/conda-forge/qgis-feedstock/issues/263 -- pyopencl=2023.1 - -- pip -- pip: -# For desktop environment - - jupyter-desktop-server==0.1.3 - - -r infra-requirements.txt - - otter-grader==3.1.4 - # for notebook exporting - - nbconvert==7.6.0 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 diff --git a/deployments/cee/image/infra-requirements.txt b/deployments/cee/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/cee/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/cee/image/postBuild b/deployments/cee/image/postBuild deleted file mode 100755 index 0b401ab73..000000000 --- a/deployments/cee/image/postBuild +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -eou pipefail - -# HORRIBLE HACK to get qgis working. conda-forge brings in libgsl 2.7, but this seems -# to require libgsl 2.5? And I can't force conda to install that. This might break something -# deep somewhere. See https://github.com/conda-forge/qgis-feedstock/issues/263 -# for upstream discussion -#ln -s /srv/conda/envs/notebook/lib/libgsl.so.27 /srv/conda/envs/notebook/lib/libgsl.so.25 diff --git a/deployments/cee/image/qgis.desktop b/deployments/cee/image/qgis.desktop deleted file mode 100755 index 3c5efdd84..000000000 --- a/deployments/cee/image/qgis.desktop +++ /dev/null @@ -1,6 +0,0 @@ -[Desktop Entry] -Version=1.0 -Type=Application -Name=qgis -Exec=qgis -Icon=/srv/conda/envs/notebook/share/qgis/images/icons/qgis-icon-512x512.png \ No newline at end of file diff --git a/deployments/cee/image/start b/deployments/cee/image/start deleted file mode 100755 index 1ff95e436..000000000 --- a/deployments/cee/image/start +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -euo pipefail - -mkdir -p $HOME/Desktop - -cp $REPO_DIR/*.desktop $HOME/Desktop/ - -exec "$@" \ No newline at end of file From 63fdfc5c7e6514e1d63959db698a957ae73a99e7 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Fri, 6 Sep 2024 10:03:42 -0700 Subject: [PATCH 159/190] don't enable deploys... yet --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index d34a68037..af3c1948f 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) # astro biology cee) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) # astro biology cee) From f3a266cda05d3934bd3f6d4c558ca3fe299f932c Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Fri, 6 Sep 2024 10:43:37 -0700 Subject: [PATCH 160/190] enable deployment of astro/bio/cee --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index af3c1948f..d34a68037 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) # astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101) # astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) From 0ebfa58295e68803c84b9b3d5f68a8500488f910 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:50:11 +0000 Subject: [PATCH 161/190] update astro image tag to ffc76921fb47: deployments/astro/hubploy.yaml --- deployments/astro/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/astro/hubploy.yaml b/deployments/astro/hubploy.yaml index a82b84c7b..29a6aa87b 100644 --- a/deployments/astro/hubploy.yaml +++ b/deployments/astro/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/astro-user-image:placeholder + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/astro-user-image:ffc76921fb47 cluster: provider: gcloud From 2a2201b1b3d7f6677748f12aad7a9c5efc39c0f1 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Fri, 6 Sep 2024 10:53:28 -0700 Subject: [PATCH 162/190] fixing typo --- deployments/cee/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/cee/hubploy.yaml b/deployments/cee/hubploy.yaml index 7ad9e5d0a..9a72e0742 100644 --- a/deployments/cee/hubploy.yaml +++ b/deployments/cee/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/cee-image:placeholder + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/cee-user-image:placeholder cluster: provider: gcloud From 433aec9ee75c3cc9241f77af037a07b11dfd8280 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:54:59 +0000 Subject: [PATCH 163/190] update cee image tag to 5e393b8d22ce: deployments/cee/hubploy.yaml --- deployments/cee/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/cee/hubploy.yaml b/deployments/cee/hubploy.yaml index 9a72e0742..a851ce268 100644 --- a/deployments/cee/hubploy.yaml +++ b/deployments/cee/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/cee-user-image:placeholder + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/cee-user-image:5e393b8d22ce cluster: provider: gcloud From 90953d3346b2c567fd0e239e88f1082de72c3376 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Fri, 6 Sep 2024 18:22:51 +0000 Subject: [PATCH 164/190] update biology image tag to 24ed6d3bf1a6: deployments/biology/hubploy.yaml --- deployments/biology/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/biology/hubploy.yaml b/deployments/biology/hubploy.yaml index 8ac0aeabd..83c9f483a 100644 --- a/deployments/biology/hubploy.yaml +++ b/deployments/biology/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:placeholder + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:24ed6d3bf1a6 cluster: provider: gcloud From 60fe67ab7d1a31d6b211cd1a7b8e701970ab9e28 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 13:36:00 -0700 Subject: [PATCH 165/190] Remove contents of shiny image. --- deployments/shiny/image/Dockerfile | 97 ------------------- deployments/shiny/image/class-libs.R | 20 ---- deployments/shiny/image/environment.yml | 22 ----- deployments/shiny/image/file-locks | 13 --- .../shiny/image/infra-requirements.txt | 29 ------ .../shiny/image/install-mambaforge.bash | 38 -------- .../image/r-packages/2023-fall-stat-135.r | 12 --- .../image/r-packages/2023-spring-stat-20.r | 16 --- .../image/r-packages/2024-spring-gradebook.r | 20 ---- 9 files changed, 267 deletions(-) delete mode 100644 deployments/shiny/image/Dockerfile delete mode 100644 deployments/shiny/image/class-libs.R delete mode 100644 deployments/shiny/image/environment.yml delete mode 100644 deployments/shiny/image/file-locks delete mode 100644 deployments/shiny/image/infra-requirements.txt delete mode 100755 deployments/shiny/image/install-mambaforge.bash delete mode 100644 deployments/shiny/image/r-packages/2023-fall-stat-135.r delete mode 100644 deployments/shiny/image/r-packages/2023-spring-stat-20.r delete mode 100644 deployments/shiny/image/r-packages/2024-spring-gradebook.r diff --git a/deployments/shiny/image/Dockerfile b/deployments/shiny/image/Dockerfile deleted file mode 100644 index 9d26e61eb..000000000 --- a/deployments/shiny/image/Dockerfile +++ /dev/null @@ -1,97 +0,0 @@ -FROM rocker/geospatial:4.4.1 - -ENV NB_USER rstudio -ENV NB_UID 1000 -ENV CONDA_DIR /srv/conda - -# Set ENV for all programs... -ENV PATH ${CONDA_DIR}/bin:$PATH - -# Pick up rocker's default TZ -ENV TZ=Etc/UTC - -# And set ENV for R! It doesn't read from the environment... -RUN echo "TZ=${TZ}" >> /usr/local/lib/R/etc/Renviron.site -RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site - -# Add PATH to /etc/profile so it gets picked up by the terminal -RUN echo "PATH=${PATH}" >> /etc/profile -RUN echo "export PATH" >> /etc/profile - -ENV HOME /home/${NB_USER} - -WORKDIR ${HOME} - -# Install packages needed by notebook-as-pdf -# nodejs for installing notebook / jupyterhub from source -# libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -RUN apt-get update > /dev/null && \ - apt-get install --yes \ - less \ - libx11-xcb1 \ - libxtst6 \ - libxrandr2 \ - libasound2 \ - libpangocairo-1.0-0 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libgtk-3-0 \ - libnss3 \ - libxss1 \ - fonts-symbola \ - gdebi-core \ - tini \ - pandoc \ - texlive-xetex \ - texlive-latex-extra \ - texlive-fonts-recommended \ - # provides FandolSong-Regular.otf for issue #2714 - texlive-lang-chinese \ - texlive-plain-generic \ - nodejs npm > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV SHINY_SERVER_URL https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-1.5.22.1017-amd64.deb -RUN curl --silent --location --fail ${SHINY_SERVER_URL} > /tmp/shiny-server.deb && \ - apt install --no-install-recommends --yes /tmp/shiny-server.deb && \ - rm /tmp/shiny-server.deb - -# google-chrome is for pagedown; chromium doesn't work nicely with it (snap?) -RUN wget --quiet -O /tmp/google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - apt-get update > /dev/null && \ - apt -y install /tmp/google-chrome-stable_current_amd64.deb > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/environment.yml -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -COPY class-libs.R /tmp/class-libs.R - -COPY r-packages/2024-spring-gradebook.r /tmp/r-packages/ -RUN r /tmp/r-packages/2024-spring-gradebook.r - -COPY r-packages/2023-spring-stat-20.r /tmp/r-packages/ -RUN r /tmp/r-packages/2023-spring-stat-20.r - -COPY r-packages/2023-fall-stat-135.r /tmp/r-packages/ -RUN r /tmp/r-packages/2023-fall-stat-135.r - -# Configure locking behavior -COPY file-locks /etc/rstudio/file-locks - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/shiny/image/class-libs.R b/deployments/shiny/image/class-libs.R deleted file mode 100644 index 1c9343537..000000000 --- a/deployments/shiny/image/class-libs.R +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env Rscript - -class_libs_install_version <- function(class_name, class_libs) { - print(paste("Installing packages for", class_name)) - for (i in seq(1, length(class_libs), 2)) { - installed_packages <- rownames(installed.packages()) - package_name = class_libs[i] - version = class_libs[i+1] - # Only install packages if they haven't already been installed! - # devtools doesn't do that by default - if (!package_name %in% installed_packages) { - print(paste("Installing", package_name, version)) - devtools::install_version(package_name, version, quiet=TRUE) - } else { - # FIXME: This ignores version incompatibilities :'( - print(paste("Not installing", package_name, " as it is already installed")) - } - } - print(paste("Done installing packages for", class_name)) -} diff --git a/deployments/shiny/image/environment.yml b/deployments/shiny/image/environment.yml deleted file mode 100644 index 88a784784..000000000 --- a/deployments/shiny/image/environment.yml +++ /dev/null @@ -1,22 +0,0 @@ -dependencies: -- appmode==0.9.0 -- ipywidgets==8.1.2 -- jupyter-archive==3.4.0 -- jupyter-resource-usage==1.0.1 -- jupyter-server-proxy==4.2.0 -- jupyter-rsession-proxy==2.2.0 -- jupyter-syncthing-proxy==1.0.3 -- jupyterhub==4.1.6 -- jupyterlab==4.0.11 -- nbgitpuller==1.2.1 -- notebook==7.0.7 -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets==5.9.0 -- syncthing==1.27.3 -- pip: - - git-credential-helpers==0.2 - - jupyter-shiny-proxy==1.1 - - otter-grader==2.2.7 - # Measure popularity of different packages in our hubs - # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda - #- git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling diff --git a/deployments/shiny/image/file-locks b/deployments/shiny/image/file-locks deleted file mode 100644 index 7b1a3fcf4..000000000 --- a/deployments/shiny/image/file-locks +++ /dev/null @@ -1,13 +0,0 @@ -# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking - -# rocker sets this to advisory, but this might be causing NFS issues. -# lets set it to the default (default: linkbased) -lock-type=linkbased - -# we'll also reduce the frequency by 1/3 -refresh-rate=60 -timeout-interval=90 - -# log attempts -# enable-logging=1 -# log-file=/tmp/rstudio-locking.log diff --git a/deployments/shiny/image/infra-requirements.txt b/deployments/shiny/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/shiny/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/shiny/image/install-mambaforge.bash b/deployments/shiny/image/install-mambaforge.bash deleted file mode 100755 index 38c5cefcd..000000000 --- a/deployments/shiny/image/install-mambaforge.bash +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION='23.1.0-1' - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/deployments/shiny/image/r-packages/2023-fall-stat-135.r b/deployments/shiny/image/r-packages/2023-fall-stat-135.r deleted file mode 100644 index ccfd659c2..000000000 --- a/deployments/shiny/image/r-packages/2023-fall-stat-135.r +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env Rscript - -# https://github.com/berkeley-dsep-infra/datahub/issues/4907 - -source("/tmp/class-libs.R") - -class_name = "Stat 135 Fall 2023" -class_libs = c( - "mosaicData", "0.20.3", - "learnr", "0.11.1" -) -class_libs_install_version(class_name, class_libs) diff --git a/deployments/shiny/image/r-packages/2023-spring-stat-20.r b/deployments/shiny/image/r-packages/2023-spring-stat-20.r deleted file mode 100644 index b48bab49c..000000000 --- a/deployments/shiny/image/r-packages/2023-spring-stat-20.r +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env Rscript - -source("/tmp/class-libs.R") - -# This might be used more broadly than just Stat 20. -class_name = "2023 Spring Stat 20" - -class_libs = c( - "DT", "0.26", - "shinyFiles", "0.9.3", - "shinyTime", "1.0.3" -) - -class_libs_install_version(class_name, class_libs) - -print(paste("Done installing packages for",class_name)) diff --git a/deployments/shiny/image/r-packages/2024-spring-gradebook.r b/deployments/shiny/image/r-packages/2024-spring-gradebook.r deleted file mode 100644 index 6984f872e..000000000 --- a/deployments/shiny/image/r-packages/2024-spring-gradebook.r +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env Rscript - -source("/tmp/class-libs.R") - -# This might be used more broadly than just Stat 20. -class_name = "2024 Spring Gradebook" - -class_libs = c( - "DT", "0.32", - "Hmisc", "5.1-1", - "purrr", "1.0.2", - "shinyFiles", "0.9.3", - "shinyTime", "1.0.3", - "shinyWidgets", "0.8.1", - "shinydashboard", "0.7.2" -) - -class_libs_install_version(class_name, class_libs) - -print(paste("Done installing packages for",class_name)) From e2fbb67c27937aaca29a3350586d587adad1e4d4 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 13:36:23 -0700 Subject: [PATCH 166/190] Remove shiny from CircleCI. --- .circleci/config.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae9cdeca5..570f856f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,12 +277,6 @@ jobs: hubploy deploy --timeout 30m r hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy shiny - command: | - hubploy deploy --timeout 30m shiny hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy workshop command: | @@ -394,15 +388,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: shiny - name: shiny image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod deploy: jobs: @@ -504,15 +489,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: shiny - name: shiny image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging # Build images only during the staging deploy. All merges # to prod need to go via staging, so prod should *never* # use images not built for staging. By enforcing this at the @@ -530,7 +506,6 @@ workflows: - julia hub image build # - logodev image build - publichealth image build - - shiny image build filters: branches: From 1b35a7f5e33921eb4e5cafd139cca93c3477704e Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 13:36:56 -0700 Subject: [PATCH 167/190] Deploy shiny hub via github actions. --- .github/workflows/deploy-hubs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index d34a68037..68c5ed957 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' From 3fcf1a9139c2d6ddfbd2d2111b4613c7abd7ab8d Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 13:37:11 -0700 Subject: [PATCH 168/190] Specify new shiny image configuration. --- deployments/shiny/hubploy.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/deployments/shiny/hubploy.yaml b/deployments/shiny/hubploy.yaml index 9b00330b6..0ba28d8cd 100644 --- a/deployments/shiny/hubploy.yaml +++ b/deployments/shiny/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-image - path: image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-image:PLACEHOLDER cluster: provider: gcloud From b6fb7f1343d9cc2b2fd2e1d9196dc4e7b1846574 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 13:40:00 -0700 Subject: [PATCH 169/190] Use shiny image for gradebook. --- deployments/gradebook/hubploy.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/deployments/gradebook/hubploy.yaml b/deployments/gradebook/hubploy.yaml index 4891ccd7d..0ba28d8cd 100644 --- a/deployments/gradebook/hubploy.yaml +++ b/deployments/gradebook/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-image - path: ../shiny/image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-image:PLACEHOLDER cluster: provider: gcloud From 32cf2dfb637b811272cd6f810d6c7aadc73ff88c Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 16:14:47 -0700 Subject: [PATCH 170/190] Also remove gradebook from CircleCI. --- .circleci/config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 570f856f0..98e7fe8bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,12 +229,6 @@ jobs: hubploy deploy --timeout 30m eecs hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy gradebook - command: | - hubploy deploy --timeout 30m gradebook hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy highschool hub command: | From fc86072edbfb928376f244ccc33839ccbd45a562 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 16:14:58 -0700 Subject: [PATCH 171/190] Delay adding shiny to this action until a later PR. --- .github/workflows/deploy-hubs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 68c5ed957..d34a68037 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' From e3bdae518e0d15f597f71460cd4776d69384bc09 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Fri, 6 Sep 2024 17:31:35 -0700 Subject: [PATCH 172/190] Add gradebook and shiny to github action. --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index d34a68037..c133fea96 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee) From 782019bc4166a208a164442644a54582f32ad2f6 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Sat, 7 Sep 2024 09:04:51 -0700 Subject: [PATCH 173/190] fix shiny image path --- deployments/gradebook/hubploy.yaml | 2 +- deployments/shiny/hubploy.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/gradebook/hubploy.yaml b/deployments/gradebook/hubploy.yaml index 0ba28d8cd..19f9ef626 100644 --- a/deployments/gradebook/hubploy.yaml +++ b/deployments/gradebook/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-image:PLACEHOLDER + - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/shiny/hubploy.yaml b/deployments/shiny/hubploy.yaml index 0ba28d8cd..19f9ef626 100644 --- a/deployments/shiny/hubploy.yaml +++ b/deployments/shiny/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-image:PLACEHOLDER + - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:PLACEHOLDER cluster: provider: gcloud From 44966f71c1fc93ddcb9708b84f8aaf1efcd4fc26 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Sat, 7 Sep 2024 16:07:25 +0000 Subject: [PATCH 174/190] update shiny image tag to 0f550a37d042: deployments/gradebook/hubploy.yaml deployments/shiny/hubploy.yaml --- deployments/gradebook/hubploy.yaml | 2 +- deployments/shiny/hubploy.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/gradebook/hubploy.yaml b/deployments/gradebook/hubploy.yaml index 19f9ef626..a14d38613 100644 --- a/deployments/gradebook/hubploy.yaml +++ b/deployments/gradebook/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:PLACEHOLDER + - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:0f550a37d042 cluster: provider: gcloud diff --git a/deployments/shiny/hubploy.yaml b/deployments/shiny/hubploy.yaml index 19f9ef626..a14d38613 100644 --- a/deployments/shiny/hubploy.yaml +++ b/deployments/shiny/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:PLACEHOLDER + - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:0f550a37d042 cluster: provider: gcloud From a4b20da5a86ea7e93687e210801ed6955ddfc9c1 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Sat, 7 Sep 2024 09:16:11 -0700 Subject: [PATCH 175/190] fixing image name stanza --- deployments/gradebook/hubploy.yaml | 2 +- deployments/shiny/hubploy.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/gradebook/hubploy.yaml b/deployments/gradebook/hubploy.yaml index 19f9ef626..158ee1d5c 100644 --- a/deployments/gradebook/hubploy.yaml +++ b/deployments/gradebook/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:0f550a37d042 cluster: provider: gcloud diff --git a/deployments/shiny/hubploy.yaml b/deployments/shiny/hubploy.yaml index 19f9ef626..158ee1d5c 100644 --- a/deployments/shiny/hubploy.yaml +++ b/deployments/shiny/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/shiny-user-image:0f550a37d042 cluster: provider: gcloud From 2666b189557ab91debdba87c496c3f31dbf85c60 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Mon, 9 Sep 2024 11:04:39 -0700 Subject: [PATCH 176/190] moving dev to its own image repos --- .circleci/config.yml | 52 ------- deployments/dev/hubploy.yaml | 13 +- deployments/dev/image/README.md | 5 + deployments/dev/images/default/Dockerfile | 139 ------------------ deployments/dev/images/default/Rprofile.site | 23 --- deployments/dev/images/default/apt.txt | 37 ----- deployments/dev/images/default/class-libs.R | 20 --- .../dev/images/default/environment.yml | 21 --- .../dev/images/default/infra-requirements.txt | 29 ---- .../images/default/install-mambaforge.bash | 41 ------ deployments/dev/images/default/install.R | 18 --- deployments/dev/images/default/rsession.conf | 2 - .../dev/images/secondary/Rprofile.site | 23 --- deployments/dev/images/secondary/apt.txt | 18 --- .../dev/images/secondary/environment.yml | 48 ------ .../images/secondary/infra-requirements.txt | 29 ---- deployments/dev/images/secondary/install.R | 33 ----- .../dev/images/secondary/rsession.conf | 2 - deployments/dev/images/secondary/runtime.txt | 1 - 19 files changed, 7 insertions(+), 547 deletions(-) create mode 100644 deployments/dev/image/README.md delete mode 100644 deployments/dev/images/default/Dockerfile delete mode 100644 deployments/dev/images/default/Rprofile.site delete mode 100644 deployments/dev/images/default/apt.txt delete mode 100644 deployments/dev/images/default/class-libs.R delete mode 100644 deployments/dev/images/default/environment.yml delete mode 100644 deployments/dev/images/default/infra-requirements.txt delete mode 100755 deployments/dev/images/default/install-mambaforge.bash delete mode 100755 deployments/dev/images/default/install.R delete mode 100644 deployments/dev/images/default/rsession.conf delete mode 100644 deployments/dev/images/secondary/Rprofile.site delete mode 100644 deployments/dev/images/secondary/apt.txt delete mode 100644 deployments/dev/images/secondary/environment.yml delete mode 100644 deployments/dev/images/secondary/infra-requirements.txt delete mode 100755 deployments/dev/images/secondary/install.R delete mode 100644 deployments/dev/images/secondary/rsession.conf delete mode 100644 deployments/dev/images/secondary/runtime.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 98e7fe8bb..09f4b12bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,12 +205,6 @@ jobs: hubploy deploy --timeout 30m datahub hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy dev - command: | - hubploy deploy --timeout 30m dev hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy dlab command: | @@ -247,12 +241,6 @@ jobs: hubploy deploy --timeout 30m julia hub ${CIRCLE_BRANCH} no_output_timeout: 30m -# - run: -# name: Deploy logodev -# command: | -# hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH} -# no_output_timeout: 30m - - run: name: Deploy prob140 command: | @@ -317,17 +305,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - # This has no effect yet because docker-py doesn't support buildkit. - #environment: - # DOCKER_BUILDKIT: 1 - deployment: dev - name: dev image build - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: edx name: edx image build @@ -364,15 +341,6 @@ workflows: ignore: - staging - prod -# - hubploy/build-image: -# deployment: logodev -# name: logodev hub image build -# # Filters can only be per-job? wtf -# filters: -# branches: -# ignore: -# - staging -# - prod - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -420,15 +388,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: dev - name: dev image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging - hubploy/build-image: deployment: edx name: edx image build @@ -465,15 +424,6 @@ workflows: branches: only: - staging -# - hubploy/build-image: -# deployment: logodev -# name: logodev image build -# push: true -# # Filters can only be per-job? wtf -# filters: -# branches: -# only: -# - staging - hubploy/build-image: deployment: publichealth name: publichealth image build @@ -493,12 +443,10 @@ workflows: - data100 image build - data102 image build - datahub image build - - dev image build - edx image build - eecs image build - ischool image build - julia hub image build -# - logodev image build - publichealth image build filters: diff --git a/deployments/dev/hubploy.yaml b/deployments/dev/hubploy.yaml index 73a41f621..ea38158a1 100644 --- a/deployments/dev/hubploy.yaml +++ b/deployments/dev/hubploy.yaml @@ -1,16 +1,7 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-user-image - path: images/default - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary - path: images/secondary - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-primary-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/dev/image/README.md b/deployments/dev/image/README.md new file mode 100644 index 000000000..8a53d6ab6 --- /dev/null +++ b/deployments/dev/image/README.md @@ -0,0 +1,5 @@ +# Dev Image + +These images are now located [in their](https://github.com/berkeley-dsep-infra/dev-primary-image) [own repos](https://github.com/berkeley-dsep-infra/dev-secondary-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/dev-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/dev/images/default/Dockerfile b/deployments/dev/images/default/Dockerfile deleted file mode 100644 index bf9307b03..000000000 --- a/deployments/dev/images/default/Dockerfile +++ /dev/null @@ -1,139 +0,0 @@ -FROM buildpack-deps:jammy-scm as base - -# Set up common env variables -ENV TZ=America/Los_Angeles -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEBIAN_FRONTEND=noninteractive -ENV NB_USER jovyan -ENV NB_UID 1000 -# These are used by the python, R, and final stages -ENV CONDA_DIR /srv/conda -ENV R_LIBS_USER /srv/r - -RUN apt-get -qq update --yes && \ - apt-get -qq install --yes locales && \ - echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen - -RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER} - -# Install all apt packages -COPY apt.txt /tmp/apt.txt -RUN apt-get -qq update --yes && \ - apt-get -qq install --yes --no-install-recommends \ - $(grep -v ^# /tmp/apt.txt) && \ - apt-get -qq purge && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* - -# Install R. -# These packages must be installed into the base stage since they are in system -# paths rather than /srv. -# Pre-built R packages from rspm are built against system libs in jammy. -ENV R_VERSION=4.4.1-1.2204.0 -ENV LITTLER_VERSION=0.3.19-1.2204.0 -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 -RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" > /etc/apt/sources.list.d/cran.list -RUN curl --silent --location --fail https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc > /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc -RUN apt-get update -qq --yes > /dev/null && \ - apt-get install --yes -qq \ - r-base-core=${R_VERSION} \ - r-base-dev=${R_VERSION} \ - littler=${LITTLER_VERSION} > /dev/null - -ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2024.04.2-764-amd64.deb -RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \ - apt install --no-install-recommends --yes /tmp/rstudio.deb && \ - rm /tmp/rstudio.deb - -ENV SHINY_SERVER_URL https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-1.5.20.1002-amd64.deb -RUN curl --silent --location --fail ${SHINY_SERVER_URL} > /tmp/shiny.deb && \ - apt install --no-install-recommends --yes /tmp/shiny.deb && \ - rm /tmp/shiny.deb - -# Set CRAN mirror to rspm before we install anything -COPY Rprofile.site /usr/lib/R/etc/Rprofile.site -# RStudio needs its own config -COPY rsession.conf /etc/rstudio/rsession.conf - -# R_LIBS_USER is set by default in /etc/R/Renviron, which RStudio loads. -# We uncomment the default, and set what we wanna - so it picks up -# the packages we install. Without this, RStudio doesn't see the packages -# that R does. -# Stolen from https://github.com/jupyterhub/repo2docker/blob/6a07a48b2df48168685bb0f993d2a12bd86e23bf/repo2docker/buildpacks/r.py -# To try fight https://community.rstudio.com/t/timedatectl-had-status-1/72060, -# which shows up sometimes when trying to install packages that want the TZ -# timedatectl expects systemd running, which isn't true in our containers -RUN sed -i -e '/^R_LIBS_USER=/s/^/#/' /etc/R/Renviron && \ - echo "R_LIBS_USER=${R_LIBS_USER}" >> /etc/R/Renviron && \ - echo "TZ=${TZ}" >> /etc/R/Renviron - -# ============================================================================= -# This stage exists to build /srv/r. -FROM base as srv-r - -# Create user owned R libs dir -# This lets users temporarily install packages -RUN install -d -o ${NB_USER} -g ${NB_USER} ${R_LIBS_USER} - -# Install R libraries as our user -USER ${NB_USER} - -COPY class-libs.R /tmp/class-libs.R -RUN mkdir -p /tmp/r-packages - -# Our install.R needs devtools which needs install2.r which needs docopt. -# install2.r is not reproducible, but our install.R script is. -RUN Rscript -e "install.packages('docopt')" -RUN /usr/lib/R/site-library/littler/examples/install2.r devtools - -# Install all our base R packages -COPY install.R /tmp/install.R -RUN /tmp/install.R && rm -rf /tmp/downloaded_packages - -# ============================================================================= -# This stage exists to build /srv/conda. -FROM base as srv-conda - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -# Install conda environment as our user -USER ${NB_USER} - -ENV PATH ${CONDA_DIR}/bin:$PATH - -COPY infra-requirements.txt /tmp/infra-requirements.txt -COPY environment.yml /tmp/environment.yml - -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -# ============================================================================= -# This stage consumes base and import /srv/r and /srv/conda. -FROM base as final -COPY --from=srv-r /srv/r /srv/r -COPY --from=srv-conda /srv/conda /srv/conda - -# Install IR kernelspec. Requires python and R. -ENV PATH ${CONDA_DIR}/bin:${PATH}:${R_LIBS_USER}/bin -RUN R -e "IRkernel::installspec(user = FALSE, prefix='${CONDA_DIR}')" - -# clear out /tmp -USER root -#COPY postBuild /tmp/postBuild -#RUN chmod +x /tmp/postBuild -#RUN /tmp/postBuild -RUN rm -rf /tmp/* - - -USER ${NB_USER} -WORKDIR /home/${NB_USER} - -EXPOSE 8888 - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/dev/images/default/Rprofile.site b/deployments/dev/images/default/Rprofile.site deleted file mode 100644 index a75374482..000000000 --- a/deployments/dev/images/default/Rprofile.site +++ /dev/null @@ -1,23 +0,0 @@ -# Use RStudio's CRAN mirror to get binary packages. "latest" just means it has -# all available versions. We can specify version numbers in -# devtools::install_version. -options( - repos = c( - CRAN = "https://packagemanager.rstudio.com/all/__linux__/jammy/latest" - ) -) - -# RStudio sets the HTTPUserAgent option which causes it to download binaries, -# rather than source, from the Posit Public Package Manager. In order for the -# RStudio terminal, Jupyter terminal, Jupyter R notebook, or anything else to -# do the same, we must manually set HTTPUserAgent. This speeds up package -# installation. -# We fake rstudio version because `RStudio.Version()$version` is only available -# when we are in RStudio. -options( - HTTPUserAgent = sprintf( - "RStudio Server (%s); R (%s)", - "2024.04.2.764", - paste(getRversion(), R.version$platform, R.version$arch, R.version$os) - ) -) diff --git a/deployments/dev/images/default/apt.txt b/deployments/dev/images/default/apt.txt deleted file mode 100644 index 5682ae796..000000000 --- a/deployments/dev/images/default/apt.txt +++ /dev/null @@ -1,37 +0,0 @@ -# our entry point -tini - -# useful utilities -tar -less -vim - -# for nbconvert, knitr, etc. -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# provides FandolSong-Regular.otf for issue #2714 -texlive-lang-chinese - -# for notebook-as-pdf -libx11-xcb1 -libxtst6 -libxrandr2 -libasound2 -libpangocairo-1.0-0 -libatk1.0-0 -libatk-bridge2.0-0 -libgtk-3-0 -libnss3 -libxss1 -fonts-noto-color-emoji - -# for rstudio -#psmisc -#sudo -#libapparmor1 -lsb-release -libclang-dev - -# Used by littler for IRkernel::installspec -libzmq5 diff --git a/deployments/dev/images/default/class-libs.R b/deployments/dev/images/default/class-libs.R deleted file mode 100644 index 1c9343537..000000000 --- a/deployments/dev/images/default/class-libs.R +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env Rscript - -class_libs_install_version <- function(class_name, class_libs) { - print(paste("Installing packages for", class_name)) - for (i in seq(1, length(class_libs), 2)) { - installed_packages <- rownames(installed.packages()) - package_name = class_libs[i] - version = class_libs[i+1] - # Only install packages if they haven't already been installed! - # devtools doesn't do that by default - if (!package_name %in% installed_packages) { - print(paste("Installing", package_name, version)) - devtools::install_version(package_name, version, quiet=TRUE) - } else { - # FIXME: This ignores version incompatibilities :'( - print(paste("Not installing", package_name, " as it is already installed")) - } - } - print(paste("Done installing packages for", class_name)) -} diff --git a/deployments/dev/images/default/environment.yml b/deployments/dev/images/default/environment.yml deleted file mode 100644 index b8fabc4d7..000000000 --- a/deployments/dev/images/default/environment.yml +++ /dev/null @@ -1,21 +0,0 @@ -dependencies: -- python=3.11.* # mamba installation with py3.11 fails: No module named 'conda.cliconda' -- pip=23.1.* -- nodejs=18.* -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* - -- jupyter-server-proxy==4.2.0 -- jupyter-rsession-proxy==2.2.0 - -- syncthing==1.23.5 -- pyppeteer==1.0.2 - -# for nbconvert -- pandoc==3.1.3 - -# for jupyter-tree-download -- zip==3.0 -- pip: - - -r infra-requirements.txt - - jupyter-shiny-proxy==1.1 diff --git a/deployments/dev/images/default/infra-requirements.txt b/deployments/dev/images/default/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/dev/images/default/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/dev/images/default/install-mambaforge.bash b/deployments/dev/images/default/install-mambaforge.bash deleted file mode 100755 index ddab8ceb1..000000000 --- a/deployments/dev/images/default/install-mambaforge.bash +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# -# Download and install a pinned version of mambaforge. -# - -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION=24.3.0-0 - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# Empty the conda history file, which seems to result in some effective pinning -# of packages in the initial env, which we don't intend. This file must not be -# removed. -> ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf ${HOME}/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/deployments/dev/images/default/install.R b/deployments/dev/images/default/install.R deleted file mode 100755 index 4888be50a..000000000 --- a/deployments/dev/images/default/install.R +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env r - -# Install devtools so we can install versioned packages -#install.packages("devtools") - -source("/tmp/class-libs.R") - -# R packages to be installed that aren't from apt -# Combination of informal requests & rocker image suggestions -# Some of these were already in datahub image -cran_packages = c( - "BiocManager", "1.30.21", - "IRkernel", "1.3.2", - "rmarkdown", "2.22", - "shiny", "1.7.4" -) - -class_libs_install_version("Base packages", cran_packages) diff --git a/deployments/dev/images/default/rsession.conf b/deployments/dev/images/default/rsession.conf deleted file mode 100644 index 1f82b6b54..000000000 --- a/deployments/dev/images/default/rsession.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Use binary packages! -r-cran-repos=https://packagemanager.rstudio.com/all/__linux__/jammy/latest diff --git a/deployments/dev/images/secondary/Rprofile.site b/deployments/dev/images/secondary/Rprofile.site deleted file mode 100644 index 51f9daba8..000000000 --- a/deployments/dev/images/secondary/Rprofile.site +++ /dev/null @@ -1,23 +0,0 @@ -# Use RStudio's CRAN mirror to get binary packages. "latest" just means it has -# all available versions. We can specify version numbers in -# devtools::install_version. -options( - repos = c( - CRAN = "https://packagemanager.rstudio.com/all/__linux__/jammy/latest" - ) -) - -# RStudio sets the HTTPUserAgent option which causes it to download binaries, -# rather than source, from the Posit Public Package Manager. In order for the -# RStudio terminal, Jupyter terminal, Jupyter R notebook, or anything else to -# do the same, we must manually set HTTPUserAgent. This speeds up package -# installation. -# We fake rstudio version because `RStudio.Version()$version` is only available -# when we are in RStudio. -options( - HTTPUserAgent = sprintf( - "RStudio Server (%s); R (%s)", - "2022.7.1.554", - paste(getRversion(), R.version$platform, R.version$arch, R.version$os) - ) -) diff --git a/deployments/dev/images/secondary/apt.txt b/deployments/dev/images/secondary/apt.txt deleted file mode 100644 index 86941ae7d..000000000 --- a/deployments/dev/images/secondary/apt.txt +++ /dev/null @@ -1,18 +0,0 @@ -# useful utilities -tar -less -vim - -# for nbconvert, knitr, etc. -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# provides FandolSong-Regular.otf for issue #2714 -texlive-lang-chinese - -# for rstudio -lsb-release -libclang-dev - -# Used by littler for IRkernel::installspec -libzmq5 diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml deleted file mode 100644 index f7d4337af..000000000 --- a/deployments/dev/images/secondary/environment.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: dashboard-image - -channels: -- conda-forge -- pytorch - -dependencies: -- python==3.11.* -- git==2.39.1 -- jupyterhub==4.1.6 -- jupyter_app_launcher==0.2.1 -- jupyter-cache==1.0.0 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -# Install myst for generating dashboards -- jupyterlab-myst==2.4.2 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 -- folium==0.14.0 -- ipywidgets==8.0.7 -- jupyter-archive==3.4.0 -- matplotlib==3.7.1 -# Install myst markdown for generating dashboards -- mystmd==1.2.9 -- numpy==1.24.2 -- pandas==2.0.2 -- plotly==5.13.1 -- requests==2.28.2 -- scikit-image==0.19.3 -- seaborn==0.12.2 -# Spring 2024 Econ 148 Packages -- geopandas==0.14.2 -- geopy==2.4.1 -- pycountry==22.3.5 -# Install voila and voici for generating voila dashboards -- voila==0.5.7 -- voici==0.6.1 -# Adding more data tools to make this image comprehensive -- dash==2.17.1 -- bokeh==3.4.0 -- panel==1.4.5 -- datashader==0.16.3 -- streamlit=1.38.0 -#- jupyter-containds=0.2.2 diff --git a/deployments/dev/images/secondary/infra-requirements.txt b/deployments/dev/images/secondary/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/dev/images/secondary/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/dev/images/secondary/install.R b/deployments/dev/images/secondary/install.R deleted file mode 100755 index d4e8763e4..000000000 --- a/deployments/dev/images/secondary/install.R +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env R - -# Function to install R packages with specific versions -class_libs_install_version <- function(class_name, class_libs) { - print(paste("Installing packages for", class_name)) - for (i in seq(1, length(class_libs), 2)) { - installed_packages <- rownames(installed.packages()) - package_name = class_libs[i] - version = class_libs[i+1] - # Only install packages if they haven't already been installed! - # devtools doesn't do that by default - if (!package_name %in% installed_packages) { - print(paste("Installing", package_name, version)) - devtools::install_version(package_name, version, quiet=TRUE) - } else { - # FIXME: This ignores version incompatibilities :'( - print(paste("Not installing", package_name, " as it is already installed")) - } - } - print(paste("Done installing packages for", class_name)) -} - -# R packages to be installed that aren't from apt -# Combination of informal requests & rocker image suggestions -# Some of these were already in datahub image -cran_packages = c( - "BiocManager", "1.30.21", - "IRkernel", "1.3.2", - "rmarkdown", "2.22", - "shiny", "1.7.4" -) - -class_libs_install_version("Base packages", cran_packages) diff --git a/deployments/dev/images/secondary/rsession.conf b/deployments/dev/images/secondary/rsession.conf deleted file mode 100644 index 1f82b6b54..000000000 --- a/deployments/dev/images/secondary/rsession.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Use binary packages! -r-cran-repos=https://packagemanager.rstudio.com/all/__linux__/jammy/latest diff --git a/deployments/dev/images/secondary/runtime.txt b/deployments/dev/images/secondary/runtime.txt deleted file mode 100644 index fbfde9239..000000000 --- a/deployments/dev/images/secondary/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -r-4.3.1-2023-08-01 From 8a90c608dea4092976a6d36fc824a3bca053833f Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Mon, 9 Sep 2024 11:06:34 -0700 Subject: [PATCH 177/190] wonder if we will have issues deploying the secondary image w/no hash in here --- deployments/dev/config/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml index 8c634dc6e..58e58144e 100644 --- a/deployments/dev/config/common.yaml +++ b/deployments/dev/config/common.yaml @@ -74,7 +74,7 @@ jupyterhub: - display_name: "Dashboard image" description: "A newer repo2docker-based image with a primary focus of building dashboards." kubespawner_override: - image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary:36280727a + image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image - display_name: "1524699: DataHub Infrastructure" slug: "1524699" description: "Regular image with per-course subpath." From f6e5edde350f726d7db27d6a76389ae10217929e Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Mon, 9 Sep 2024 11:11:31 -0700 Subject: [PATCH 178/190] enabling dev hub cicd --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index c133fea96..d14c90b51 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev) From b1858db5ff6bd53997c857fccd92b6d9796960cb Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Mon, 9 Sep 2024 11:57:15 -0700 Subject: [PATCH 179/190] fixing dev hubploy --- deployments/dev/hubploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/dev/hubploy.yaml b/deployments/dev/hubploy.yaml index ea38158a1..60990c8c0 100644 --- a/deployments/dev/hubploy.yaml +++ b/deployments/dev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-primary-image:PLACEHOLDER - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-primary-image:6000a5694eab + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image:df11f4f1caa1 cluster: provider: gcloud From adbce295900613b167a47ba8b6004a20bab2c798 Mon Sep 17 00:00:00 2001 From: Balaji Alwar <balajialwar@berkeley.edu> Date: Mon, 9 Sep 2024 11:57:58 -0700 Subject: [PATCH 180/190] Link the github issue --- deployments/datahub/config/common.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index c51f1cd34..f39a81f1e 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -209,7 +209,6 @@ jupyterhub: mem_limit: 4096M mem_guarantee: 4096M -# Data C131A, https://uctech.slack.com/archives/CC3NKQA3B/p1725853680045669 (Reported as bug by Stat 131A instructor via Slack) - course::1537932: + course::1537932: # Data C131A, https://github.com/berkeley-dsep-infra/datahub/issues/6121 mem_limit: 2048M mem_guarantee: 2048M \ No newline at end of file From 879c27c73c0f3e1fd72f3b5d6a2c21f9ccbe126a Mon Sep 17 00:00:00 2001 From: Balaji Alwar <balajialwar@berkeley.edu> Date: Mon, 9 Sep 2024 12:06:28 -0700 Subject: [PATCH 181/190] Move Stanza for Stat 131A to R Hub --- deployments/datahub/config/common.yaml | 4 ---- deployments/r/config/common.yaml | 11 +++++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index f39a81f1e..6eae3623f 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -208,7 +208,3 @@ jupyterhub: course::1539545: # Demog 213, https://github.com/berkeley-dsep-infra/datahub/issues/6094 mem_limit: 4096M mem_guarantee: 4096M - - course::1537932: # Data C131A, https://github.com/berkeley-dsep-infra/datahub/issues/6121 - mem_limit: 2048M - mem_guarantee: 2048M \ No newline at end of file diff --git a/deployments/r/config/common.yaml b/deployments/r/config/common.yaml index 7dbe64ff5..ebe79685b 100644 --- a/deployments/r/config/common.yaml +++ b/deployments/r/config/common.yaml @@ -89,3 +89,14 @@ jupyterhub: limit: 1G image: {} + custom: + group_profiles: + # DataHub Infrastructure staff + # https://bcourses.berkeley.edu/courses/1524699/groups#tab-80607 + course::1524699::group::all-admins: + mem_limit: 2048M + mem_guarantee: 2048M + course::1537932: # Data C131A, https://github.com/berkeley-dsep-infra/datahub/issues/6121 + mem_limit: 2048M + mem_guarantee: 2048M + From 9104afe0e8b3a40bbedf127cb9ae9bb0e77f4fed Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Mon, 9 Sep 2024 12:51:24 -0700 Subject: [PATCH 182/190] fixing secondary image for dev --- deployments/dev/config/common.yaml | 2 +- deployments/dev/hubploy.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml index 58e58144e..baa137c55 100644 --- a/deployments/dev/config/common.yaml +++ b/deployments/dev/config/common.yaml @@ -74,7 +74,7 @@ jupyterhub: - display_name: "Dashboard image" description: "A newer repo2docker-based image with a primary focus of building dashboards." kubespawner_override: - image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image + image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image:df11f4f1caa1 - display_name: "1524699: DataHub Infrastructure" slug: "1524699" description: "Regular image with per-course subpath." diff --git a/deployments/dev/hubploy.yaml b/deployments/dev/hubploy.yaml index 60990c8c0..e69af6021 100644 --- a/deployments/dev/hubploy.yaml +++ b/deployments/dev/hubploy.yaml @@ -1,3 +1,7 @@ +# you will also need to update config/common.yaml to include the following for +# the secondary image tag: +# kubespawner_override: +# image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary-image:df11f4f1caa1 images: images: - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-primary-image:6000a5694eab From d257546e6b3e1bc319285901e118308bdfea92a3 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Mon, 9 Sep 2024 22:59:39 +0000 Subject: [PATCH 183/190] update biology image tag to 8540bb31ea44: deployments/biology/hubploy.yaml --- deployments/biology/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/biology/hubploy.yaml b/deployments/biology/hubploy.yaml index 83c9f483a..2a4021fdb 100644 --- a/deployments/biology/hubploy.yaml +++ b/deployments/biology/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:24ed6d3bf1a6 + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:8540bb31ea44 cluster: provider: gcloud From 914d201d81e571175fe7e6a72acde4a31027066d Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:19:45 +0000 Subject: [PATCH 184/190] update stat20 image tag to 130d6247d491: deployments/stat20/hubploy.yaml --- deployments/stat20/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stat20/hubploy.yaml b/deployments/stat20/hubploy.yaml index fdf2ad5ca..16005bca4 100644 --- a/deployments/stat20/hubploy.yaml +++ b/deployments/stat20/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:feaa2df801d9 + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/stat20-user-image:130d6247d491 cluster: provider: gcloud From 787f9ed7ffae5de23b39ecfcabb150806e10ae66 Mon Sep 17 00:00:00 2001 From: Ryan Lovett <rylo@berkeley.edu> Date: Tue, 10 Sep 2024 11:15:27 -0700 Subject: [PATCH 185/190] Add docs on transitioning user images. --- docs/_quarto.yml | 1 + docs/admins/howto/transition-image.qmd | 96 ++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 docs/admins/howto/transition-image.qmd diff --git a/docs/_quarto.yml b/docs/_quarto.yml index dbc9ba1b1..eeb5b82f5 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -50,6 +50,7 @@ website: - admins/howto/rebuild-hub-image.qmd - admins/howto/rebuild-postgres-image.qmd - admins/howto/new-image.qmd + - admins/howto/transition-image.qmd - admins/howto/new-packages.qmd - admins/howto/course-config.qmd - admins/howto/calendar-scaler.qmd diff --git a/docs/admins/howto/transition-image.qmd b/docs/admins/howto/transition-image.qmd new file mode 100644 index 000000000..c860a7e50 --- /dev/null +++ b/docs/admins/howto/transition-image.qmd @@ -0,0 +1,96 @@ +--- +title: Transition Single User Image to GitHub Actions +--- + +Single user images have been maintained within the main datahub repo since its inception, however we decided to move them into their own repositories. It will make testing notebooks easier, and we will be able to delegate write access to course staff if necessary. + +This is the process for transitioning images to their own repositories. Eventually, once all repositories have been migrated, we can update our documentation on creating new single user image repositories, and maintaining them. + + +## Prerequisites + +You will need to install `git-filter-repo`. + +```bash +wget -O ~/bin/git-filter-repo https://raw.githubusercontent.com/newren/git-filter-repo/main/git-filter-repo + chmod +x ~/bin/git-filter-repo +``` + +## Create the repository + +1. Go to https://github.com/berkeley-dsep-infra/hub-user-image-template. Click "Use this template" > "Create a new repository". +1. Set the owner to `berkeley-dsep-infra`. Name the image `{hub}-user-image`, or some approximation of there are multiple images per hub. +1. Click create repository. +1. In the new repository, visit Settings > Secrets and variables > Actions > Variables tab. Create new variables: + a. Set HUB to the hub deployment, e.g. `shiny`. + b. Set IMAGE to ucb-datahub-2018/user-images/{hub}-user-image, e.g. `ucb-datahub-2018/user-images/shiny-user-image`. +1. Fork the new image repo into your own github account. + +## Preparing working directories + +As part of this process, we will pull the previous image's git history into the new image repo. + +1. Clone the *datahub* repo into a new directory named after the image repo. + ```bash + git clone git@github.com:berkeley-dsep-infra/datahub.git {hub}-user-image --origin source + ``` +1. Change into the directory. +1. Run `git-filter-repo`: + ```bash + git filter-repo --subdirectory-filter deployments/{hub}/image --force + ``` +1. Add new git remotes: + ```bash + git remote add origin git@github.com:{your_git_account}/{hub}-user-image.git + git remote add upstream git@github.com:berkeley-dsep-infra/{hub}-user-image.git + ``` +1. Pull in the contents of the new user image that was created from the template. + ```bash + git fetch upstream + git checkout main # pulls in .github + ``` + +1. Merge the contents of the previous datahub image with the new user image. + ```bash + git rm environment.yml + git commit -m "Remove default environment.yml file." + git merge staging --allow-unrelated-histories -m 'Bringing in image directory from deployment repo' + git push upstream main + git push origin main + ``` + +## Preparing continuous integration + +1. In the berkeley-dsep-infra org settings, https://github.com/organizations/berkeley-dsep-infra/settings/profile, visit Secrets and variables > Actions, https://github.com/organizations/berkeley-dsep-infra/settings/secrets/actions. Edit the secrets for DATAHUB_CREATE_PR and GAR_SECRET_KEY, and enable the new repo to access each. + +1. In the datahub repo, in one PR: + a. remove the hub deployment steps for the hub: + - Deploy {hub} + - hubploy/build-image {hub} image build (x2) + + a. under `deployments/{hub}/hubploy.yaml`, remove the registry entry, and set the image_name to have `PLACEHOLDER` for the tag. + + a. In the datahub repo, under the deployment image directory, update the README to point to the new repo. Delete everything else in the image directory. + +1. Merge these changes to datahub staging. + +1. Make a commit to trigger a build of the image in its repo. + +1. In a PR in the datahub repo, under .github/workflows/deploy-hubs.yaml, add the hub with the new image under `determine-hub-deployments.py --only-deploy`. + +1. Make another commit to the image repo to trigger a build. When these jobs finish, a commit will be pushed to the datahub repo. Make a PR, and merge to staging after canceling the CircleCI builds. (these builds are an artifact of the CircleCI-to-GitHub migration -- we won't need to do that long term) + +1. Subscribe the *#ucb-datahubs-bots* channel in UC Tech slack to the repo. + ```bash + /github subscribe berkeley-dsep-infra/<repo> + ``` + +## Making changes + +Once the image repo is set up, you will need to follow this procedure to update it and make it available to the hub. + +1. Make a change in your fork of the image repo. +1. Make a pull request to the repo in `berkeley-dsep-infra`. This will trigger a github action that will test to see if the image builds successfully. +1. If the build succeeds, someone with sufficient access (DataHub staff, or course staff with elevated privileges) can merge the PR. This will trigger another build, and will then push the image to the image registry. +1. In order for the newly built and pushed image to be referenced by datahub, you will need to make PR at datahub. Visit the previous merge action's *update-deployment-image-tag* entry and expand the *Create feature branch, add, commit and push changes* step. Find the URL beneath, *Create a pull request for 'update-{hub}-image-tag-{slug}*, and visit it. This will draft a new PR at datahub for you to create. +1. Once the PR is submitted, an action will run. It is okay if CircleCI-related tasks fail here. Merge the PR into staging once the action is complete. From 9c1086a9eaa152ca6b7c11cda4cafb0613d28c1c Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Tue, 10 Sep 2024 23:48:39 +0000 Subject: [PATCH 186/190] update data101 image tag to 010ca9abfb71: deployments/data101/hubploy.yaml --- deployments/data101/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/data101/hubploy.yaml b/deployments/data101/hubploy.yaml index cee47a052..a43aea791 100644 --- a/deployments/data101/hubploy.yaml +++ b/deployments/data101/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:0e1d41cf9936 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data101-user-image:010ca9abfb71 cluster: provider: gcloud From 366ac119c8dfc40b50acbc350ae6a23f665ccd56 Mon Sep 17 00:00:00 2001 From: "Image Builder Bot[tm]" <github-action-bot@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:12:10 +0000 Subject: [PATCH 187/190] update biology image tag to 16c6e661dcd5: deployments/biology/hubploy.yaml --- deployments/biology/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/biology/hubploy.yaml b/deployments/biology/hubploy.yaml index 2a4021fdb..35803ba46 100644 --- a/deployments/biology/hubploy.yaml +++ b/deployments/biology/hubploy.yaml @@ -1,5 +1,5 @@ images: - image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:8540bb31ea44 + image_name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/biology-user-image:16c6e661dcd5 cluster: provider: gcloud From be9fd6e09319d0f37867d31674c0a345da2a1a04 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 11 Sep 2024 10:06:12 -0700 Subject: [PATCH 188/190] moving publichealth to its own repo --- .circleci/config.yml | 25 ----- deployments/publichealth/hubploy.yaml | 6 -- deployments/publichealth/image/Dockerfile | 101 ------------------ deployments/publichealth/image/README.md | 5 + deployments/publichealth/image/class-libs.R | 20 ---- .../publichealth/image/environment.yml | 17 --- deployments/publichealth/image/file-locks | 13 --- .../publichealth/image/infra-requirements.txt | 29 ----- .../image/install-mambaforge.bash | 42 -------- deployments/publichealth/image/install.R | 6 -- .../Untitled-checkpoint.ipynb | 6 -- .../image/r-packages/2021-spring-phw-272a.r | 11 -- .../publichealth/image/r-packages/ottr.r | 6 -- .../publichealth/image/r-packages/ph-142.r | 29 ----- .../publichealth/image/r-packages/ph-252.r | 32 ------ .../publichealth/image/r-packages/ph-290.r | 16 --- 16 files changed, 5 insertions(+), 359 deletions(-) delete mode 100644 deployments/publichealth/image/Dockerfile create mode 100644 deployments/publichealth/image/README.md delete mode 100644 deployments/publichealth/image/class-libs.R delete mode 100644 deployments/publichealth/image/environment.yml delete mode 100644 deployments/publichealth/image/file-locks delete mode 100644 deployments/publichealth/image/infra-requirements.txt delete mode 100755 deployments/publichealth/image/install-mambaforge.bash delete mode 100755 deployments/publichealth/image/install.R delete mode 100644 deployments/publichealth/image/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 deployments/publichealth/image/r-packages/2021-spring-phw-272a.r delete mode 100755 deployments/publichealth/image/r-packages/ottr.r delete mode 100755 deployments/publichealth/image/r-packages/ph-142.r delete mode 100755 deployments/publichealth/image/r-packages/ph-252.r delete mode 100755 deployments/publichealth/image/r-packages/ph-290.r diff --git a/.circleci/config.yml b/.circleci/config.yml index 09f4b12bb..d6f6f6c01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -247,12 +247,6 @@ jobs: hubploy deploy --timeout 30m prob140 hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy publichealth - command: | - hubploy deploy --timeout 30m publichealth hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy R command: | @@ -341,15 +335,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: publichealth - name: publichealth image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod deploy: jobs: @@ -424,15 +409,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: publichealth - name: publichealth image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging # Build images only during the staging deploy. All merges # to prod need to go via staging, so prod should *never* # use images not built for staging. By enforcing this at the @@ -447,7 +423,6 @@ workflows: - eecs image build - ischool image build - julia hub image build - - publichealth image build filters: branches: diff --git a/deployments/publichealth/hubploy.yaml b/deployments/publichealth/hubploy.yaml index fd6bb6108..ffc67a75e 100644 --- a/deployments/publichealth/hubploy.yaml +++ b/deployments/publichealth/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/publichealth-user-image - path: image - registry: - provider: gcloud - gcloud: - project: ucb-datahub-2018 - service_key: gcr-key.json cluster: provider: gcloud diff --git a/deployments/publichealth/image/Dockerfile b/deployments/publichealth/image/Dockerfile deleted file mode 100644 index 802e5baec..000000000 --- a/deployments/publichealth/image/Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -FROM ghcr.io/rocker-org/geospatial:4.4.1 - -ENV NB_USER rstudio -ENV NB_UID 1000 -ENV CONDA_DIR /srv/conda - -# Set ENV for all programs... -ENV PATH ${CONDA_DIR}/bin:$PATH - -# And set ENV for R! It doesn't read from the environment... -RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site - -# Add PATH to /etc/profile so it gets picked up by the terminal -RUN echo "PATH=${PATH}" >> /etc/profile -RUN echo "export PATH" >> /etc/profile - -ENV HOME /home/${NB_USER} - -WORKDIR ${HOME} - -# Install packages needed by notebook-as-pdf -# nodejs for installing notebook / jupyterhub from source -# libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -# tini is necessary because it is our ENTRYPOINT below. -RUN apt-get update > /dev/null && \ - apt-get install --yes \ - less \ - libx11-xcb1 \ - libxtst6 \ - libxrandr2 \ - libasound2 \ - libpangocairo-1.0-0 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libgtk-3-0 \ - libnss3 \ - libnspr4 \ - libxss1 \ - fonts-symbola \ - gdebi-core \ - tini \ - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-plain-generic \ - nodejs npm > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -RUN /tmp/install-mambaforge.bash - -USER ${NB_USER} - -COPY environment.yml /tmp/environment.yml -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -COPY infra-requirements.txt /tmp/infra-requirements.txt -RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt - -# DH-331, very similar to what was done for datahub in DH-164 -ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} -RUN playwright install chromium - -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -COPY class-libs.R /tmp/class-libs.R -RUN mkdir -p /tmp/r-packages - -# Workaround to install ottr, issue 3342 -#RUN wget https://github.com/ucbds-infra/ottr/archive/refs/tags/0.1.0.tar.gz -O /tmp/ottr.tar.gz && R CMD INSTALL /tmp/ottr.tar.gz && rm /tmp/ottr.tar.gz - -#COPY install.R /tmp/install.R -#RUN /tmp/install.R && rm -rf /tmp/downloaded_packages - -COPY r-packages/ottr.r /tmp/r-packages/ -RUN r /tmp/r-packages/ottr.r - -COPY r-packages/ph-290.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-290.r - -COPY r-packages/ph-142.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-142.r - -COPY r-packages/ph-252.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-252.r - -COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ -RUN r /tmp/r-packages/2021-spring-phw-272a.r - -# Use simpler locking strategy -COPY file-locks /etc/rstudio/file-locks - -ENTRYPOINT ["tini", "--"] diff --git a/deployments/publichealth/image/README.md b/deployments/publichealth/image/README.md new file mode 100644 index 000000000..b3eb242db --- /dev/null +++ b/deployments/publichealth/image/README.md @@ -0,0 +1,5 @@ +# Publichealth Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/publichealth-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/publichealth-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/publichealth/image/class-libs.R b/deployments/publichealth/image/class-libs.R deleted file mode 100644 index 1c9343537..000000000 --- a/deployments/publichealth/image/class-libs.R +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env Rscript - -class_libs_install_version <- function(class_name, class_libs) { - print(paste("Installing packages for", class_name)) - for (i in seq(1, length(class_libs), 2)) { - installed_packages <- rownames(installed.packages()) - package_name = class_libs[i] - version = class_libs[i+1] - # Only install packages if they haven't already been installed! - # devtools doesn't do that by default - if (!package_name %in% installed_packages) { - print(paste("Installing", package_name, version)) - devtools::install_version(package_name, version, quiet=TRUE) - } else { - # FIXME: This ignores version incompatibilities :'( - print(paste("Not installing", package_name, " as it is already installed")) - } - } - print(paste("Done installing packages for", class_name)) -} diff --git a/deployments/publichealth/image/environment.yml b/deployments/publichealth/image/environment.yml deleted file mode 100644 index a6a138564..000000000 --- a/deployments/publichealth/image/environment.yml +++ /dev/null @@ -1,17 +0,0 @@ -dependencies: -- pip -- syncthing==1.18.6 -- jupyter-server-proxy==4.2.0 -- jupyter-rsession-proxy==2.2.0 -- pip: - # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - - traitlets==5.9.0 - # - -r /tmp/infra-requirements.txt - # RStudio support - - otter-grader==2.2.7 - # for notebook exporting - - nbconvert[webpdf]==7.16.4 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 diff --git a/deployments/publichealth/image/file-locks b/deployments/publichealth/image/file-locks deleted file mode 100644 index 7b1a3fcf4..000000000 --- a/deployments/publichealth/image/file-locks +++ /dev/null @@ -1,13 +0,0 @@ -# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking - -# rocker sets this to advisory, but this might be causing NFS issues. -# lets set it to the default (default: linkbased) -lock-type=linkbased - -# we'll also reduce the frequency by 1/3 -refresh-rate=60 -timeout-interval=90 - -# log attempts -# enable-logging=1 -# log-file=/tmp/rstudio-locking.log diff --git a/deployments/publichealth/image/infra-requirements.txt b/deployments/publichealth/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/publichealth/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/publichealth/image/install-mambaforge.bash b/deployments/publichealth/image/install-mambaforge.bash deleted file mode 100755 index 812319edc..000000000 --- a/deployments/publichealth/image/install-mambaforge.bash +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of mambaforge -set -ex - -cd $(dirname $0) -MAMBAFORGE_VERSION=22.9.0-2 - -URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/mambaforge-installer.sh - -# make sure we don't do anything funky with user's $HOME -# since this is run as root -unset HOME - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing mambaforge -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/deployments/publichealth/image/install.R b/deployments/publichealth/image/install.R deleted file mode 100755 index 2f337bd89..000000000 --- a/deployments/publichealth/image/install.R +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env r - -source("/tmp/class-libs.R") - -# install ottr, needs to go first issue #3216, #3263 -devtools::install_github('ucbds-infra/ottr', ref='0.1.0', upgrade_dependencies=FALSE, quiet=FALSE) diff --git a/deployments/publichealth/image/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/deployments/publichealth/image/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 7fec51502..000000000 --- a/deployments/publichealth/image/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/deployments/publichealth/image/r-packages/2021-spring-phw-272a.r b/deployments/publichealth/image/r-packages/2021-spring-phw-272a.r deleted file mode 100644 index 3344c476c..000000000 --- a/deployments/publichealth/image/r-packages/2021-spring-phw-272a.r +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env Rscript - -source("/tmp/class-libs.R") - -class_name = "PHW 272a Spring 2021" -class_libs = c( - "ggmap", "3.0.0", - "tmap", "3.3-2", - "rgdal", "1.5-28" -) -class_libs_install_version(class_name, class_libs) diff --git a/deployments/publichealth/image/r-packages/ottr.r b/deployments/publichealth/image/r-packages/ottr.r deleted file mode 100755 index 2a5174f31..000000000 --- a/deployments/publichealth/image/r-packages/ottr.r +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env Rscript -# From https://github.com/berkeley-dsep-infra/datahub/issues/3403 - -print("Installing packages for ottr") -devtools::install_version("ottr", version = "1.1.4", repos = "https://cran.r-project.org", upgrade = "never", quiet = FALSE) -print("Done installing packages for ottr") diff --git a/deployments/publichealth/image/r-packages/ph-142.r b/deployments/publichealth/image/r-packages/ph-142.r deleted file mode 100755 index b83c95cad..000000000 --- a/deployments/publichealth/image/r-packages/ph-142.r +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env Rscript -# From https://github.com/berkeley-dsep-infra/datahub/issues/881 -print("Installing packages for PH142") - -source("/tmp/class-libs.R") - -class_name = "PH142" - -class_libs = c( - "fGarch", "3042.83.2", - "SASxport", "1.7.0", - "googlesheets", "0.3.0", - "googledrive", "1.0.1", - "ggrepel", "0.9.0", - "infer", "0.5.3", - "janitor", "2.1.0", - "latex2exp", "0.4.0", - "measurements", "1.4.0", - "dagitty", "0.3-0", - "cowplot", "1.1.1", - "patchwork", "1.1.1", - "tigris", "1.0", - "googlesheets4", "1.0.0", - "coxed", "0.3.3" -) - -class_libs_install_version(class_name, class_libs) - -print("Done installing packages for PH142") diff --git a/deployments/publichealth/image/r-packages/ph-252.r b/deployments/publichealth/image/r-packages/ph-252.r deleted file mode 100755 index 41e266524..000000000 --- a/deployments/publichealth/image/r-packages/ph-252.r +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env Rscript -# For https://github.com/berkeley-dsep-infra/datahub/issues/2556 -# For https://github.com/berkeley-dsep-infra/datahub/issues/2524 -# For https://github.com/berkeley-dsep-infra/datahub/issues/2748 -# For https://github.com/berkeley-dsep-infra/datahub/issues/2788 -# Fall 2021 -print("Installing packages for PH 252") - -source("/tmp/class-libs.R") - -class_name = "PH 252" - -class_libs = c( - "foreign", "0.8-81", - "blm", "2013.2.4.4", - "geepack", "1.3-2", - "multcomp", "1.4-17", - "epitools", "0.5-10.1", - "rms", "6.2-0", - "lmtest","0.9-38", - "car","3.0-10", - "SurvRegCensCov","1.4", - "ProfileLikelihood","1.1", - "biostat3","0.1.5" -) - -devtools::install_github('cran/epi', ref='06efd3f', upgrade_dependencies=FALSE, quiet=FALSE) -devtools::install_github('kaz-yos/tableone', ref='7cfcd71', upgrade_dependencies=FALSE, quiet=FALSE) - -class_libs_install_version(class_name, class_libs) - -print("Done installing packages for PH 252") diff --git a/deployments/publichealth/image/r-packages/ph-290.r b/deployments/publichealth/image/r-packages/ph-290.r deleted file mode 100755 index 8e49fc4ca..000000000 --- a/deployments/publichealth/image/r-packages/ph-290.r +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env Rscript -# For https://github.com/berkeley-dsep-infra/datahub/issues/1921 -print("Installing packages for PH 290W") - -source("/tmp/class-libs.R") - -class_name = "PH 290W" - -class_libs = c( - "kableExtra", "1.3.1", - "plotly", "4.9.2.2", - "ggthemes", "4.2.0", - "formattable", "0.2.1" -) - -class_libs_install_version(class_name, class_libs) From 2de6672833774ad6fa616e1e7628a17533e617a1 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 11 Sep 2024 10:07:52 -0700 Subject: [PATCH 189/190] enable publichealth deployment --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index d14c90b51..a6ccaabd2 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth) From 4097610f2335e10427d62f29af333d76a9c4c630 Mon Sep 17 00:00:00 2001 From: shane knapp <incomplete@gmail.com> Date: Wed, 11 Sep 2024 10:18:57 -0700 Subject: [PATCH 190/190] fixing publichealth hubploy --- deployments/publichealth/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/publichealth/hubploy.yaml b/deployments/publichealth/hubploy.yaml index ffc67a75e..a1be7ec94 100644 --- a/deployments/publichealth/hubploy.yaml +++ b/deployments/publichealth/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/publichealth-user-image + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/publichealth-user-image:PLACEHOLDER cluster: provider: gcloud