From e4d1023bc911e7a749e926d25dd4aa0e5478ec9b Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Thu, 19 Sep 2024 10:51:30 -0700 Subject: [PATCH 1/6] Increase RAM to 16GB for data100 instructors --- deployments/data100/config/common.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/deployments/data100/config/common.yaml b/deployments/data100/config/common.yaml index 8da6b5791..4d244ff4d 100644 --- a/deployments/data100/config/common.yaml +++ b/deployments/data100/config/common.yaml @@ -112,10 +112,13 @@ jupyterhub: admin: true mem_limit: 4G - # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376 - #course::1531798::group::Admins: # Spring 2024, Data 100 Admins, ensured 4G RAM - # mem_limit: 4G - # mem_guarantee: 4G + # Data 100, Fall 2024, https://github.com/berkeley-dsep-infra/datahub/issues/6167 + course::1537664::enrollment_type::teacher: # Fall 2024, Data 100 Instructors, ensured 16G RAM + mem_limit: 16G + mem_guarantee: 16G + course::1537664::enrollment_type::ta: # Fall 2024, Data 100 TAs, ensured 16G RAM + mem_limit: 16G + mem_guarantee: 16G admin: mem_guarantee: 2G extraVolumeMounts: From 1c979b8e28731227feec50898036497ac983f0f1 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Thu, 19 Sep 2024 10:54:07 -0700 Subject: [PATCH 2/6] Adding CPU limits and guarantee --- deployments/data100/config/common.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deployments/data100/config/common.yaml b/deployments/data100/config/common.yaml index 4d244ff4d..ec250bab5 100644 --- a/deployments/data100/config/common.yaml +++ b/deployments/data100/config/common.yaml @@ -116,9 +116,13 @@ jupyterhub: course::1537664::enrollment_type::teacher: # Fall 2024, Data 100 Instructors, ensured 16G RAM mem_limit: 16G mem_guarantee: 16G + cpu_limit: 2 + cpu_guarantee: 2 course::1537664::enrollment_type::ta: # Fall 2024, Data 100 TAs, ensured 16G RAM mem_limit: 16G mem_guarantee: 16G + cpu_limit: 2 + cpu_guarantee: 2 admin: mem_guarantee: 2G extraVolumeMounts: From 5457a3278de8c44cde14427dcbb797f7681cc036 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 19 Sep 2024 11:24:36 -0700 Subject: [PATCH 3/6] moving data102 image to it's own repo --- .circleci/config.yml | 23 ------ deployments/data102/hubploy.yaml | 5 +- deployments/data102/image/README.md | 5 ++ deployments/data102/image/apt.txt | 81 ------------------- deployments/data102/image/environment.yml | 56 ------------- .../data102/image/infra-requirements.txt | 29 ------- deployments/data102/image/postBuild | 5 -- deployments/data102/image/start | 5 -- scripts/user-image-management/repos.txt | 1 + 9 files changed, 7 insertions(+), 203 deletions(-) create mode 100644 deployments/data102/image/README.md delete mode 100644 deployments/data102/image/apt.txt delete mode 100644 deployments/data102/image/environment.yml delete mode 100644 deployments/data102/image/infra-requirements.txt delete mode 100644 deployments/data102/image/postBuild delete mode 100644 deployments/data102/image/start diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ac875604..1d59b0f9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -193,12 +193,6 @@ jobs: hubploy deploy --timeout 30m data100 hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy data102 - command: | - hubploy deploy --timeout 30m data102 hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy datahub command: | @@ -269,14 +263,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: data102 - name: data102 image build - filters: - branches: - ignore: - - staging - - prod - hubploy/build-image: deployment: datahub # Workshop hub also uses this @@ -325,14 +311,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: data102 - name: data102 image build - push: true - filters: - branches: - only: - - staging - hubploy/build-image: # workshop hub also uses this image deployment: datahub @@ -369,7 +347,6 @@ workflows: requires: - data8 image build - data100 image build - - data102 image build - datahub image build - edx image build - ischool image build diff --git a/deployments/data102/hubploy.yaml b/deployments/data102/hubploy.yaml index 9ad828810..62b8ceb0f 100644 --- a/deployments/data102/hubploy.yaml +++ b/deployments/data102/hubploy.yaml @@ -1,9 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data102-user-image - path: image/ - repo2docker: - base_image: docker.io/library/buildpack-deps:jammy + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data102-user-image:PLACEHOLDER registry: provider: gcloud diff --git a/deployments/data102/image/README.md b/deployments/data102/image/README.md new file mode 100644 index 000000000..6182fc4a6 --- /dev/null +++ b/deployments/data102/image/README.md @@ -0,0 +1,5 @@ +# Data102 Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/data102-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/data102-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/data102/image/apt.txt b/deployments/data102/image/apt.txt deleted file mode 100644 index 2cc25eaf7..000000000 --- a/deployments/data102/image/apt.txt +++ /dev/null @@ -1,81 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# installing less as more just isn't enough -less - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download tools -curl -wget - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -# GUI text editors -emacs -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip - -# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 -libnss3 -libnspr4 diff --git a/deployments/data102/image/environment.yml b/deployments/data102/image/environment.yml deleted file mode 100644 index 0d70715dd..000000000 --- a/deployments/data102/image/environment.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: data102-FA23 - -channels: -- conda-forge -- pytorch - -dependencies: -- python==3.11.* -- git==2.39.1 -- bambi==0.12.0 -- graphviz==8.0.3 -- jupyter-archive==3.4.0 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.2 -- ipywidgets==8.0.7 -- jinja2==3.1.2 -- markupsafe==2.1.3 -- matplotlib==3.7.1 -- numpy==1.24.2 -- nbformat==5.9.2 -- nbresuse==0.4.0 -- otter-grader==4.2.0 -- pandas==2.0.2 -- pandoc==3.1.3 -# data102, Fall 2023, https://github.com/berkeley-dsep-infra/datahub/issues/5037 -- pymc==5.8.1 -- python-graphviz==0.20.1 -- scikit-learn==1.2.2 -- scipy==1.10.1 -- seaborn==0.12.2 -- statsmodels==0.14.0 -# fall 2023: https://github.com/berkeley-dsep-infra/datahub/issues/5130 -- pytorch==2.1.0 -- torchvision==0.16.0 -- torchaudio==2.1.0 -- cpuonly==2.0 -- pip -- pip: - # - -r infra-requirements.txt - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.6 - - nbconvert[webpdf]==7.16.4 - - gh-scoped-creds==4.1 - - nb2pdf==0.6.2 - - nbpdfexport==0.2.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 diff --git a/deployments/data102/image/infra-requirements.txt b/deployments/data102/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/data102/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/data102/image/postBuild b/deployments/data102/image/postBuild deleted file mode 100644 index 1b6bcc9d1..000000000 --- a/deployments/data102/image/postBuild +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -playwright install chromium diff --git a/deployments/data102/image/start b/deployments/data102/image/start deleted file mode 100644 index c3a978b7f..000000000 --- a/deployments/data102/image/start +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# See https://jira-secure.berkeley.edu/browse/DH-305 -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -exec "$@" diff --git a/scripts/user-image-management/repos.txt b/scripts/user-image-management/repos.txt index e7e40caa8..6864d75f3 100644 --- a/scripts/user-image-management/repos.txt +++ b/scripts/user-image-management/repos.txt @@ -3,6 +3,7 @@ git@github.com:berkeley-dsep-infra/astro-user-image.git git@github.com:berkeley-dsep-infra/biology-user-image.git git@github.com:berkeley-dsep-infra/cee-user-image.git git@github.com:berkeley-dsep-infra/data101-user-image.git +git@github.com:berkeley-dsep-infra/data102-user-image.git git@github.com:berkeley-dsep-infra/dev-primary-image.git git@github.com:berkeley-dsep-infra/dev-secondary-image.git git@github.com:berkeley-dsep-infra/eecs-user-image.git From e390be1a3b4d206c5df2da24091f9241c9320d1e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 19 Sep 2024 11:26:44 -0700 Subject: [PATCH 4/6] enable cicd for data102 --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index e6aaf91cb..13a5c8e57 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102) From abb9b61d71ba2de6c53965897d74a8bc732eb7b6 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 19 Sep 2024 11:48:35 -0700 Subject: [PATCH 5/6] [DH-382] removing chp timeouts --- hub/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hub/values.yaml b/hub/values.yaml index 9c814485e..1883d69db 100644 --- a/hub/values.yaml +++ b/hub/values.yaml @@ -38,14 +38,14 @@ jupyterhub: # https://github.com/http-party/node-http-proxy/pull/1559 # https://github.com/Jimbly/http-proxy-node16/commit/56283e33edfc7aad8c2605dd493da8a196b4371d # https://github.com/consideRatio/configurable-http-proxy/commits/main/ - # + # https://jira-secure.berkeley.edu/browse/DH-382 timeouts break stuff image: tag: 4.6.2 - extraCommandLineFlags: + # extraCommandLineFlags: # set the timeout and proxyTimeout to 5 seconds # https://github.com/http-party/node-http-proxy?tab=readme-ov-file#options - - "--timeout=5000" - - "--proxy-timeout=5000" + # - "--timeout=5000" + # - "--proxy-timeout=5000" resources: requests: # FIXME: We want no guarantees here!!! From 865703be331bb555d9177e04c2bfc440f3e057a1 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 19 Sep 2024 12:33:38 -0700 Subject: [PATCH 6/6] temp disable data102 --- .github/workflows/deploy-hubs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 13a5c8e57..dbef0753b 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -103,7 +103,7 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub staging echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia) # data102) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -198,4 +198,4 @@ jobs: echo "Deploying single-user image and hub config to ${deployment}" hubploy --verbose deploy --timeout 30m ${deployment} hub prod echo - done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia data102) + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy gradebook logodev shiny stat159 stat20 nature a11y ugr01 data101 astro biology cee dev publichealth eecs julia) # data102)