From 4a6f531b4d3bc98c0b3f17328505b676b0208948 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 24 Sep 2024 12:51:45 -0700 Subject: [PATCH 1/2] preparing to move edx to its own image repo --- .circleci/config.yml | 25 ------ deployments/edx/hubploy.yaml | 8 +- deployments/edx/image/README.md | 5 ++ deployments/edx/image/apt.txt | 90 -------------------- deployments/edx/image/environment.yml | 53 ------------ deployments/edx/image/infra-requirements.txt | 29 ------- deployments/edx/image/ipython_config.py | 3 - deployments/edx/image/overrides.json | 5 -- deployments/edx/image/postBuild | 10 --- scripts/user-image-management/repos.txt | 1 + 10 files changed, 7 insertions(+), 222 deletions(-) create mode 100644 deployments/edx/image/README.md delete mode 100644 deployments/edx/image/apt.txt delete mode 100644 deployments/edx/image/environment.yml delete mode 100644 deployments/edx/image/infra-requirements.txt delete mode 100644 deployments/edx/image/ipython_config.py delete mode 100644 deployments/edx/image/overrides.json delete mode 100644 deployments/edx/image/postBuild diff --git a/.circleci/config.yml b/.circleci/config.yml index dce3a85af..45bd7aead 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,12 +205,6 @@ jobs: hubploy deploy --timeout 30m dlab hub ${CIRCLE_BRANCH} no_output_timeout: 30m - - run: - name: Deploy edx - command: | - hubploy deploy --timeout 30m edx hub ${CIRCLE_BRANCH} - no_output_timeout: 30m - - run: name: Deploy highschool hub command: | @@ -267,15 +261,6 @@ workflows: ignore: - staging - prod - - hubploy/build-image: - deployment: edx - name: edx image build - # Filters can only be per-job? wtf - filters: - branches: - ignore: - - staging - - prod deploy: jobs: @@ -306,15 +291,6 @@ workflows: branches: only: - staging - - hubploy/build-image: - deployment: edx - name: edx image build - push: true - # Filters can only be per-job? wtf - filters: - branches: - only: - - staging # Build images only during the staging deploy. All merges # to prod need to go via staging, so prod should *never* # use images not built for staging. By enforcing this at the @@ -324,7 +300,6 @@ workflows: - data8 image build - data100 image build - datahub image build - - edx image build filters: branches: diff --git a/deployments/edx/hubploy.yaml b/deployments/edx/hubploy.yaml index 3692cd14e..95734f7d2 100644 --- a/deployments/edx/hubploy.yaml +++ b/deployments/edx/hubploy.yaml @@ -1,12 +1,6 @@ images: images: - - name: gcr.io/data8x-scratch/edx-user-image - path: image/ - registry: - provider: gcloud - gcloud: - project: data8x-scratch - service_key: gcloud-service-key.json + - name: us-central1-docker.pkg.dev/data8x-scratch/edx-user-image:PLACEHOLDER cluster: provider: gcloud diff --git a/deployments/edx/image/README.md b/deployments/edx/image/README.md new file mode 100644 index 000000000..e8935c742 --- /dev/null +++ b/deployments/edx/image/README.md @@ -0,0 +1,5 @@ +# edx Image + +This image is now located [in its own repo](https://github.com/berkeley-dsep-infra/edx-user-image). + +Please see [the contribution guide](https://github.com/berkeley-dsep-infra/edx-user-image/blob/main/CONTRIBUTING.md) for instructions on how to propose changes to the image. diff --git a/deployments/edx/image/apt.txt b/deployments/edx/image/apt.txt deleted file mode 100644 index bfc2a1031..000000000 --- a/deployments/edx/image/apt.txt +++ /dev/null @@ -1,90 +0,0 @@ -# Some linux packages for basic terminal work, particularly -# oriented at users new to Unix/cmd line environments. - -# Basic unix tools -man -man-db -manpages-posix -manpages-dev -manpages-posix-dev - -# Download tools -curl -wget - -# Core text editors on a *nix box: vim -vim - -# A couple of CLI editors that are easier than vim -# micro # currently not working on 18.04 -nano -jed -jed-extra - -# powerful terminal-based file manager, better than the one in JLab -mc - -# for easily managing multiple repositories with one command (perl-doc -# is needed for its help pages to work) -mr -perl-doc - -# Regular build tools for compiling common stuff -build-essential -gfortran - -# Dependencies for nbconvert -texlive-xetex -texlive-fonts-recommended -texlive-plain-generic -# https://github.com/berkeley-dsep-infra/datahub/issues/3719 -texlive-lang-chinese -lmodern - -# Other useful document-related tools -pandoc -latexdiff - -# Some useful git utilities use basic Ruby -ruby - -# Other niceties for command-line work and life -ack # powerful grep-like tool -pydf # colorized disk usage -tmux -screen -htop -nnn # cmd line file manager -zsh -rsync -tig # console UI for git -multitail - -# For later, these are not available in 18.04 -#browsh # text-based web browser, occasionally handy -#dasel # json/yml/csv/etc data wrangling at the terminal -#fzf # fuzzy file finder - -## This section adds tools for desktop environment usage -dbus-x11 -xorg -xubuntu-icon-theme -xfce4 -xfce4-goodies -xclip -xsel -firefox -chromium-browser - -# GUI text editors -vim-gtk3 -gedit - -# Git clients and tools -git-gui -gitg -qgit -meld - -# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip diff --git a/deployments/edx/image/environment.yml b/deployments/edx/image/environment.yml deleted file mode 100644 index 99235e91a..000000000 --- a/deployments/edx/image/environment.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: edx - -channels: -- conda-forge - -dependencies: -- python==3.11.0 -- git==2.39.1 -- jupyter-archive==3.4.0 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 -- folium==0.14.0 -- h5netcdf==1.0.2 -- ipywidgets==8.0.7 -- jupysql==0.8.0 -- matplotlib==3.7.1 -- mdit-py-plugins==0.4.0 -- numpy==1.24.2 -- pandas==2.0.2 -- plotly==5.13.1 -- requests==2.28.2 -- scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 -- seaborn==0.12.2 -- statsmodels==0.14.0 -- tensorflow-cpu==2.12.1 -- sqlalchemy==2.0.16 -- pip -- pip: - # - -r infra-requirements.txt - - datascience==0.17.6 - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.6 - - jupyterlab-accessible-themes==0.2.0 - - nbconvert==7.6.0 - - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ipyleaflet==0.18.1 - - otter-grader==4.4.1 - - otter-submit==0.1.2 - - sympy==1.12 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 diff --git a/deployments/edx/image/infra-requirements.txt b/deployments/edx/image/infra-requirements.txt deleted file mode 100644 index 0fb0bd930..000000000 --- a/deployments/edx/image/infra-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.7 -jupyterlab==4.0.11 -nbgitpuller==1.2.1 -jupyter-resource-usage==1.0.1 -# Matches version in images/hub/Dockerfile -jupyterhub==4.1.6 -appmode==0.8.0 -ipywidgets==8.0.7 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Measure popularity of different packages in our hubs -# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling -# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 -# syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 -# jupyter archival tool for easy user downloads -jupyter-archive==3.4.0 diff --git a/deployments/edx/image/ipython_config.py b/deployments/edx/image/ipython_config.py deleted file mode 100644 index a8774967b..000000000 --- a/deployments/edx/image/ipython_config.py +++ /dev/null @@ -1,3 +0,0 @@ -# Disable history manager, we don't really use it -# and by default it puts an sqlite file on NFS, which is not something we wanna do -c.HistoryManager.enabled = False # noqa: F821 diff --git a/deployments/edx/image/overrides.json b/deployments/edx/image/overrides.json deleted file mode 100644 index 52ea5d9ab..000000000 --- a/deployments/edx/image/overrides.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@jupyterlab/apputils-extension:themes": { - "theme": "GitHub Light" - } -} diff --git a/deployments/edx/image/postBuild b/deployments/edx/image/postBuild deleted file mode 100644 index 803dbb6a2..000000000 --- a/deployments/edx/image/postBuild +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -l -set -euo pipefail - -# Create ipython config directory if it doesn't exist -mkdir -p ${CONDA_DIR}/etc/ipython -cp ipython_config.py ${CONDA_DIR}/etc/ipython/ipython_config.py - -# set default theme -mkdir -p ${NB_PYTHON_PREFIX}/share/jupyter/lab/settings -cp -p overrides.json ${NB_PYTHON_PREFIX}/share/jupyter/lab/settings diff --git a/scripts/user-image-management/repos.txt b/scripts/user-image-management/repos.txt index eeeb68632..46d9398fa 100644 --- a/scripts/user-image-management/repos.txt +++ b/scripts/user-image-management/repos.txt @@ -6,6 +6,7 @@ git@github.com:berkeley-dsep-infra/data101-user-image.git git@github.com:berkeley-dsep-infra/data102-user-image.git git@github.com:berkeley-dsep-infra/dev-primary-image.git git@github.com:berkeley-dsep-infra/dev-secondary-image.git +git@github.com:berkeley-dsep-infra/edx-user-image.git git@github.com:berkeley-dsep-infra/eecs-user-image.git git@github.com:berkeley-dsep-infra/ischool-user-image.git git@github.com:berkeley-dsep-infra/julia-user-image.git From e93ccdc350ce44e05479ccdaa1cd5e3c3dd983b2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 24 Sep 2024 13:03:53 -0700 Subject: [PATCH 2/2] put the images in a subdir --- deployments/edx/hubploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/edx/hubploy.yaml b/deployments/edx/hubploy.yaml index 95734f7d2..c3145f4ce 100644 --- a/deployments/edx/hubploy.yaml +++ b/deployments/edx/hubploy.yaml @@ -1,6 +1,6 @@ images: images: - - name: us-central1-docker.pkg.dev/data8x-scratch/edx-user-image:PLACEHOLDER + - name: us-central1-docker.pkg.dev/data8x-scratch/user-images/edx-user-image:PLACEHOLDER cluster: provider: gcloud