From 3dea7a82e2ec008bc02d83dc3dfad746310a6db9 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 6 Jun 2024 12:10:14 -0700 Subject: [PATCH 1/4] a new image for logodev --- deployments/logodev/hubploy.yaml | 6 +- deployments/logodev/image/apt.txt | 93 +++++++++++++++++++ deployments/logodev/image/environment.yml | 63 +++++++++++++ .../logodev/image/infra-requirements.txt | 29 ++++++ deployments/logodev/image/postBuild | 6 ++ 5 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 deployments/logodev/image/apt.txt create mode 100644 deployments/logodev/image/environment.yml create mode 100644 deployments/logodev/image/infra-requirements.txt create mode 100644 deployments/logodev/image/postBuild diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index a82066120..4dc39da05 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,9 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/primary-user-image - path: ../datahub/images/default + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image + path: image/ + repo2docker: + base_image: docker.io/library/buildpack-deps:jammy registry: provider: gcloud gcloud: diff --git a/deployments/logodev/image/apt.txt b/deployments/logodev/image/apt.txt new file mode 100644 index 000000000..a6c4589ed --- /dev/null +++ b/deployments/logodev/image/apt.txt @@ -0,0 +1,93 @@ +# Some linux packages for basic terminal work, particularly +# oriented at users new to Unix/cmd line environments. + +# installing less as more just isn't enough +less + +# Basic unix tools +man +man-db +manpages-posix +manpages-dev +manpages-posix-dev + +# Download tools +curl +wget + +# Core text editors on a *nix box: vim +vim + +# A couple of CLI editors that are easier than vim +# micro # currently not working on 18.04 +nano +jed +jed-extra + +# powerful terminal-based file manager, better than the one in JLab +mc + +# for easily managing multiple repositories with one command (perl-doc +# is needed for its help pages to work) +mr +perl-doc + +# Regular build tools for compiling common stuff +build-essential +gfortran + +# Dependencies for nbconvert +texlive-xetex +texlive-fonts-recommended +texlive-plain-generic +# https://github.com/berkeley-dsep-infra/datahub/issues/3719 +texlive-lang-chinese +lmodern + +# Other useful document-related tools +pandoc +latexdiff + +# Some useful git utilities use basic Ruby +ruby + +# Other niceties for command-line work and life +ack # powerful grep-like tool +pydf # colorized disk usage +tmux +screen +htop +nnn # cmd line file manager +zsh +rsync +tig # console UI for git +multitail + +# For later, these are not available in 18.04 +#browsh # text-based web browser, occasionally handy +#dasel # json/yml/csv/etc data wrangling at the terminal +#fzf # fuzzy file finder + +## This section adds tools for desktop environment usage +dbus-x11 +xorg +xubuntu-icon-theme +xfce4 +xfce4-goodies +xclip +xsel +firefox +chromium-browser + +# GUI text editors +vim-gtk3 +gedit + +# Git clients and tools +git-gui +gitg +qgit +meld + +# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 +zip diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml new file mode 100644 index 000000000..2f72be25e --- /dev/null +++ b/deployments/logodev/image/environment.yml @@ -0,0 +1,63 @@ +name: data100-FA23 + +channels: +- conda-forge +- pytorch + +dependencies: +- python==3.11.* +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- nbgitpuller==1.2.1 +- notebook==7.0.7 +- folium==0.14.0 +- h5netcdf==1.0.2 +- ipywidgets==8.0.7 +- jupysql==0.8.0 +- jupyter-archive==3.4.0 +- matplotlib==3.7.1 +- mdit-py-plugins==0.4.0 +- numpy==1.24.2 +- pandas==2.0.2 +- plotly==5.13.1 +- requests==2.28.2 +- scikit-image==0.19.3 +- scikit-learn==1.2.2 +- scipy==1.10.1 +- seaborn==0.12.2 +- statsmodels==0.14.0 +- tensorflow-cpu==2.12.1 +- sqlalchemy==2.0.16 +- mlxtend==0.23.0 +# Spring 2024 data 100 +- pytorch==2.1.2 +- cpuonly==2.0 +- transformers==4.37.1 +# Spring 2024 table demos +- lxml==5.1.0 +# Spring 2024 Econ 148 Packages +- geopandas==0.14.2 +- geopy==2.4.1 +- lifelines==0.27.8 +- pycountry==22.3.5 +- pip +- pip: + # - -r infra-requirements.txt + - ipywidgets==8.0.7 + # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) + # - jupyter_collaboration==1.0.1 + - jupyterhub==4.1.5 + - nbconvert[webpdf] + # - pyppeteer==2.0.0 + - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 + - otter-grader==5.4.0 + - duckdb==0.10.1 + - duckdb_engine==0.11.2 diff --git a/deployments/logodev/image/infra-requirements.txt b/deployments/logodev/image/infra-requirements.txt new file mode 100644 index 000000000..b01c9d80d --- /dev/null +++ b/deployments/logodev/image/infra-requirements.txt @@ -0,0 +1,29 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +notebook==7.0.7 +jupyterlab==4.0.11 +nbgitpuller==1.2.1 +jupyter-resource-usage==1.0.1 +# Matches version in images/hub/Dockerfile +jupyterhub==4.1.5 +appmode==0.8.0 +ipywidgets==8.0.7 +jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling +# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 +RISE==5.7.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.3 +# jupyter archival tool for easy user downloads +jupyter-archive==3.4.0 diff --git a/deployments/logodev/image/postBuild b/deployments/logodev/image/postBuild new file mode 100644 index 000000000..9514de431 --- /dev/null +++ b/deployments/logodev/image/postBuild @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +npm install -g playwright +playwright install chromium From c000bd91297d42e07e12aae54c03f6130778585d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 6 Jun 2024 12:21:01 -0700 Subject: [PATCH 2/4] trigger a new build i hope --- deployments/logodev/hubploy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index 4dc39da05..546a6a4f7 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,5 +1,6 @@ images: images: + # temporary update - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image path: image/ repo2docker: From 09eccfef283e6f9ee6ee1dcbca111710a2cae67c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 6 Jun 2024 12:21:39 -0700 Subject: [PATCH 3/4] trigger a new build i hope 2 --- deployments/logodev/image/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml index 2f72be25e..197f1a5c5 100644 --- a/deployments/logodev/image/environment.yml +++ b/deployments/logodev/image/environment.yml @@ -23,7 +23,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- pandas==2.0.2 +- pandas==2.2.2 - plotly==5.13.1 - requests==2.28.2 - scikit-image==0.19.3 From 077cd2302731f46955549407b8bef7df256c333f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 6 Jun 2024 12:25:52 -0700 Subject: [PATCH 4/4] circleci is a PITA --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fa508f65..399a528bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -581,6 +581,15 @@ workflows: ignore: - staging - prod + - hubploy/build-image: + deployment: logodev + name: logodev hub image build + # Filters can only be per-job? wtf + filters: + branches: + ignore: + - staging + - prod - hubploy/build-image: deployment: publichealth name: publichealth image build