From 241e38461d86c17286c2e6807a501fb532b5c01f Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Tue, 2 Jul 2024 13:05:21 -0700
Subject: [PATCH 01/29] squelch dependabot

---
 deployments/datahub/images/default/requirements.txt.disabled | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployments/datahub/images/default/requirements.txt.disabled b/deployments/datahub/images/default/requirements.txt.disabled
index 98b6233a8..0c69d6921 100644
--- a/deployments/datahub/images/default/requirements.txt.disabled
+++ b/deployments/datahub/images/default/requirements.txt.disabled
@@ -5,7 +5,7 @@ git+https://github.com/yuvipanda/ok-client@6961d778741fe61911be4d00beff9bd8afc1e
 folium==0.12.1
 #
 # r
-jupyter-server-proxy==3.2.1
+jupyter-server-proxy==4.2.0
 jupyter-rsession-proxy==2.0.1
 jupyter-shiny-proxy==1.1
 #

From 982f82cf8ceb60d5fc4667327c12218da41167bb Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Tue, 2 Jul 2024 13:13:35 -0700
Subject: [PATCH 02/29] bump urllib3

---
 deployments/a11y/image/environment.yml        |  2 +-
 .../node-placeholder-scaler/requirements.in   |  2 +-
 .../node-placeholder-scaler/requirements.txt  | 19 ++++++++++---------
 requirements.txt                              |  2 +-
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/deployments/a11y/image/environment.yml b/deployments/a11y/image/environment.yml
index e966560f9..a6fbaaeb0 100644
--- a/deployments/a11y/image/environment.yml
+++ b/deployments/a11y/image/environment.yml
@@ -20,7 +20,7 @@ dependencies:
 - plotly==5.13.1
 - pyopenssl==23.1.0
 - requests==2.28.2
-- urllib3==1.26.15
+- urllib3==2.2.2
 - websockify==0.11.0
 - pip==23.0.1
 - jupyterthemes==0.20.0
diff --git a/images/node-placeholder-scaler/requirements.in b/images/node-placeholder-scaler/requirements.in
index 63da1e2a9..48a95bbfe 100644
--- a/images/node-placeholder-scaler/requirements.in
+++ b/images/node-placeholder-scaler/requirements.in
@@ -3,4 +3,4 @@ DateTime
 ical==5.0.0
 requests
 ruamel.yaml
-urllib3==2.0.7
+urllib3==2.2.2
diff --git a/images/node-placeholder-scaler/requirements.txt b/images/node-placeholder-scaler/requirements.txt
index 1091ad7b0..a3afca6ac 100644
--- a/images/node-placeholder-scaler/requirements.txt
+++ b/images/node-placeholder-scaler/requirements.txt
@@ -1,10 +1,10 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile requirements.in
 #
-annotated-types==0.6.0
+annotated-types==0.7.0
     # via pydantic
 certifi==2023.7.22
     # via
@@ -14,15 +14,15 @@ charset-normalizer==3.3.2
     # via requests
 datetime==5.5
     # via -r requirements.in
-emoji==2.11.1
+emoji==2.12.1
     # via ical
 ical==5.0.0
     # via -r requirements.in
 idna==3.7
     # via requests
-pydantic==2.7.1
+pydantic==2.8.0
     # via ical
-pydantic-core==2.18.2
+pydantic-core==2.20.0
     # via pydantic
 pyparsing==3.1.2
     # via ical
@@ -30,7 +30,7 @@ python-dateutil==2.9.0.post0
     # via ical
 pytz==2024.1
     # via datetime
-requests==2.31.0
+requests==2.32.3
     # via -r requirements.in
 ruamel-yaml==0.18.6
     # via -r requirements.in
@@ -38,17 +38,18 @@ ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
 six==1.16.0
     # via python-dateutil
-typing-extensions==4.11.0
+typing-extensions==4.12.2
     # via
+    #   emoji
     #   pydantic
     #   pydantic-core
 tzdata==2024.1
     # via ical
-urllib3==2.0.7
+urllib3==2.2.2
     # via
     #   -r requirements.in
     #   requests
-zope-interface==6.3
+zope-interface==6.4.post2
     # via datetime
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/requirements.txt b/requirements.txt
index 79bd808f8..b46b6945e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,4 +9,4 @@ myst-parser
 chardet
 # requests==2.31.0
 requests<2.29.0
-urllib3<2.0
+urllib3s==2.2.2

From a9286749506496f3b09eea1d453bef3cee2f9950 Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Tue, 2 Jul 2024 13:16:02 -0700
Subject: [PATCH 03/29] python 3.11

---
 images/node-placeholder-scaler/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/images/node-placeholder-scaler/requirements.txt b/images/node-placeholder-scaler/requirements.txt
index a3afca6ac..41bcf3d5f 100644
--- a/images/node-placeholder-scaler/requirements.txt
+++ b/images/node-placeholder-scaler/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile requirements.in

From 0610800b5551ede40f831d7248a0b53c13ddaa37 Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Tue, 2 Jul 2024 13:19:17 -0700
Subject: [PATCH 04/29] fat finger fix

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b46b6945e..5db4014e9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,4 +9,4 @@ myst-parser
 chardet
 # requests==2.31.0
 requests<2.29.0
-urllib3s==2.2.2
+urllib3==2.2.2

From 0fd60c2f15d17335b875b1966033077ea0c2e91e Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Tue, 2 Jul 2024 14:04:06 -0700
Subject: [PATCH 05/29] bumping requests

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 5db4014e9..ba67b08a4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,5 +8,5 @@ jupyter-repo2docker==2024.3.0
 myst-parser
 chardet
 # requests==2.31.0
-requests<2.29.0
+requests==2.32.3
 urllib3==2.2.2

From 6e3fb747215bfdaffee54300ac2678eeee40e8ee Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Tue, 2 Jul 2024 14:57:09 -0700
Subject: [PATCH 06/29] bump requests here

---
 deployments/a11y/image/environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployments/a11y/image/environment.yml b/deployments/a11y/image/environment.yml
index a6fbaaeb0..704258ef6 100644
--- a/deployments/a11y/image/environment.yml
+++ b/deployments/a11y/image/environment.yml
@@ -19,7 +19,7 @@ dependencies:
 - numpy==1.24.2
 - plotly==5.13.1
 - pyopenssl==23.1.0
-- requests==2.28.2
+- requests==2.32.3
 - urllib3==2.2.2
 - websockify==0.11.0
 - pip==23.0.1

From 2806ced2423f56ae5cd1c7326cd7cebb7717d41b Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Mon, 8 Jul 2024 11:15:00 -0700
Subject: [PATCH 07/29] addresses
 https://github.com/berkeley-dsep-infra/datahub/security/dependabot/186

---
 images/node-placeholder-scaler/requirements.in  | 2 +-
 images/node-placeholder-scaler/requirements.txt | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/images/node-placeholder-scaler/requirements.in b/images/node-placeholder-scaler/requirements.in
index 48a95bbfe..f92f4c0d1 100644
--- a/images/node-placeholder-scaler/requirements.in
+++ b/images/node-placeholder-scaler/requirements.in
@@ -1,4 +1,4 @@
-certifi==2023.7.22
+certifi==2024.07.04
 DateTime
 ical==5.0.0
 requests
diff --git a/images/node-placeholder-scaler/requirements.txt b/images/node-placeholder-scaler/requirements.txt
index 41bcf3d5f..51ad5c580 100644
--- a/images/node-placeholder-scaler/requirements.txt
+++ b/images/node-placeholder-scaler/requirements.txt
@@ -6,7 +6,7 @@
 #
 annotated-types==0.7.0
     # via pydantic
-certifi==2023.7.22
+certifi==2024.7.4
     # via
     #   -r requirements.in
     #   requests
@@ -20,9 +20,9 @@ ical==5.0.0
     # via -r requirements.in
 idna==3.7
     # via requests
-pydantic==2.8.0
+pydantic==2.8.2
     # via ical
-pydantic-core==2.20.0
+pydantic-core==2.20.1
     # via pydantic
 pyparsing==3.1.2
     # via ical

From 72ebb7f155ff4878107b01b78e136b5d51d44e8a Mon Sep 17 00:00:00 2001
From: Balaji Alwar <balajialwar@berkeley.edu>
Date: Wed, 10 Jul 2024 13:29:21 -0700
Subject: [PATCH 08/29] Add voila and voici package in dashboard image

---
 deployments/dev/images/secondary/environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml
index 08bb53833..8a3a37e1d 100644
--- a/deployments/dev/images/secondary/environment.yml
+++ b/deployments/dev/images/secondary/environment.yml
@@ -57,6 +57,8 @@ dependencies:
 #- python-duckdb==1.0.0
 #- jupyterhub==5.0.0
 - quarto==1.4.550
+- voila==0.5.7
+- voici==0.6.1
 - pip
 - pip:
   # - -r infra-requirements.txt

From 8e6a1c108d22984d90a4c4c3ea98fc95562329f2 Mon Sep 17 00:00:00 2001
From: Balaji Alwar <balajialwar@berkeley.edu>
Date: Wed, 10 Jul 2024 16:50:01 -0700
Subject: [PATCH 09/29] Install voici, mystmd and comment out unnecessary
 packages for dashboard image

---
 .../dev/images/secondary/environment.yml      | 74 +++++++++----------
 1 file changed, 36 insertions(+), 38 deletions(-)

diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml
index 8a3a37e1d..8c679c4a2 100644
--- a/deployments/dev/images/secondary/environment.yml
+++ b/deployments/dev/images/secondary/environment.yml
@@ -7,71 +7,69 @@ channels:
 dependencies:
 - python==3.11.*
 - git==2.39.1
+- jupyterhub==5.0.0
 - jupyter-resource-usage==1.0.0
 - jupyterlab==4.0.11
 - jupyterlab-favorites==3.0.0
+# Install myst for generating dashboards
+- jupyterlab-myst==2.4.2
 - jupyterlab_server==2.23.0
 - jupyterlab_widgets==3.0.8
 - jupyter_server==2.7.0
 - nbgitpuller==1.2.1
 - notebook==7.0.7
 - folium==0.14.0
-- h5netcdf==1.0.2
+  #- h5netcdf==1.0.2
 - ipywidgets==8.0.7
-- jupysql==0.8.0
+  #- jupysql==0.8.0
 - jupyter-archive==3.4.0
 - matplotlib==3.7.1
-- mdit-py-plugins==0.4.0
+# Install myst markdown for generating dashboards
+- mystmd==1.2.5
+  #- mdit-py-plugins==0.4.0
 - numpy==1.24.2
 - pandas==2.0.2
 - plotly==5.13.1
 - requests==2.28.2
 - scikit-image==0.19.3
-- scikit-learn==1.2.2
-- scipy==1.10.1
+  #- scikit-learn==1.2.2
+  #- scipy==1.10.1
 - seaborn==0.12.2
-- statsmodels==0.14.0
-- sqlalchemy==2.0.16
-- mlxtend==0.23.0
+  #- statsmodels==0.14.0
+  #- tensorflow-cpu==2.12.1
+  #- sqlalchemy==2.0.16
+  #- mlxtend==0.23.0
 # Spring 2024 data 100
-- pytorch==2.1.2
-- cpuonly==2.0
-- transformers==4.37.1
+#- pytorch==2.1.2
+#- cpuonly==2.0
+#- transformers==4.37.1
 # Spring 2024 table demos
-- lxml==5.1.0
+#- lxml==5.1.0
 # Spring 2024 Econ 148 Packages
 - geopandas==0.14.2
 - geopy==2.4.1
-- lifelines==0.27.8
+  #- lifelines==0.27.8
 - pycountry==22.3.5
-# Install voila for generating dashboards
-- voila==0.5.7
-# Install myst for generating dashboards
-- mystmd==1.2.5
-- jupyterlab-myst==2.4.2
-#- ipywidgets==8.1.3
-#- pytest-notebook==0.10.0
-#- gh-scoped-creds==4.1
-#- ydata-profiling==4.8.3
-#- otter-grader==5.5.0
-#- python-duckdb==1.0.0
-#- jupyterhub==5.0.0
-- quarto==1.4.550
+# Install voila and voici for generating voila dashboards
 - voila==0.5.7
 - voici==0.6.1
-- pip
-- pip:
+  #- pytest-notebook==0.10.0
+  #- gh-scoped-creds==4.1
+  #- ydata-profiling==4.8.3
+  #- otter-grader==5.5.0
+  #- python-duckdb==1.0.0
+  #- pip
+  #- pip:
   # - -r infra-requirements.txt
-  - ipywidgets==8.0.7
   # disable until fixed (probably this:  https://github.com/jupyterlab/jupyter-collaboration/issues/162)
   # - jupyter_collaboration==1.0.1
-  - jupyterhub==4.1.5
-  - nbconvert[webpdf]
+  #- jupyterhub==4.1.5
+  # - nbconvert[webpdf]
   #  - pyppeteer==2.0.0
-  - pytest-notebook==0.8.1
-  - gh-scoped-creds==4.1
-  - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
-  - ydata-profiling==4.6.4
-  - otter-grader==5.4.0
-  - duckdb==0.10.1
-  - duckdb_engine==0.11.2
+  #- pytest-notebook==0.8.1
+  #- gh-scoped-creds==4.1
+  # - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
+  #- ydata-profiling==4.6.4
+  #- otter-grader==5.4.0
+  #- duckdb==0.10.1
+  #  - duckdb_engine==0.11.2

From 557772f13aba26ac4350f3c4882383687e59a821 Mon Sep 17 00:00:00 2001
From: Balaji Alwar <balajialwar@berkeley.edu>
Date: Thu, 11 Jul 2024 13:14:35 -0700
Subject: [PATCH 10/29] Addressing review comments

---
 .../dev/images/secondary/environment.yml      | 40 +------------------
 1 file changed, 2 insertions(+), 38 deletions(-)

diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml
index 8c679c4a2..cf4608931 100644
--- a/deployments/dev/images/secondary/environment.yml
+++ b/deployments/dev/images/secondary/environment.yml
@@ -7,7 +7,7 @@ channels:
 dependencies:
 - python==3.11.*
 - git==2.39.1
-- jupyterhub==5.0.0
+- jupyterhub==4.1.5
 - jupyter-resource-usage==1.0.0
 - jupyterlab==4.0.11
 - jupyterlab-favorites==3.0.0
@@ -19,57 +19,21 @@ dependencies:
 - nbgitpuller==1.2.1
 - notebook==7.0.7
 - folium==0.14.0
-  #- h5netcdf==1.0.2
 - ipywidgets==8.0.7
-  #- jupysql==0.8.0
 - jupyter-archive==3.4.0
 - matplotlib==3.7.1
 # Install myst markdown for generating dashboards
-- mystmd==1.2.5
-  #- mdit-py-plugins==0.4.0
+- mystmd==1.2.9
 - numpy==1.24.2
 - pandas==2.0.2
 - plotly==5.13.1
 - requests==2.28.2
 - scikit-image==0.19.3
-  #- scikit-learn==1.2.2
-  #- scipy==1.10.1
 - seaborn==0.12.2
-  #- statsmodels==0.14.0
-  #- tensorflow-cpu==2.12.1
-  #- sqlalchemy==2.0.16
-  #- mlxtend==0.23.0
-# Spring 2024 data 100
-#- pytorch==2.1.2
-#- cpuonly==2.0
-#- transformers==4.37.1
-# Spring 2024 table demos
-#- lxml==5.1.0
 # Spring 2024 Econ 148 Packages
 - geopandas==0.14.2
 - geopy==2.4.1
-  #- lifelines==0.27.8
 - pycountry==22.3.5
 # Install voila and voici for generating voila dashboards
 - voila==0.5.7
 - voici==0.6.1
-  #- pytest-notebook==0.10.0
-  #- gh-scoped-creds==4.1
-  #- ydata-profiling==4.8.3
-  #- otter-grader==5.5.0
-  #- python-duckdb==1.0.0
-  #- pip
-  #- pip:
-  # - -r infra-requirements.txt
-  # disable until fixed (probably this:  https://github.com/jupyterlab/jupyter-collaboration/issues/162)
-  # - jupyter_collaboration==1.0.1
-  #- jupyterhub==4.1.5
-  # - nbconvert[webpdf]
-  #  - pyppeteer==2.0.0
-  #- pytest-notebook==0.8.1
-  #- gh-scoped-creds==4.1
-  # - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
-  #- ydata-profiling==4.6.4
-  #- otter-grader==5.4.0
-  #- duckdb==0.10.1
-  #  - duckdb_engine==0.11.2

From 9828c7fb22204e28862cbf536d5d073de9243b58 Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Fri, 12 Jul 2024 14:23:20 -0700
Subject: [PATCH 11/29] remove as its not needed

---
 .../images/default/requirements.txt.disabled  | 189 ------------------
 1 file changed, 189 deletions(-)
 delete mode 100644 deployments/datahub/images/default/requirements.txt.disabled

diff --git a/deployments/datahub/images/default/requirements.txt.disabled b/deployments/datahub/images/default/requirements.txt.disabled
deleted file mode 100644
index 0c69d6921..000000000
--- a/deployments/datahub/images/default/requirements.txt.disabled
+++ /dev/null
@@ -1,189 +0,0 @@
-# data8; foundation
-datascience==0.17.0
-# Until https://github.com/okpy/ok-client/pull/473 is merged
-git+https://github.com/yuvipanda/ok-client@6961d778741fe61911be4d00beff9bd8afc1edf7
-folium==0.12.1
-#
-# r
-jupyter-server-proxy==4.2.0
-jupyter-rsession-proxy==2.0.1
-jupyter-shiny-proxy==1.1
-#
-# cogsci88;
-ggplot==0.11.5
-#
-# cogsci131; spring 2018
-nose==1.3.7
-#
-# modules
-beautifulsoup4==4.9.3
-# nb2pdf==0.6.2 commented out by sknapp 06.10.2023 to unblock https://github.com/berkeley-dsep-infra/datahub/issues/5062
-#
-# ls 88-3; neuro
-lxml==4.9.1
-tqdm==4.62.1
-mne==0.23.0
-nibabel==3.2.1
-h5py==3.7.0
-numexpr==2.7.3
-openpyxl==3.0.7
-nilearn==0.8.0
-
-# data100, geog88
-# https://github.com/berkeley-dsep-infra/datahub/issues/2838
-geopandas==0.10.*
-geopy==2.2.*
-pysal==2.5.*
-rtree==0.9.*
-netcdf4==1.6.*
-mplleaflet==0.0.5
-# phys 151;
-emcee==3.1.0
-daft==0.1.2
-corner==2.2.1
-#
-# ce88;
-pymdptoolbox==4.0-b3
-#
-# data-x; DL
-tensorflow==2.11.1
-scikit-image==0.19.*
-tables==3.7.0
-opencv-python==4.6.0.66
-
-# astr 128/256; spring 2021
-astroquery==0.4.6
-astropy==5.3.3
-dustmaps==1.0.9
-george==0.4.0
-exoplanet==0.5.2
-torch==1.13.1
-torchvision==0.14.1
-pyvo==1.1
-joblib==1.3.1
-theano-pymc==1.1.2
-pymc3==3.11.2
-
-# eep 153; spring 2019
-requests==2.31.0
-Pint==0.17
-# Google spreadsheets, Eric Van Dusen / Keeley Takimoto / Modules
-gspread-pandas==2.3.0
-gspread==4.0.1
-
-# eps 109; fall 2019
-ffmpeg-python==0.2.0
-
-# data 88e; spring 2021
-plotly==5.2.1
-mpmath==1.3.0
-# sympy==1.6.2
-chart-studio==1.1.0
-csaps==1.1.0
-nbforms==0.5.1
-
-# issue #875, global 150Q/pacs 190 - fall 2019
-wordcloud==1.8.1
-
-# issue #929, SW 282 - fall 2019
-pyreadstat==1.1.2
-
-# issue 954, EPS24 - fall 2019
-xarray==0.19.0
-
-# issue 1001, Physics 188/288 - fall 2019
-umap-learn==0.5.1
-hdbscan==0.8.31
-
-# espm 125/bio 105; fall 2019
-# see https://github.com/berkeley-dsep-infra/datahub/issues/1796
-bitarray==2.3.0
-nlmpy==1.0.1
-
-# physics 188/288 fall, 2019
-getdist==1.3.1
-tensorflow-hub==0.12.0
-tensorflow-probability==0.13.0
-
-# cs16A/B, spring 2020
-lcapy==0.96
-
-# EPS 256, https://github.com/berkeley-dsep-infra/datahub/issues/1775
-obspy==1.3.0
-
-# ds198 mch infodemiology, fall 2020/spring 2021
-# google apis
-google-api-python-client==2.15.0
-google-auth-httplib2==0.1.0
-google-auth-oauthlib==0.4.5
-google==3.0.0
-
-# issue 1847; LS 22 spring 2021
-graphviz==0.17
-
-# issue #1903, data h195A fall 2020
-habanero==0.7.4
-
-# https://github.com/berkeley-dsep-infra/datahub/issues/1981
-ipycanvas==0.9.0
-
-# data100 scientific packages
-ray==1.13.*
-xlrd==2.0.1
-
-# data100 visualization
-colorlover==0.3.0
-cufflinks==0.17.3
-altair==4.1.0
-
-# data100 tools to access things
-tweepy==3.10.0
-pytz==2021.1
-psycopg2==2.9.1
-
-# data100 teaching
-jassign==0.0.7
-dsassign==0.0.8
-
-# data102
-dask==2021.10.0
-distributed==2021.10.0
-keras-applications==1.0.8
-keras-preprocessing==1.1.2
-keras==2.11.0
-keras-vis==0.4.1
-plotly-express==0.4.1
-cytoolz==0.11.0
-pyro-ppl==1.7.0
-lime==0.2.0.1
-shap==0.39.0
-
-# prob140 2021 Spring
-prob140==0.4.1.5
-sympy==1.8
-
-# ESPM 167 - https://github.com/berkeley-dsep-infra/datahub/issues/2278
-contextily==1.1.0
-
-# INDENG 142 Spring 2021 (future semesters as well) - https://github.com/berkeley-dsep-infra/datahub/issues/2314
-fancyimpute==0.7.0
-
-# JupyterLab pypi extensions
-jupyterlab-geojson==3.1.2
-
-# IB 105 / ESPM 125, Fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2696
-geonomics==1.3.0
-
-# data100, fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2767
-ipython-sql==0.4.0
-
-# data100, fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2875
-metpy==1.1.0
-pooch==1.5.2
-
-# PS88 https://github.com/berkeley-dsep-infra/datahub/issues/2925
-linearmodels==4.27
-
-# https://github.com/berkeley-dsep-infra/datahub/issues/2950
-# Needed to work with a new enough version of httplib2
-httplib2>=0.20.2

From a639156decdefaece43c15a1a4e41208f364b0d7 Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Fri, 12 Jul 2024 15:37:21 -0700
Subject: [PATCH 12/29] [DH-323] fixes DH-323

---
 deployments/datahub/images/default/environment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml
index 7e21be4d2..757b532da 100644
--- a/deployments/datahub/images/default/environment.yml
+++ b/deployments/datahub/images/default/environment.yml
@@ -202,6 +202,8 @@ dependencies:
 
   # Engineering 7 https://github.com/berkeley-dsep-infra/datahub/issues/5337
   - otter-grader==4.4.1
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
 
   # Stat 165/265 requires prophet, Spring, 2024
   - prophet==1.1.5

From cae646acd65884feff5c6083bc5a64f92fd7f675 Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Mon, 15 Jul 2024 10:46:16 -0700
Subject: [PATCH 13/29] pin jupytext on hubs that need it

---
 deployments/a11y/image/environment.yml         | 2 ++
 deployments/data100/image/environment.yml      | 2 ++
 deployments/data101/image/environment.yml      | 2 ++
 deployments/data102/image/environment.yml      | 2 ++
 deployments/data8/image/environment.yml        | 3 ++-
 deployments/edx/image/environment.yml          | 2 ++
 deployments/logodev/image/environment.yml      | 2 ++
 deployments/publichealth/image/environment.yml | 2 ++
 deployments/stat159/image/environment.yml      | 2 ++
 9 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/deployments/a11y/image/environment.yml b/deployments/a11y/image/environment.yml
index cf78731a6..a67f3415d 100644
--- a/deployments/a11y/image/environment.yml
+++ b/deployments/a11y/image/environment.yml
@@ -36,6 +36,8 @@ dependencies:
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
   - jupyterlab-a11y-checker==0.1.2
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
   # ###
   # The items below are from infra-requirements, however lab conflicts with the
   # alpha notebook.
diff --git a/deployments/data100/image/environment.yml b/deployments/data100/image/environment.yml
index 2f72be25e..ffbd747dd 100644
--- a/deployments/data100/image/environment.yml
+++ b/deployments/data100/image/environment.yml
@@ -61,3 +61,5 @@ dependencies:
   - otter-grader==5.4.0
   - duckdb==0.10.1
   - duckdb_engine==0.11.2
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/data101/image/environment.yml b/deployments/data101/image/environment.yml
index 99cf7e93f..00c74a51c 100644
--- a/deployments/data101/image/environment.yml
+++ b/deployments/data101/image/environment.yml
@@ -106,3 +106,5 @@ dependencies:
   - pymongo==4.4.1
   - dbt-postgres==1.6.0
   - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/data102/image/environment.yml b/deployments/data102/image/environment.yml
index 8595687fe..c312f3f52 100644
--- a/deployments/data102/image/environment.yml
+++ b/deployments/data102/image/environment.yml
@@ -52,3 +52,5 @@ dependencies:
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
   - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/data8/image/environment.yml b/deployments/data8/image/environment.yml
index 21ba91e63..6d57b52ab 100644
--- a/deployments/data8/image/environment.yml
+++ b/deployments/data8/image/environment.yml
@@ -43,4 +43,5 @@ dependencies:
 # Conda only has 0.12.0
   - okpy==1.18.1
   - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
-
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/edx/image/environment.yml b/deployments/edx/image/environment.yml
index 871dd725b..df01bc927 100644
--- a/deployments/edx/image/environment.yml
+++ b/deployments/edx/image/environment.yml
@@ -49,3 +49,5 @@ dependencies:
   - otter-grader==4.4.1
   - otter-submit==0.1.2
   - sympy==1.12
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml
index 197f1a5c5..386806bea 100644
--- a/deployments/logodev/image/environment.yml
+++ b/deployments/logodev/image/environment.yml
@@ -61,3 +61,5 @@ dependencies:
   - otter-grader==5.4.0
   - duckdb==0.10.1
   - duckdb_engine==0.11.2
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/publichealth/image/environment.yml b/deployments/publichealth/image/environment.yml
index 6a1ca02fd..63f3c750d 100644
--- a/deployments/publichealth/image/environment.yml
+++ b/deployments/publichealth/image/environment.yml
@@ -13,3 +13,5 @@ dependencies:
   - nbconvert==7.6.0
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
+  # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+  - jupytext==1.16.2
diff --git a/deployments/stat159/image/environment.yml b/deployments/stat159/image/environment.yml
index 61c660a13..46d679246 100644
--- a/deployments/stat159/image/environment.yml
+++ b/deployments/stat159/image/environment.yml
@@ -169,3 +169,5 @@ dependencies:
     - nbconvert==7.6.0
     - nb2pdf==0.6.2
     - nbpdfexport==0.2.1
+    # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
+    - jupytext==1.16.2

From 04bee0b1e536ae12b16c523b4652129ae4301301 Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Mon, 15 Jul 2024 10:50:31 -0700
Subject: [PATCH 14/29] bump server proxy for stat159

---
 deployments/stat159/image/environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployments/stat159/image/environment.yml b/deployments/stat159/image/environment.yml
index 46d679246..b3f7a0358 100644
--- a/deployments/stat159/image/environment.yml
+++ b/deployments/stat159/image/environment.yml
@@ -137,7 +137,7 @@ dependencies:
   - syncthing==1.23.0
   - websockify==0.11.0
 
-  - jupyter-server-proxy==4.2.0
+  - jupyter-server-proxy==4.3.0
   # VS Code support
   - jupyter-vscode-proxy==0.2
   - code-server==4.10.1

From 1582342ac8dcbab0f4df435af51af5843e954ff5 Mon Sep 17 00:00:00 2001
From: Balaji Alwar <balajialwar@berkeley.edu>
Date: Tue, 16 Jul 2024 09:27:41 -0700
Subject: [PATCH 15/29] Enable gh-scoped-creds based on bcourses profile

---
 deployments/dev/config/common.yaml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml
index 94ef7bd92..5f0a27d46 100644
--- a/deployments/dev/config/common.yaml
+++ b/deployments/dev/config/common.yaml
@@ -71,9 +71,6 @@ jupyterhub:
         description: "A newer repo2docker-based image with a primary focus of building dashboards."
         kubespawner_override:
           image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary:36280727a
-          env:
-            GH_SCOPED_CREDS_CLIENT_ID: "Iv23liFjgKFgkSBWph4C"
-            GH_SCOPED_CREDS_APP_URL: "https://github.com/apps/test-gh-scoped-creds"
       - display_name: "1524699: DataHub Infrastructure"
         slug: "1524699"
         description: "Regular image with per-course subpath."
@@ -95,6 +92,14 @@ jupyterhub:
 
   custom:
     group_profiles:
+      # DataHub Infrastructure staff
+      # https://bcourses.berkeley.edu/courses/1524699/groups#tab-80607
+      course::1524699::group::all-admins:
+        mem_limit: 12288M
+        mem_guarantee: 12288M
+        env:
+          GH_SCOPED_CREDS_CLIENT_ID: "Iv23liFjgKFgkSBWph4C"
+          GH_SCOPED_CREDS_APP_URL: "https://github.com/apps/test-gh-scoped-creds"
   #
   #    # Example: increase memory for everyone affiliated with a course.
   #

From 2382e3ab2bfbb132cba4de4954c1829c79751a47 Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 11:19:24 -0700
Subject: [PATCH 16/29] [DH-325] make webpdf export work

---
 deployments/data8/image/apt.txt         | 4 ++++
 deployments/data8/image/environment.yml | 3 ++-
 deployments/data8/image/postBuild       | 5 +++++
 deployments/data8/image/start           | 5 +++++
 4 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 deployments/data8/image/postBuild
 create mode 100644 deployments/data8/image/start

diff --git a/deployments/data8/image/apt.txt b/deployments/data8/image/apt.txt
index e9646bf15..37e4c7a7c 100644
--- a/deployments/data8/image/apt.txt
+++ b/deployments/data8/image/apt.txt
@@ -77,3 +77,7 @@ meld
 
 # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979
 zip
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-325
+libnss3
+libnspr4
diff --git a/deployments/data8/image/environment.yml b/deployments/data8/image/environment.yml
index 6d57b52ab..ad3b5a542 100644
--- a/deployments/data8/image/environment.yml
+++ b/deployments/data8/image/environment.yml
@@ -37,9 +37,10 @@ dependencies:
 #- -r infra-requirements.txt
   - jupyter-tree-download==1.0.1
 # Export notebooks as PDFs with Chrome
-  - nbconvert==7.6.0
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
+# Enable webpdf export DH-325
+  - nbconvert[webpdf]==7.16.4
 # Conda only has 0.12.0
   - okpy==1.18.1
   - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
diff --git a/deployments/data8/image/postBuild b/deployments/data8/image/postBuild
new file mode 100644
index 000000000..1b6bcc9d1
--- /dev/null
+++ b/deployments/data8/image/postBuild
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# installing chromium browser to enable webpdf conversion using nbconvert
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+playwright install chromium
diff --git a/deployments/data8/image/start b/deployments/data8/image/start
new file mode 100644
index 000000000..c3a978b7f
--- /dev/null
+++ b/deployments/data8/image/start
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# See https://jira-secure.berkeley.edu/browse/DH-305
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+exec "$@"

From 6191586dd1ff29e56988cf479aa38e9608f2b985 Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 11:40:33 -0700
Subject: [PATCH 17/29] [DH-326] make webpdf export work for eecs

---
 deployments/eecs/image/apt.txt         | 4 ++++
 deployments/eecs/image/environment.yml | 2 ++
 deployments/eecs/image/postBuild       | 6 +++++-
 deployments/eecs/image/start           | 5 +++++
 4 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 deployments/eecs/image/start

diff --git a/deployments/eecs/image/apt.txt b/deployments/eecs/image/apt.txt
index 56c697992..6c3d13462 100644
--- a/deployments/eecs/image/apt.txt
+++ b/deployments/eecs/image/apt.txt
@@ -38,3 +38,7 @@ texlive-lang-chinese
 
 # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979			
 zip
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-325
+libnss3
+libnspr4
diff --git a/deployments/eecs/image/environment.yml b/deployments/eecs/image/environment.yml
index c481bdd5a..bfb1fb8d0 100644
--- a/deployments/eecs/image/environment.yml
+++ b/deployments/eecs/image/environment.yml
@@ -64,6 +64,8 @@ dependencies:
   - PyQt5==5.15.4
 # From https://github.com/berkeley-dsep-infra/datahub/issues/1363#issuecomment-598916469
   - lcapy==0.96
+# make webpdf export work DH-326
+  - nbconvert[webpdf]==7.16.4
 # For desktop environment
   - jupyter-desktop-server==0.1.3
   - -r infra-requirements.txt
diff --git a/deployments/eecs/image/postBuild b/deployments/eecs/image/postBuild
index 7c3e8550c..b1b290345 100644
--- a/deployments/eecs/image/postBuild
+++ b/deployments/eecs/image/postBuild
@@ -1,6 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 # 2024-01-13 sknapp: incompatible due to notebook 7
 # jupyter contrib nbextensions install --sys-prefix --symlink
 # jupyter nbextensions_configurator enable --sys-prefix
+
+# installing chromium browser to enable webpdf conversion using nbconvert
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+playwright install chromium
diff --git a/deployments/eecs/image/start b/deployments/eecs/image/start
new file mode 100644
index 000000000..c3a978b7f
--- /dev/null
+++ b/deployments/eecs/image/start
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# See https://jira-secure.berkeley.edu/browse/DH-305
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+exec "$@"

From 9fd1fa3208267eeb4fdba75f0e1774880944ad23 Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 13:26:23 -0700
Subject: [PATCH 18/29] [DH-327] make webpdf work for ischool

---
 deployments/ischool/image/Dockerfile      | 5 +++++
 deployments/ischool/image/environment.yml | 3 ++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/deployments/ischool/image/Dockerfile b/deployments/ischool/image/Dockerfile
index 0cee86939..2b3e4c1ad 100644
--- a/deployments/ischool/image/Dockerfile
+++ b/deployments/ischool/image/Dockerfile
@@ -34,6 +34,7 @@ RUN apt-get update > /dev/null && \
             libatk-bridge2.0-0 \
             libgtk-3-0 \
             libnss3 \
+            libnspr4 \
             libxss1 \
             nodejs \
             npm \
@@ -57,6 +58,10 @@ COPY infra-requirements.txt /tmp/infra-requirements.txt
 RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \
         mamba clean -afy
 
+# DH-327, very similar to what was done for datahub in DH-164
+ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
+RUN playwright install chromium
+
 # Install IRKernel
 RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \
     R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')"
diff --git a/deployments/ischool/image/environment.yml b/deployments/ischool/image/environment.yml
index 877fbc307..28fd179df 100644
--- a/deployments/ischool/image/environment.yml
+++ b/deployments/ischool/image/environment.yml
@@ -16,7 +16,8 @@ dependencies:
 - pip:
   - -r /tmp/infra-requirements.txt
   - otter-grader==3.1.4
+  # DH-327
+  - nbconvert[webpdf]==7.16.4
   # for notebook exporting
-  - nbconvert==7.6.0
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1

From c264c8e0f739eed60f21a41cd85423b2d9bfe5df Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 13:56:39 -0700
Subject: [PATCH 19/29] [DH-328] make webpdf export work for data101

---
 deployments/data101/image/apt.txt         | 4 ++++
 deployments/data101/image/environment.yml | 2 +-
 deployments/data101/image/postBuild       | 4 ++++
 deployments/data101/image/start           | 5 +++++
 4 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 deployments/data101/image/start

diff --git a/deployments/data101/image/apt.txt b/deployments/data101/image/apt.txt
index b78191883..9b4955ded 100644
--- a/deployments/data101/image/apt.txt
+++ b/deployments/data101/image/apt.txt
@@ -94,3 +94,7 @@ zip
 
 # DH-93, DH94
 postgresql-client
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-305
+libnss3
+libnspr4
diff --git a/deployments/data101/image/environment.yml b/deployments/data101/image/environment.yml
index 00c74a51c..313b9c50b 100644
--- a/deployments/data101/image/environment.yml
+++ b/deployments/data101/image/environment.yml
@@ -101,7 +101,7 @@ dependencies:
   # disable until fixed (probably this:  https://github.com/jupyterlab/jupyter-collaboration/issues/162)
   # - jupyter_collaboration==1.0.1
   - jupyterhub==4.1.5
-  - nbconvert==7.6.0
+  - nbconvert[webpdf]==7.16.4
   - pytest-notebook==0.8.1
   - pymongo==4.4.1
   - dbt-postgres==1.6.0
diff --git a/deployments/data101/image/postBuild b/deployments/data101/image/postBuild
index 6116a2436..8b16a7df1 100644
--- a/deployments/data101/image/postBuild
+++ b/deployments/data101/image/postBuild
@@ -13,3 +13,7 @@ chmod 775 ${CONDA_DIR}/bin/mongosh
 chmod 775 ${CONDA_DIR}/lib/mongosh_crypt_v1.so
 cd /
 rm -rf /tmp/mongosh
+
+# installing chromium browser to enable webpdf conversion using nbconvert
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+playwright install chromium
diff --git a/deployments/data101/image/start b/deployments/data101/image/start
new file mode 100644
index 000000000..c3a978b7f
--- /dev/null
+++ b/deployments/data101/image/start
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# See https://jira-secure.berkeley.edu/browse/DH-305
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+exec "$@"

From 873598b8c6acaa534e9febcda244a7836d45005b Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 14:16:45 -0700
Subject: [PATCH 20/29] [DH-329] make webpdf export work for data102

---
 deployments/data102/image/apt.txt         | 4 ++++
 deployments/data102/image/environment.yml | 2 +-
 deployments/data102/image/postBuild       | 5 +++++
 deployments/data102/image/start           | 5 +++++
 4 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 deployments/data102/image/postBuild
 create mode 100644 deployments/data102/image/start

diff --git a/deployments/data102/image/apt.txt b/deployments/data102/image/apt.txt
index 2aa86a867..2cc25eaf7 100644
--- a/deployments/data102/image/apt.txt
+++ b/deployments/data102/image/apt.txt
@@ -75,3 +75,7 @@ meld
 
 # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979			
 zip
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-305
+libnss3
+libnspr4
diff --git a/deployments/data102/image/environment.yml b/deployments/data102/image/environment.yml
index c312f3f52..2e74cf4b6 100644
--- a/deployments/data102/image/environment.yml
+++ b/deployments/data102/image/environment.yml
@@ -47,7 +47,7 @@ dependencies:
   # disable until fixed (probably this:  https://github.com/jupyterlab/jupyter-collaboration/issues/162)
   # - jupyter_collaboration==1.0.1
   - jupyterhub==4.1.5
-  - nbconvert==7.6.0
+  - nbconvert[webpdf]==7.16.4
   - gh-scoped-creds==4.1
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
diff --git a/deployments/data102/image/postBuild b/deployments/data102/image/postBuild
new file mode 100644
index 000000000..1b6bcc9d1
--- /dev/null
+++ b/deployments/data102/image/postBuild
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# installing chromium browser to enable webpdf conversion using nbconvert
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+playwright install chromium
diff --git a/deployments/data102/image/start b/deployments/data102/image/start
new file mode 100644
index 000000000..c3a978b7f
--- /dev/null
+++ b/deployments/data102/image/start
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# See https://jira-secure.berkeley.edu/browse/DH-305
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+exec "$@"

From 80fb480b1b035d385aa510bfe56c20f7192589eb Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 15:38:36 -0700
Subject: [PATCH 21/29] [DH-330] make webpdf export work for julia

---
 deployments/julia/image/Dockerfile      | 4 ++++
 deployments/julia/image/apt.txt         | 4 ++++
 deployments/julia/image/environment.yml | 2 +-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/deployments/julia/image/Dockerfile b/deployments/julia/image/Dockerfile
index 27cf5ba06..9c0262d1e 100644
--- a/deployments/julia/image/Dockerfile
+++ b/deployments/julia/image/Dockerfile
@@ -66,6 +66,10 @@ RUN mamba env update -p ${CONDA_DIR}  -f /tmp/environment.yml && mamba clean -af
 COPY infra-requirements.txt /tmp/infra-requirements.txt
 RUN pip install --no-cache -r /tmp/infra-requirements.txt
 
+# DH-330, similar to DH-164
+ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
+RUN playwright install chromium
+
 # 2024-01-13 sknapp: incompatible due to notebook 7
 # RUN jupyter contrib nbextensions install --sys-prefix --symlink && \
 #     jupyter nbextensions_configurator enable --sys-prefix
diff --git a/deployments/julia/image/apt.txt b/deployments/julia/image/apt.txt
index 14c246c04..30ae36d48 100644
--- a/deployments/julia/image/apt.txt
+++ b/deployments/julia/image/apt.txt
@@ -9,3 +9,7 @@ lmodern
 # Other useful document-related tools
 pandoc
 latexdiff
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-305
+libnss3
+libnspr4
diff --git a/deployments/julia/image/environment.yml b/deployments/julia/image/environment.yml
index d2faefc2e..13ea0630e 100644
--- a/deployments/julia/image/environment.yml
+++ b/deployments/julia/image/environment.yml
@@ -23,6 +23,6 @@ dependencies:
   - jupyter-desktop-server
   - otter-grader==3.1.4
   # for notebook exporting
-  - nbconvert==7.6.0
+  - nbconvert[webpdf]==7.16.4
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1

From 5b9d50c174b54cdeba76fbc1a457a01dfc14908d Mon Sep 17 00:00:00 2001
From: Balaji Alwar <balajialwar@berkeley.edu>
Date: Tue, 16 Jul 2024 16:05:45 -0700
Subject: [PATCH 22/29] Increase RAM for to 4 GB  Data 100 summer assignment

---
 deployments/data100/config/common.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/deployments/data100/config/common.yaml b/deployments/data100/config/common.yaml
index 8f92b32ad..e7cda9b77 100644
--- a/deployments/data100/config/common.yaml
+++ b/deployments/data100/config/common.yaml
@@ -102,13 +102,13 @@ jupyterhub:
         admin: true
         mem_limit: 4G
 
-      # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376
-      #course::1531798::group::Content: # Spring 2024 Content Team, ensured 4G RAM
-      #  mem_limit: 4G
-      #  mem_guarantee: 4G
+      # Data 100, Summer 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5862
+      course::1535115:
+        mem_limit: 4G
+        mem_guarantee: 4G
 
-      # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376
-      #course::1531798::group::Admins: # Spring 2024, Data 100 Admins, ensured 4G RAM
+      # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376, to be removed by 7/29
+      #course::1531798::group::Admins: # Summer 2024, Data 100 Admins, ensured 4G RAM
       #  mem_limit: 4G
       #  mem_guarantee: 4G
     admin:

From afe86d69994278ccf902ddc9075e9a808008ea1c Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 16:10:54 -0700
Subject: [PATCH 23/29] [DH-331] make webpdf export work for publichealth

---
 deployments/publichealth/image/Dockerfile      | 5 +++++
 deployments/publichealth/image/environment.yml | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/deployments/publichealth/image/Dockerfile b/deployments/publichealth/image/Dockerfile
index 5a233b9f9..ded0c4edd 100644
--- a/deployments/publichealth/image/Dockerfile
+++ b/deployments/publichealth/image/Dockerfile
@@ -37,6 +37,7 @@ RUN apt-get update > /dev/null && \
             libatk-bridge2.0-0 \
             libgtk-3-0 \
             libnss3 \
+            libnspr4 \
             libxss1 \
             fonts-symbola \
             gdebi-core \
@@ -62,6 +63,10 @@ RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \
 COPY infra-requirements.txt /tmp/infra-requirements.txt
 RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt
 
+# DH-331, very similar to what was done for datahub in DH-164
+ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
+RUN playwright install chromium
+
 # Install IRKernel
 RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \
     R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')"
diff --git a/deployments/publichealth/image/environment.yml b/deployments/publichealth/image/environment.yml
index 63f3c750d..a6a138564 100644
--- a/deployments/publichealth/image/environment.yml
+++ b/deployments/publichealth/image/environment.yml
@@ -10,7 +10,7 @@ dependencies:
   # RStudio support
   - otter-grader==2.2.7
   # for notebook exporting
-  - nbconvert==7.6.0
+  - nbconvert[webpdf]==7.16.4
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
   # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323

From f010d43b44442edeb7edcd5fce61977209ec2f0e Mon Sep 17 00:00:00 2001
From: Balaji Alwar <balajialwar@berkeley.edu>
Date: Tue, 16 Jul 2024 16:12:46 -0700
Subject: [PATCH 24/29] Fixing typos

---
 deployments/data100/config/common.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/deployments/data100/config/common.yaml b/deployments/data100/config/common.yaml
index e7cda9b77..ba7daa602 100644
--- a/deployments/data100/config/common.yaml
+++ b/deployments/data100/config/common.yaml
@@ -102,13 +102,13 @@ jupyterhub:
         admin: true
         mem_limit: 4G
 
-      # Data 100, Summer 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5862
+      # Data 100, Summer 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5862, to be removed by 7/29
       course::1535115:
         mem_limit: 4G
         mem_guarantee: 4G
 
-      # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376, to be removed by 7/29
-      #course::1531798::group::Admins: # Summer 2024, Data 100 Admins, ensured 4G RAM
+      # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376
+      #course::1531798::group::Admins: # Spring 2024, Data 100 Admins, ensured 4G RAM
       #  mem_limit: 4G
       #  mem_guarantee: 4G
     admin:

From ee576440e49c0f1106187ff12f865b7a39747c47 Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 16:27:42 -0700
Subject: [PATCH 25/29] [DH-333] making webpdf export work for astro

---
 deployments/astro/image/apt.txt         | 4 ++++
 deployments/astro/image/environment.yml | 2 +-
 deployments/astro/image/postBuild       | 6 +++++-
 deployments/astro/image/start           | 5 +++++
 4 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 deployments/astro/image/start

diff --git a/deployments/astro/image/apt.txt b/deployments/astro/image/apt.txt
index 895f0e6c3..422e2ccbc 100644
--- a/deployments/astro/image/apt.txt
+++ b/deployments/astro/image/apt.txt
@@ -24,3 +24,7 @@ vim
 
 # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979			
 zip
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-305
+libnss3
+libnspr4
diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml
index 16d4ab815..325a15304 100644
--- a/deployments/astro/image/environment.yml
+++ b/deployments/astro/image/environment.yml
@@ -106,6 +106,6 @@ dependencies:
   - twine
   - otter-grader==3.1.4
   # for notebook exporting
-  - nbconvert==7.6.0
+  - nbconvert[webpdf]==7.16.4
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1
diff --git a/deployments/astro/image/postBuild b/deployments/astro/image/postBuild
index 7c3e8550c..b1b290345 100644
--- a/deployments/astro/image/postBuild
+++ b/deployments/astro/image/postBuild
@@ -1,6 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 # 2024-01-13 sknapp: incompatible due to notebook 7
 # jupyter contrib nbextensions install --sys-prefix --symlink
 # jupyter nbextensions_configurator enable --sys-prefix
+
+# installing chromium browser to enable webpdf conversion using nbconvert
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+playwright install chromium
diff --git a/deployments/astro/image/start b/deployments/astro/image/start
new file mode 100644
index 000000000..c3a978b7f
--- /dev/null
+++ b/deployments/astro/image/start
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# See https://jira-secure.berkeley.edu/browse/DH-305
+export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR}
+exec "$@"

From cd1ed2ad2b3a9e802ad3ef599807c65576f45e98 Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 18:31:38 -0700
Subject: [PATCH 26/29] [DH-330] adding strace to try and debug 100% cpu issue
 with chromium

---
 deployments/julia/image/apt.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deployments/julia/image/apt.txt b/deployments/julia/image/apt.txt
index 30ae36d48..2a0dd4de6 100644
--- a/deployments/julia/image/apt.txt
+++ b/deployments/julia/image/apt.txt
@@ -11,5 +11,6 @@ pandoc
 latexdiff
 
 # playwright deps https://jira-secure.berkeley.edu/browse/DH-305
+strace
 libnss3
 libnspr4

From 29247c102eb315b58bd0b5abd91e2ba8885baffc Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Tue, 16 Jul 2024 19:11:07 -0700
Subject: [PATCH 27/29] [DH-330] trying dependency install

---
 deployments/julia/image/Dockerfile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/deployments/julia/image/Dockerfile b/deployments/julia/image/Dockerfile
index 9c0262d1e..d4095a73a 100644
--- a/deployments/julia/image/Dockerfile
+++ b/deployments/julia/image/Dockerfile
@@ -66,6 +66,13 @@ RUN mamba env update -p ${CONDA_DIR}  -f /tmp/environment.yml && mamba clean -af
 COPY infra-requirements.txt /tmp/infra-requirements.txt
 RUN pip install --no-cache -r /tmp/infra-requirements.txt
 
+USER root
+ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
+RUN playwright install-deps
+RUN chown -Rh jovyan:jovyan /srv/conda
+
+USER ${NB_USER}
+
 # DH-330, similar to DH-164
 ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
 RUN playwright install chromium

From 883fe83f4a490d05bc6296adbd79e74988a8bbcf Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Wed, 17 Jul 2024 00:39:10 -0700
Subject: [PATCH 28/29] [DH-332] make webpdf export work for biology

---
 deployments/biology/image/Dockerfile      | 11 +++++++++++
 deployments/biology/image/environment.yml |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile
index 0dc7046f4..8e11cbfbb 100644
--- a/deployments/biology/image/Dockerfile
+++ b/deployments/biology/image/Dockerfile
@@ -172,6 +172,17 @@ COPY infra-requirements.txt /tmp/
 RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \
     mamba clean -afy
 
+USER root
+ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
+RUN playwright install-deps
+RUN chown -Rh jovyan:jovyan /srv/conda
+
+USER ${NB_USER}
+
+# DH-333
+ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR}
+RUN playwright install chromium
+
 # 2024-01-13 sknapp: incompatible due to notebook 7
 # RUN jupyter contrib nbextensions install --sys-prefix --symlink && \
 #     jupyter nbextensions_configurator enable --sys-prefix
diff --git a/deployments/biology/image/environment.yml b/deployments/biology/image/environment.yml
index e4c9924eb..43f807b0c 100644
--- a/deployments/biology/image/environment.yml
+++ b/deployments/biology/image/environment.yml
@@ -74,6 +74,6 @@ dependencies:
   - allensdk==2.16.2
   - otter-grader==3.1.4
 # for exporting notebooks to pdf
-  - nbconvert==7.6.0
+  - nbconvert[webpdf]==7.16.4
   - nb2pdf==0.6.2
   - nbpdfexport==0.2.1

From 86af01baaa715aa2d01d181500c465b1a475e5ad Mon Sep 17 00:00:00 2001
From: Jonathan Felder <felder@berkeley.edu>
Date: Wed, 17 Jul 2024 12:10:37 -0700
Subject: [PATCH 29/29] [DH-330] disabling webpdf due to DISPLAY variable being
 set causing 100% cpu spike and hang

---
 deployments/julia/config/common.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deployments/julia/config/common.yaml b/deployments/julia/config/common.yaml
index 498b399a4..9e8300a67 100644
--- a/deployments/julia/config/common.yaml
+++ b/deployments/julia/config/common.yaml
@@ -43,6 +43,8 @@ jupyterhub:
         stringData: |
           c.QtPDFExporter.enabled = False
           c.QtPNGExporter.enabled = False
+          # Causes 100% cpu spike and hang if DISPLAY is set - see DH-330
+          c.WebPDFExporter.enabled = False
     extraEnv:
       # Tell code where to display GUIs
       # The VNC /desktop link must be opened already for this to work