diff --git a/deployments/a11y/image/environment.yml b/deployments/a11y/image/environment.yml index 6845a2d6a..a67f3415d 100644 --- a/deployments/a11y/image/environment.yml +++ b/deployments/a11y/image/environment.yml @@ -19,8 +19,8 @@ dependencies: - numpy==1.24.2 - plotly==5.13.1 - pyopenssl==23.1.0 -- requests==2.28.2 -- urllib3==1.26.15 +- requests==2.32.3 +- urllib3==2.2.2 - websockify==0.11.0 - pip==23.0.1 - jupyterthemes==0.20.0 @@ -36,6 +36,8 @@ dependencies: - nb2pdf==0.6.2 - nbpdfexport==0.2.1 - jupyterlab-a11y-checker==0.1.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 # ### # The items below are from infra-requirements, however lab conflicts with the # alpha notebook. diff --git a/deployments/astro/image/apt.txt b/deployments/astro/image/apt.txt index 895f0e6c3..422e2ccbc 100644 --- a/deployments/astro/image/apt.txt +++ b/deployments/astro/image/apt.txt @@ -24,3 +24,7 @@ vim # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 zip + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 +libnss3 +libnspr4 diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 16d4ab815..325a15304 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -106,6 +106,6 @@ dependencies: - twine - otter-grader==3.1.4 # for notebook exporting - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 diff --git a/deployments/astro/image/postBuild b/deployments/astro/image/postBuild index 7c3e8550c..b1b290345 100644 --- a/deployments/astro/image/postBuild +++ b/deployments/astro/image/postBuild @@ -1,6 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # 2024-01-13 sknapp: incompatible due to notebook 7 # jupyter contrib nbextensions install --sys-prefix --symlink # jupyter nbextensions_configurator enable --sys-prefix + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +playwright install chromium diff --git a/deployments/astro/image/start b/deployments/astro/image/start new file mode 100644 index 000000000..c3a978b7f --- /dev/null +++ b/deployments/astro/image/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index 0dc7046f4..8e11cbfbb 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -172,6 +172,17 @@ COPY infra-requirements.txt /tmp/ RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ mamba clean -afy +USER root +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install-deps +RUN chown -Rh jovyan:jovyan /srv/conda + +USER ${NB_USER} + +# DH-333 +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install chromium + # 2024-01-13 sknapp: incompatible due to notebook 7 # RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ # jupyter nbextensions_configurator enable --sys-prefix diff --git a/deployments/biology/image/environment.yml b/deployments/biology/image/environment.yml index e4c9924eb..43f807b0c 100644 --- a/deployments/biology/image/environment.yml +++ b/deployments/biology/image/environment.yml @@ -74,6 +74,6 @@ dependencies: - allensdk==2.16.2 - otter-grader==3.1.4 # for exporting notebooks to pdf - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 diff --git a/deployments/data100/config/common.yaml b/deployments/data100/config/common.yaml index 8f92b32ad..ba7daa602 100644 --- a/deployments/data100/config/common.yaml +++ b/deployments/data100/config/common.yaml @@ -102,10 +102,10 @@ jupyterhub: admin: true mem_limit: 4G - # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376 - #course::1531798::group::Content: # Spring 2024 Content Team, ensured 4G RAM - # mem_limit: 4G - # mem_guarantee: 4G + # Data 100, Summer 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5862, to be removed by 7/29 + course::1535115: + mem_limit: 4G + mem_guarantee: 4G # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376 #course::1531798::group::Admins: # Spring 2024, Data 100 Admins, ensured 4G RAM diff --git a/deployments/data100/image/environment.yml b/deployments/data100/image/environment.yml index 2f72be25e..ffbd747dd 100644 --- a/deployments/data100/image/environment.yml +++ b/deployments/data100/image/environment.yml @@ -61,3 +61,5 @@ dependencies: - otter-grader==5.4.0 - duckdb==0.10.1 - duckdb_engine==0.11.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/data101/image/apt.txt b/deployments/data101/image/apt.txt index b78191883..9b4955ded 100644 --- a/deployments/data101/image/apt.txt +++ b/deployments/data101/image/apt.txt @@ -94,3 +94,7 @@ zip # DH-93, DH94 postgresql-client + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 +libnss3 +libnspr4 diff --git a/deployments/data101/image/environment.yml b/deployments/data101/image/environment.yml index 99cf7e93f..313b9c50b 100644 --- a/deployments/data101/image/environment.yml +++ b/deployments/data101/image/environment.yml @@ -101,8 +101,10 @@ dependencies: # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - jupyterhub==4.1.5 - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - pytest-notebook==0.8.1 - pymongo==4.4.1 - dbt-postgres==1.6.0 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/data101/image/postBuild b/deployments/data101/image/postBuild index 6116a2436..8b16a7df1 100644 --- a/deployments/data101/image/postBuild +++ b/deployments/data101/image/postBuild @@ -13,3 +13,7 @@ chmod 775 ${CONDA_DIR}/bin/mongosh chmod 775 ${CONDA_DIR}/lib/mongosh_crypt_v1.so cd / rm -rf /tmp/mongosh + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +playwright install chromium diff --git a/deployments/data101/image/start b/deployments/data101/image/start new file mode 100644 index 000000000..c3a978b7f --- /dev/null +++ b/deployments/data101/image/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" diff --git a/deployments/data102/image/apt.txt b/deployments/data102/image/apt.txt index 2aa86a867..2cc25eaf7 100644 --- a/deployments/data102/image/apt.txt +++ b/deployments/data102/image/apt.txt @@ -75,3 +75,7 @@ meld # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 zip + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 +libnss3 +libnspr4 diff --git a/deployments/data102/image/environment.yml b/deployments/data102/image/environment.yml index 8595687fe..2e74cf4b6 100644 --- a/deployments/data102/image/environment.yml +++ b/deployments/data102/image/environment.yml @@ -47,8 +47,10 @@ dependencies: # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - jupyterhub==4.1.5 - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - gh-scoped-creds==4.1 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/data102/image/postBuild b/deployments/data102/image/postBuild new file mode 100644 index 000000000..1b6bcc9d1 --- /dev/null +++ b/deployments/data102/image/postBuild @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +playwright install chromium diff --git a/deployments/data102/image/start b/deployments/data102/image/start new file mode 100644 index 000000000..c3a978b7f --- /dev/null +++ b/deployments/data102/image/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" diff --git a/deployments/data8/image/apt.txt b/deployments/data8/image/apt.txt index e9646bf15..37e4c7a7c 100644 --- a/deployments/data8/image/apt.txt +++ b/deployments/data8/image/apt.txt @@ -77,3 +77,7 @@ meld # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 zip + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-325 +libnss3 +libnspr4 diff --git a/deployments/data8/image/environment.yml b/deployments/data8/image/environment.yml index 21ba91e63..ad3b5a542 100644 --- a/deployments/data8/image/environment.yml +++ b/deployments/data8/image/environment.yml @@ -37,10 +37,12 @@ dependencies: #- -r infra-requirements.txt - jupyter-tree-download==1.0.1 # Export notebooks as PDFs with Chrome - - nbconvert==7.6.0 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 +# Enable webpdf export DH-325 + - nbconvert[webpdf]==7.16.4 # Conda only has 0.12.0 - okpy==1.18.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/data8/image/postBuild b/deployments/data8/image/postBuild new file mode 100644 index 000000000..1b6bcc9d1 --- /dev/null +++ b/deployments/data8/image/postBuild @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +playwright install chromium diff --git a/deployments/data8/image/start b/deployments/data8/image/start new file mode 100644 index 000000000..c3a978b7f --- /dev/null +++ b/deployments/data8/image/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 7e21be4d2..757b532da 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -202,6 +202,8 @@ dependencies: # Engineering 7 https://github.com/berkeley-dsep-infra/datahub/issues/5337 - otter-grader==4.4.1 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 # Stat 165/265 requires prophet, Spring, 2024 - prophet==1.1.5 diff --git a/deployments/datahub/images/default/requirements.txt.disabled b/deployments/datahub/images/default/requirements.txt.disabled deleted file mode 100644 index 98b6233a8..000000000 --- a/deployments/datahub/images/default/requirements.txt.disabled +++ /dev/null @@ -1,189 +0,0 @@ -# data8; foundation -datascience==0.17.0 -# Until https://github.com/okpy/ok-client/pull/473 is merged -git+https://github.com/yuvipanda/ok-client@6961d778741fe61911be4d00beff9bd8afc1edf7 -folium==0.12.1 -# -# r -jupyter-server-proxy==3.2.1 -jupyter-rsession-proxy==2.0.1 -jupyter-shiny-proxy==1.1 -# -# cogsci88; -ggplot==0.11.5 -# -# cogsci131; spring 2018 -nose==1.3.7 -# -# modules -beautifulsoup4==4.9.3 -# nb2pdf==0.6.2 commented out by sknapp 06.10.2023 to unblock https://github.com/berkeley-dsep-infra/datahub/issues/5062 -# -# ls 88-3; neuro -lxml==4.9.1 -tqdm==4.62.1 -mne==0.23.0 -nibabel==3.2.1 -h5py==3.7.0 -numexpr==2.7.3 -openpyxl==3.0.7 -nilearn==0.8.0 - -# data100, geog88 -# https://github.com/berkeley-dsep-infra/datahub/issues/2838 -geopandas==0.10.* -geopy==2.2.* -pysal==2.5.* -rtree==0.9.* -netcdf4==1.6.* -mplleaflet==0.0.5 -# phys 151; -emcee==3.1.0 -daft==0.1.2 -corner==2.2.1 -# -# ce88; -pymdptoolbox==4.0-b3 -# -# data-x; DL -tensorflow==2.11.1 -scikit-image==0.19.* -tables==3.7.0 -opencv-python==4.6.0.66 - -# astr 128/256; spring 2021 -astroquery==0.4.6 -astropy==5.3.3 -dustmaps==1.0.9 -george==0.4.0 -exoplanet==0.5.2 -torch==1.13.1 -torchvision==0.14.1 -pyvo==1.1 -joblib==1.3.1 -theano-pymc==1.1.2 -pymc3==3.11.2 - -# eep 153; spring 2019 -requests==2.31.0 -Pint==0.17 -# Google spreadsheets, Eric Van Dusen / Keeley Takimoto / Modules -gspread-pandas==2.3.0 -gspread==4.0.1 - -# eps 109; fall 2019 -ffmpeg-python==0.2.0 - -# data 88e; spring 2021 -plotly==5.2.1 -mpmath==1.3.0 -# sympy==1.6.2 -chart-studio==1.1.0 -csaps==1.1.0 -nbforms==0.5.1 - -# issue #875, global 150Q/pacs 190 - fall 2019 -wordcloud==1.8.1 - -# issue #929, SW 282 - fall 2019 -pyreadstat==1.1.2 - -# issue 954, EPS24 - fall 2019 -xarray==0.19.0 - -# issue 1001, Physics 188/288 - fall 2019 -umap-learn==0.5.1 -hdbscan==0.8.31 - -# espm 125/bio 105; fall 2019 -# see https://github.com/berkeley-dsep-infra/datahub/issues/1796 -bitarray==2.3.0 -nlmpy==1.0.1 - -# physics 188/288 fall, 2019 -getdist==1.3.1 -tensorflow-hub==0.12.0 -tensorflow-probability==0.13.0 - -# cs16A/B, spring 2020 -lcapy==0.96 - -# EPS 256, https://github.com/berkeley-dsep-infra/datahub/issues/1775 -obspy==1.3.0 - -# ds198 mch infodemiology, fall 2020/spring 2021 -# google apis -google-api-python-client==2.15.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==0.4.5 -google==3.0.0 - -# issue 1847; LS 22 spring 2021 -graphviz==0.17 - -# issue #1903, data h195A fall 2020 -habanero==0.7.4 - -# https://github.com/berkeley-dsep-infra/datahub/issues/1981 -ipycanvas==0.9.0 - -# data100 scientific packages -ray==1.13.* -xlrd==2.0.1 - -# data100 visualization -colorlover==0.3.0 -cufflinks==0.17.3 -altair==4.1.0 - -# data100 tools to access things -tweepy==3.10.0 -pytz==2021.1 -psycopg2==2.9.1 - -# data100 teaching -jassign==0.0.7 -dsassign==0.0.8 - -# data102 -dask==2021.10.0 -distributed==2021.10.0 -keras-applications==1.0.8 -keras-preprocessing==1.1.2 -keras==2.11.0 -keras-vis==0.4.1 -plotly-express==0.4.1 -cytoolz==0.11.0 -pyro-ppl==1.7.0 -lime==0.2.0.1 -shap==0.39.0 - -# prob140 2021 Spring -prob140==0.4.1.5 -sympy==1.8 - -# ESPM 167 - https://github.com/berkeley-dsep-infra/datahub/issues/2278 -contextily==1.1.0 - -# INDENG 142 Spring 2021 (future semesters as well) - https://github.com/berkeley-dsep-infra/datahub/issues/2314 -fancyimpute==0.7.0 - -# JupyterLab pypi extensions -jupyterlab-geojson==3.1.2 - -# IB 105 / ESPM 125, Fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2696 -geonomics==1.3.0 - -# data100, fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2767 -ipython-sql==0.4.0 - -# data100, fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2875 -metpy==1.1.0 -pooch==1.5.2 - -# PS88 https://github.com/berkeley-dsep-infra/datahub/issues/2925 -linearmodels==4.27 - -# https://github.com/berkeley-dsep-infra/datahub/issues/2950 -# Needed to work with a new enough version of httplib2 -httplib2>=0.20.2 diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml index 94ef7bd92..5f0a27d46 100644 --- a/deployments/dev/config/common.yaml +++ b/deployments/dev/config/common.yaml @@ -71,9 +71,6 @@ jupyterhub: description: "A newer repo2docker-based image with a primary focus of building dashboards." kubespawner_override: image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary:36280727a - env: - GH_SCOPED_CREDS_CLIENT_ID: "Iv23liFjgKFgkSBWph4C" - GH_SCOPED_CREDS_APP_URL: "https://github.com/apps/test-gh-scoped-creds" - display_name: "1524699: DataHub Infrastructure" slug: "1524699" description: "Regular image with per-course subpath." @@ -95,6 +92,14 @@ jupyterhub: custom: group_profiles: + # DataHub Infrastructure staff + # https://bcourses.berkeley.edu/courses/1524699/groups#tab-80607 + course::1524699::group::all-admins: + mem_limit: 12288M + mem_guarantee: 12288M + env: + GH_SCOPED_CREDS_CLIENT_ID: "Iv23liFjgKFgkSBWph4C" + GH_SCOPED_CREDS_APP_URL: "https://github.com/apps/test-gh-scoped-creds" # # # Example: increase memory for everyone affiliated with a course. # diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml index 08bb53833..cf4608931 100644 --- a/deployments/dev/images/secondary/environment.yml +++ b/deployments/dev/images/secondary/environment.yml @@ -7,69 +7,33 @@ channels: dependencies: - python==3.11.* - git==2.39.1 +- jupyterhub==4.1.5 - jupyter-resource-usage==1.0.0 - jupyterlab==4.0.11 - jupyterlab-favorites==3.0.0 +# Install myst for generating dashboards +- jupyterlab-myst==2.4.2 - jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 - jupyter_server==2.7.0 - nbgitpuller==1.2.1 - notebook==7.0.7 - folium==0.14.0 -- h5netcdf==1.0.2 - ipywidgets==8.0.7 -- jupysql==0.8.0 - jupyter-archive==3.4.0 - matplotlib==3.7.1 -- mdit-py-plugins==0.4.0 +# Install myst markdown for generating dashboards +- mystmd==1.2.9 - numpy==1.24.2 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 - scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 - seaborn==0.12.2 -- statsmodels==0.14.0 -- sqlalchemy==2.0.16 -- mlxtend==0.23.0 -# Spring 2024 data 100 -- pytorch==2.1.2 -- cpuonly==2.0 -- transformers==4.37.1 -# Spring 2024 table demos -- lxml==5.1.0 # Spring 2024 Econ 148 Packages - geopandas==0.14.2 - geopy==2.4.1 -- lifelines==0.27.8 - pycountry==22.3.5 -# Install voila for generating dashboards +# Install voila and voici for generating voila dashboards - voila==0.5.7 -# Install myst for generating dashboards -- mystmd==1.2.5 -- jupyterlab-myst==2.4.2 -#- ipywidgets==8.1.3 -#- pytest-notebook==0.10.0 -#- gh-scoped-creds==4.1 -#- ydata-profiling==4.8.3 -#- otter-grader==5.5.0 -#- python-duckdb==1.0.0 -#- jupyterhub==5.0.0 -- quarto==1.4.550 -- pip -- pip: - # - -r infra-requirements.txt - - ipywidgets==8.0.7 - # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) - # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 - - nbconvert[webpdf] - # - pyppeteer==2.0.0 - - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 - - otter-grader==5.4.0 - - duckdb==0.10.1 - - duckdb_engine==0.11.2 +- voici==0.6.1 diff --git a/deployments/edx/image/environment.yml b/deployments/edx/image/environment.yml index 871dd725b..df01bc927 100644 --- a/deployments/edx/image/environment.yml +++ b/deployments/edx/image/environment.yml @@ -49,3 +49,5 @@ dependencies: - otter-grader==4.4.1 - otter-submit==0.1.2 - sympy==1.12 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/eecs/image/apt.txt b/deployments/eecs/image/apt.txt index 56c697992..6c3d13462 100644 --- a/deployments/eecs/image/apt.txt +++ b/deployments/eecs/image/apt.txt @@ -38,3 +38,7 @@ texlive-lang-chinese # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 zip + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-325 +libnss3 +libnspr4 diff --git a/deployments/eecs/image/environment.yml b/deployments/eecs/image/environment.yml index c481bdd5a..bfb1fb8d0 100644 --- a/deployments/eecs/image/environment.yml +++ b/deployments/eecs/image/environment.yml @@ -64,6 +64,8 @@ dependencies: - PyQt5==5.15.4 # From https://github.com/berkeley-dsep-infra/datahub/issues/1363#issuecomment-598916469 - lcapy==0.96 +# make webpdf export work DH-326 + - nbconvert[webpdf]==7.16.4 # For desktop environment - jupyter-desktop-server==0.1.3 - -r infra-requirements.txt diff --git a/deployments/eecs/image/postBuild b/deployments/eecs/image/postBuild index 7c3e8550c..b1b290345 100644 --- a/deployments/eecs/image/postBuild +++ b/deployments/eecs/image/postBuild @@ -1,6 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # 2024-01-13 sknapp: incompatible due to notebook 7 # jupyter contrib nbextensions install --sys-prefix --symlink # jupyter nbextensions_configurator enable --sys-prefix + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +playwright install chromium diff --git a/deployments/eecs/image/start b/deployments/eecs/image/start new file mode 100644 index 000000000..c3a978b7f --- /dev/null +++ b/deployments/eecs/image/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" diff --git a/deployments/ischool/image/Dockerfile b/deployments/ischool/image/Dockerfile index 0cee86939..2b3e4c1ad 100644 --- a/deployments/ischool/image/Dockerfile +++ b/deployments/ischool/image/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get update > /dev/null && \ libatk-bridge2.0-0 \ libgtk-3-0 \ libnss3 \ + libnspr4 \ libxss1 \ nodejs \ npm \ @@ -57,6 +58,10 @@ COPY infra-requirements.txt /tmp/infra-requirements.txt RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ mamba clean -afy +# DH-327, very similar to what was done for datahub in DH-164 +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install chromium + # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" diff --git a/deployments/ischool/image/environment.yml b/deployments/ischool/image/environment.yml index 877fbc307..28fd179df 100644 --- a/deployments/ischool/image/environment.yml +++ b/deployments/ischool/image/environment.yml @@ -16,7 +16,8 @@ dependencies: - pip: - -r /tmp/infra-requirements.txt - otter-grader==3.1.4 + # DH-327 + - nbconvert[webpdf]==7.16.4 # for notebook exporting - - nbconvert==7.6.0 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 diff --git a/deployments/julia/config/common.yaml b/deployments/julia/config/common.yaml index 498b399a4..9e8300a67 100644 --- a/deployments/julia/config/common.yaml +++ b/deployments/julia/config/common.yaml @@ -43,6 +43,8 @@ jupyterhub: stringData: | c.QtPDFExporter.enabled = False c.QtPNGExporter.enabled = False + # Causes 100% cpu spike and hang if DISPLAY is set - see DH-330 + c.WebPDFExporter.enabled = False extraEnv: # Tell code where to display GUIs # The VNC /desktop link must be opened already for this to work diff --git a/deployments/julia/image/Dockerfile b/deployments/julia/image/Dockerfile index 27cf5ba06..d4095a73a 100644 --- a/deployments/julia/image/Dockerfile +++ b/deployments/julia/image/Dockerfile @@ -66,6 +66,17 @@ RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && mamba clean -af COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache -r /tmp/infra-requirements.txt +USER root +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install-deps +RUN chown -Rh jovyan:jovyan /srv/conda + +USER ${NB_USER} + +# DH-330, similar to DH-164 +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install chromium + # 2024-01-13 sknapp: incompatible due to notebook 7 # RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ # jupyter nbextensions_configurator enable --sys-prefix diff --git a/deployments/julia/image/apt.txt b/deployments/julia/image/apt.txt index 14c246c04..2a0dd4de6 100644 --- a/deployments/julia/image/apt.txt +++ b/deployments/julia/image/apt.txt @@ -9,3 +9,8 @@ lmodern # Other useful document-related tools pandoc latexdiff + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 +strace +libnss3 +libnspr4 diff --git a/deployments/julia/image/environment.yml b/deployments/julia/image/environment.yml index d2faefc2e..13ea0630e 100644 --- a/deployments/julia/image/environment.yml +++ b/deployments/julia/image/environment.yml @@ -23,6 +23,6 @@ dependencies: - jupyter-desktop-server - otter-grader==3.1.4 # for notebook exporting - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 diff --git a/deployments/logodev/image/environment.yml b/deployments/logodev/image/environment.yml index 197f1a5c5..386806bea 100644 --- a/deployments/logodev/image/environment.yml +++ b/deployments/logodev/image/environment.yml @@ -61,3 +61,5 @@ dependencies: - otter-grader==5.4.0 - duckdb==0.10.1 - duckdb_engine==0.11.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/publichealth/image/Dockerfile b/deployments/publichealth/image/Dockerfile index 5a233b9f9..ded0c4edd 100644 --- a/deployments/publichealth/image/Dockerfile +++ b/deployments/publichealth/image/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update > /dev/null && \ libatk-bridge2.0-0 \ libgtk-3-0 \ libnss3 \ + libnspr4 \ libxss1 \ fonts-symbola \ gdebi-core \ @@ -62,6 +63,10 @@ RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt +# DH-331, very similar to what was done for datahub in DH-164 +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install chromium + # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" diff --git a/deployments/publichealth/image/environment.yml b/deployments/publichealth/image/environment.yml index 6a1ca02fd..a6a138564 100644 --- a/deployments/publichealth/image/environment.yml +++ b/deployments/publichealth/image/environment.yml @@ -10,6 +10,8 @@ dependencies: # RStudio support - otter-grader==2.2.7 # for notebook exporting - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/deployments/stat159/image/environment.yml b/deployments/stat159/image/environment.yml index 61c660a13..b3f7a0358 100644 --- a/deployments/stat159/image/environment.yml +++ b/deployments/stat159/image/environment.yml @@ -137,7 +137,7 @@ dependencies: - syncthing==1.23.0 - websockify==0.11.0 - - jupyter-server-proxy==4.2.0 + - jupyter-server-proxy==4.3.0 # VS Code support - jupyter-vscode-proxy==0.2 - code-server==4.10.1 @@ -169,3 +169,5 @@ dependencies: - nbconvert==7.6.0 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 diff --git a/images/node-placeholder-scaler/requirements.in b/images/node-placeholder-scaler/requirements.in index 63da1e2a9..f92f4c0d1 100644 --- a/images/node-placeholder-scaler/requirements.in +++ b/images/node-placeholder-scaler/requirements.in @@ -1,6 +1,6 @@ -certifi==2023.7.22 +certifi==2024.07.04 DateTime ical==5.0.0 requests ruamel.yaml -urllib3==2.0.7 +urllib3==2.2.2 diff --git a/images/node-placeholder-scaler/requirements.txt b/images/node-placeholder-scaler/requirements.txt index 1091ad7b0..51ad5c580 100644 --- a/images/node-placeholder-scaler/requirements.txt +++ b/images/node-placeholder-scaler/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile requirements.in # -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic -certifi==2023.7.22 +certifi==2024.7.4 # via # -r requirements.in # requests @@ -14,15 +14,15 @@ charset-normalizer==3.3.2 # via requests datetime==5.5 # via -r requirements.in -emoji==2.11.1 +emoji==2.12.1 # via ical ical==5.0.0 # via -r requirements.in idna==3.7 # via requests -pydantic==2.7.1 +pydantic==2.8.2 # via ical -pydantic-core==2.18.2 +pydantic-core==2.20.1 # via pydantic pyparsing==3.1.2 # via ical @@ -30,7 +30,7 @@ python-dateutil==2.9.0.post0 # via ical pytz==2024.1 # via datetime -requests==2.31.0 +requests==2.32.3 # via -r requirements.in ruamel-yaml==0.18.6 # via -r requirements.in @@ -38,17 +38,18 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml six==1.16.0 # via python-dateutil -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via + # emoji # pydantic # pydantic-core tzdata==2024.1 # via ical -urllib3==2.0.7 +urllib3==2.2.2 # via # -r requirements.in # requests -zope-interface==6.3 +zope-interface==6.4.post2 # via datetime # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements.txt b/requirements.txt index 79bd808f8..ba67b08a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ jupyter-repo2docker==2024.3.0 myst-parser chardet # requests==2.31.0 -requests<2.29.0 -urllib3<2.0 +requests==2.32.3 +urllib3==2.2.2