From a0ee4b6d6d82af1f94a3ab6a618d31e057ff93bd Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 7 Nov 2018 15:44:06 -0800 Subject: [PATCH 001/184] Add data100 deployment. staging only for now. --- Dockerfile | 132 +++++++++++++++++++++++++++++++++++++ Makefile | 12 ++++ environment.yml | 89 +++++++++++++++++++++++++ ipython_config.py | 3 + jupyter_notebook_config.py | 2 + 5 files changed, 238 insertions(+) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 environment.yml create mode 100644 ipython_config.py create mode 100644 jupyter_notebook_config.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0219d34 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,132 @@ +# Dockerfile for Data100 User Image for Spring 2018 + +# Basic Ubuntu image creation and setup +FROM ubuntu:18.04 + +# Core package setup +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + locales \ + python3.6 \ + python3.6-dev \ + build-essential \ + man \ + less \ + tar \ + git \ + ca-certificates \ + curl \ + wget \ + vim \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-generic-recommended \ + ; + +# Other packages for user convenience and Data100 usage +# Install these without 'recommended' packages to keep image smaller. +RUN apt-get install --no-install-recommends --yes \ + screen \ + openssh-client \ + tmux \ + emacs-nox \ + nano \ + mc \ + htop \ + postgresql-client \ + # for nbconvert + pandoc \ + default-jdk + +# Keep image size at a minimum +RUN apt-get clean + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ + locale-gen + +#################################################################### +# Prepare container user (jovyan), with paths that Conda will use + +ENV CONDA_PREFIX /srv/conda +ENV PATH ${CONDA_PREFIX}/bin:$PATH + +RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan + +RUN install -d -o jovyan -g jovyan ${CONDA_PREFIX} + +WORKDIR /home/jovyan + +USER jovyan + +#################################################################### +# Download, install and configure the Conda environment + +RUN curl -o /tmp/miniconda.sh \ + https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh + +# Install miniconda +RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} + +RUN conda config --set always_yes yes --set changeps1 no +RUN conda update -q conda + +# Encapsulate the environment info into its own yml file (which carries +# the name `data100` in it +COPY environment.yml /tmp/ +RUN conda env create -q -f /tmp/environment.yml + +# We modify the path directly since the `source activate data100` +# environment won't be preserved here. +ENV PATH ${CONDA_PREFIX}/envs/data100/bin:$PATH + +# Set bash as shell in terminado. Disable history. +ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ +ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ + +RUN pip install jupyterhub==0.9.2 + +RUN pip install psycopg2==2.7.5 + +# Installed in conda environment +RUN jupyter serverextension enable --sys-prefix --py jupyterlab + +#RUN pip install nbgrader==0.5.4 +RUN pip install git+git://github.com/berkeley-dsep-infra/nbgrader.git@7aba39a +RUN jupyter nbextension install --sys-prefix --py nbgrader +RUN jupyter nbextension enable --sys-prefix --py nbgrader +RUN jupyter serverextension enable --sys-prefix --py nbgrader + +# For dask dashboard +RUN pip install nbserverproxy==0.8.3 +RUN jupyter serverextension enable --sys-prefix --py nbserverproxy + +# Disabling the autosavetime interval +# Install nb extensions +RUN pip install jupyter_contrib_nbextensions +# We install, ignoring running servers +# RUN jupyter contrib nbextensions install --sys-prefix --skip-running-check +# Enable autosavetime which will by default disable autosave +# RUN jupyter nbextension enable autosavetime/main + +# Install nbgitpuller +RUN pip install --no-cache-dir nbgitpuller==0.6.1 + +# Install nbzip +RUN pip install --no-cache-dir nbzip==0.1.0 + +RUN jupyter serverextension enable --sys-prefix --py nbzip +RUN jupyter nbextension install --sys-prefix --py nbzip +RUN jupyter nbextension enable --sys-prefix --py nbzip + +RUN pip install ray==0.5.3 + +# Useful for debugging any issues with conda +RUN conda info -a + +#################################################################### +# Make JupyterHub ports visible +EXPOSE 8888 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..09b1868 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +VERSION=$(shell git rev-parse --short HEAD) +IMAGE=berkeleydsep/singleuser-data100 + +release: VERSION=$(shell git tag -l --points-at HEAD) + +build: + docker build -t $(IMAGE):$(VERSION) . + +push: + docker push $(IMAGE):$(VERSION) + +release: build push diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..547588e --- /dev/null +++ b/environment.yml @@ -0,0 +1,89 @@ +name: data100 +channels: +- defaults +dependencies: +- attrs=17.3.0 +- beautifulsoup4=4.6.0 +- bleach=2.1.2 +- chardet=3.0.4 +- coverage=4.4.2 +- cryptography=2.1.4 +- cycler=0.10.0 +- dask=0.16.1 +- decorator=4.2.1 +- entrypoints=0.2.3 +- expat=2.2.5 +- flake8=3.5.0 +- html5lib=1.0.1 +- hypothesis=3.38.5 +- ipykernel=4.7.0 +- ipython=6.2.1 +- ipython_genutils=0.2.0 +- ipywidgets=7.1.0 +- jedi=0.11.1 +- joblib=0.11 +- jsonschema=2.6.0 +- jupyter=1.0.0 +- jupyter_client=5.2.1 +- jupyter_console=5.2.0 +- jupyter_core=4.4.0 +- lxml=4.1.1 +- matplotlib=2.1.1 +- mistune=0.8.3 +- nbconvert=5.3.1 +- nbformat=4.4.0 +- networkx=2.0 +- notebook=5.4.1 +- numpy=1.12.1 +- pandas=0.22 +- pandoc=1.19.2.1 +- pandocfilters=1.4.2 +- patsy=0.5.0 +- pcre=8.41 +- pep8=1.7.1 +- pickleshare=0.7.4 +- pip=9.0.1 +- plotly=2.2.2 +- pluggy=0.6.0 +- prompt_toolkit=1.0.15 +- protobuf=3.4.1 +- py=1.5.2 +- pyflakes=1.5.0 +- pygments=2.2.0 +- pylint=1.8.1 +- pyparsing=2.2.0 +- pytables=3.4.4 +- pytest=3.3.2 +- python=3.6.4 +- python-dateutil=2.6.1 +- pytz=2017.3 +- pyzmq=16.0.3 +- requests=2.18.4 +- scikit-image=0.13.0 +- scikit-learn=0.19.1 +- scipy=1.0.0 +- seaborn=0.9.0 +- setuptools=38.4.0 +- simplegeneric=0.8.1 +- sip=4.18.1 +- six=1.11.0 +- sqlalchemy=1.2.1 +- sqlite=3.20.1 +- statsmodels=0.8.0 +- sympy=1.1.1 +- tensorflow=1.1.0 +- toolz=0.9.0 +- tornado=4.5.3 +- traitlets=4.3.2 +- widgetsnbextension=3.1.0 +- xarray=0.10.0 +- xz=5.2.3 +- zeromq=4.2.2 +- pip: + - colorlover==0.2.1 + - cufflinks==0.12.1 + - jupyterlab==0.31.12 + - oauthlib==2.0.6 + - requests-oauthlib==0.8.0 + - tweepy==3.5.0 + diff --git a/ipython_config.py b/ipython_config.py new file mode 100644 index 0000000..1c229b5 --- /dev/null +++ b/ipython_config.py @@ -0,0 +1,3 @@ +# Disable history manager, we don't really use it +# and by default it puts an sqlite file on NFS, which is not something we wanna do +c.Historymanager.enabled = False diff --git a/jupyter_notebook_config.py b/jupyter_notebook_config.py new file mode 100644 index 0000000..150c3ea --- /dev/null +++ b/jupyter_notebook_config.py @@ -0,0 +1,2 @@ +# This is sh (dash) by default, not $SHELL +c.NotebookApp.terminado_settings = { "shell_command": ["bash"] } From e4ef47b8e253b12ee5cfba3c72d31058424a5f4f Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 7 Nov 2018 16:04:21 -0800 Subject: [PATCH 002/184] No-op commit. Get us a non-alpha hash for helm. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0219d34..350e98f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,6 +115,8 @@ RUN pip install jupyter_contrib_nbextensions # Install nbgitpuller RUN pip install --no-cache-dir nbgitpuller==0.6.1 +RUN pip install ray==0.5.3 + # Install nbzip RUN pip install --no-cache-dir nbzip==0.1.0 @@ -122,8 +124,6 @@ RUN jupyter serverextension enable --sys-prefix --py nbzip RUN jupyter nbextension install --sys-prefix --py nbzip RUN jupyter nbextension enable --sys-prefix --py nbzip -RUN pip install ray==0.5.3 - # Useful for debugging any issues with conda RUN conda info -a From ea0ffdf181f01cfa7870f5d95fc15d37f389af83 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Tue, 15 Jan 2019 10:12:06 -0800 Subject: [PATCH 003/184] Remove data100 deployment. --- Dockerfile | 132 ------------------------------------- Makefile | 12 ---- environment.yml | 89 ------------------------- ipython_config.py | 3 - jupyter_notebook_config.py | 2 - 5 files changed, 238 deletions(-) delete mode 100644 Dockerfile delete mode 100644 Makefile delete mode 100644 environment.yml delete mode 100644 ipython_config.py delete mode 100644 jupyter_notebook_config.py diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 350e98f..0000000 --- a/Dockerfile +++ /dev/null @@ -1,132 +0,0 @@ -# Dockerfile for Data100 User Image for Spring 2018 - -# Basic Ubuntu image creation and setup -FROM ubuntu:18.04 - -# Core package setup -RUN apt-get update && \ - apt-get install --yes --no-install-recommends \ - locales \ - python3.6 \ - python3.6-dev \ - build-essential \ - man \ - less \ - tar \ - git \ - ca-certificates \ - curl \ - wget \ - vim \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-generic-recommended \ - ; - -# Other packages for user convenience and Data100 usage -# Install these without 'recommended' packages to keep image smaller. -RUN apt-get install --no-install-recommends --yes \ - screen \ - openssh-client \ - tmux \ - emacs-nox \ - nano \ - mc \ - htop \ - postgresql-client \ - # for nbconvert - pandoc \ - default-jdk - -# Keep image size at a minimum -RUN apt-get clean - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ - locale-gen - -#################################################################### -# Prepare container user (jovyan), with paths that Conda will use - -ENV CONDA_PREFIX /srv/conda -ENV PATH ${CONDA_PREFIX}/bin:$PATH - -RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan - -RUN install -d -o jovyan -g jovyan ${CONDA_PREFIX} - -WORKDIR /home/jovyan - -USER jovyan - -#################################################################### -# Download, install and configure the Conda environment - -RUN curl -o /tmp/miniconda.sh \ - https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh - -# Install miniconda -RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} - -RUN conda config --set always_yes yes --set changeps1 no -RUN conda update -q conda - -# Encapsulate the environment info into its own yml file (which carries -# the name `data100` in it -COPY environment.yml /tmp/ -RUN conda env create -q -f /tmp/environment.yml - -# We modify the path directly since the `source activate data100` -# environment won't be preserved here. -ENV PATH ${CONDA_PREFIX}/envs/data100/bin:$PATH - -# Set bash as shell in terminado. Disable history. -ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ -ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ - -RUN pip install jupyterhub==0.9.2 - -RUN pip install psycopg2==2.7.5 - -# Installed in conda environment -RUN jupyter serverextension enable --sys-prefix --py jupyterlab - -#RUN pip install nbgrader==0.5.4 -RUN pip install git+git://github.com/berkeley-dsep-infra/nbgrader.git@7aba39a -RUN jupyter nbextension install --sys-prefix --py nbgrader -RUN jupyter nbextension enable --sys-prefix --py nbgrader -RUN jupyter serverextension enable --sys-prefix --py nbgrader - -# For dask dashboard -RUN pip install nbserverproxy==0.8.3 -RUN jupyter serverextension enable --sys-prefix --py nbserverproxy - -# Disabling the autosavetime interval -# Install nb extensions -RUN pip install jupyter_contrib_nbextensions -# We install, ignoring running servers -# RUN jupyter contrib nbextensions install --sys-prefix --skip-running-check -# Enable autosavetime which will by default disable autosave -# RUN jupyter nbextension enable autosavetime/main - -# Install nbgitpuller -RUN pip install --no-cache-dir nbgitpuller==0.6.1 - -RUN pip install ray==0.5.3 - -# Install nbzip -RUN pip install --no-cache-dir nbzip==0.1.0 - -RUN jupyter serverextension enable --sys-prefix --py nbzip -RUN jupyter nbextension install --sys-prefix --py nbzip -RUN jupyter nbextension enable --sys-prefix --py nbzip - -# Useful for debugging any issues with conda -RUN conda info -a - -#################################################################### -# Make JupyterHub ports visible -EXPOSE 8888 diff --git a/Makefile b/Makefile deleted file mode 100644 index 09b1868..0000000 --- a/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -VERSION=$(shell git rev-parse --short HEAD) -IMAGE=berkeleydsep/singleuser-data100 - -release: VERSION=$(shell git tag -l --points-at HEAD) - -build: - docker build -t $(IMAGE):$(VERSION) . - -push: - docker push $(IMAGE):$(VERSION) - -release: build push diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 547588e..0000000 --- a/environment.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: data100 -channels: -- defaults -dependencies: -- attrs=17.3.0 -- beautifulsoup4=4.6.0 -- bleach=2.1.2 -- chardet=3.0.4 -- coverage=4.4.2 -- cryptography=2.1.4 -- cycler=0.10.0 -- dask=0.16.1 -- decorator=4.2.1 -- entrypoints=0.2.3 -- expat=2.2.5 -- flake8=3.5.0 -- html5lib=1.0.1 -- hypothesis=3.38.5 -- ipykernel=4.7.0 -- ipython=6.2.1 -- ipython_genutils=0.2.0 -- ipywidgets=7.1.0 -- jedi=0.11.1 -- joblib=0.11 -- jsonschema=2.6.0 -- jupyter=1.0.0 -- jupyter_client=5.2.1 -- jupyter_console=5.2.0 -- jupyter_core=4.4.0 -- lxml=4.1.1 -- matplotlib=2.1.1 -- mistune=0.8.3 -- nbconvert=5.3.1 -- nbformat=4.4.0 -- networkx=2.0 -- notebook=5.4.1 -- numpy=1.12.1 -- pandas=0.22 -- pandoc=1.19.2.1 -- pandocfilters=1.4.2 -- patsy=0.5.0 -- pcre=8.41 -- pep8=1.7.1 -- pickleshare=0.7.4 -- pip=9.0.1 -- plotly=2.2.2 -- pluggy=0.6.0 -- prompt_toolkit=1.0.15 -- protobuf=3.4.1 -- py=1.5.2 -- pyflakes=1.5.0 -- pygments=2.2.0 -- pylint=1.8.1 -- pyparsing=2.2.0 -- pytables=3.4.4 -- pytest=3.3.2 -- python=3.6.4 -- python-dateutil=2.6.1 -- pytz=2017.3 -- pyzmq=16.0.3 -- requests=2.18.4 -- scikit-image=0.13.0 -- scikit-learn=0.19.1 -- scipy=1.0.0 -- seaborn=0.9.0 -- setuptools=38.4.0 -- simplegeneric=0.8.1 -- sip=4.18.1 -- six=1.11.0 -- sqlalchemy=1.2.1 -- sqlite=3.20.1 -- statsmodels=0.8.0 -- sympy=1.1.1 -- tensorflow=1.1.0 -- toolz=0.9.0 -- tornado=4.5.3 -- traitlets=4.3.2 -- widgetsnbextension=3.1.0 -- xarray=0.10.0 -- xz=5.2.3 -- zeromq=4.2.2 -- pip: - - colorlover==0.2.1 - - cufflinks==0.12.1 - - jupyterlab==0.31.12 - - oauthlib==2.0.6 - - requests-oauthlib==0.8.0 - - tweepy==3.5.0 - diff --git a/ipython_config.py b/ipython_config.py deleted file mode 100644 index 1c229b5..0000000 --- a/ipython_config.py +++ /dev/null @@ -1,3 +0,0 @@ -# Disable history manager, we don't really use it -# and by default it puts an sqlite file on NFS, which is not something we wanna do -c.Historymanager.enabled = False diff --git a/jupyter_notebook_config.py b/jupyter_notebook_config.py deleted file mode 100644 index 150c3ea..0000000 --- a/jupyter_notebook_config.py +++ /dev/null @@ -1,2 +0,0 @@ -# This is sh (dash) by default, not $SHELL -c.NotebookApp.terminado_settings = { "shell_command": ["bash"] } From b4a617e6dc2c2558dc6674bdfded289cd368b8c1 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 17 Jul 2019 10:15:06 -0700 Subject: [PATCH 004/184] Add the old data100 image. --- Dockerfile | 111 ++++++++++++++++++++++++++++++++++++ environment.yml | 105 ++++++++++++++++++++++++++++++++++ ipython_config.py | 3 + jupyter_nbconvert_config.py | 1 + jupyter_notebook_config.py | 12 ++++ 5 files changed, 232 insertions(+) create mode 100644 Dockerfile create mode 100644 environment.yml create mode 100644 ipython_config.py create mode 100644 jupyter_nbconvert_config.py create mode 100644 jupyter_notebook_config.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b16d6d6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,111 @@ +FROM buildpack-deps:bionic-scm + +# Set up common env variables +ENV TZ=America/Los_Angeles +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive + +RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan + +RUN apt-get update --yes +RUN apt-get install --yes \ + python3.6 \ + python3.6-venv \ + python3.6-dev \ + tar \ + vim \ + locales + +# Other packages for user convenience and Data100 usage +# Install these without 'recommended' packages to keep image smaller. +RUN apt-get install --yes --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + default-jdk \ + emacs-nox \ + git \ + htop \ + less \ + man \ + mc \ + nano \ + openssh-client \ + postgresql-client \ + screen \ + tar \ + tmux \ + wget + +RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ + locale-gen + +# for nbconvert +RUN apt-get install --yes \ + # for nbconvert + pandoc \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-generic-recommended + +# for pdf export +RUN apt-get install --yes wkhtmltopdf + +# Keep image size at a minimum +RUN apt-get clean + +ENV CONDA_PREFIX /srv/conda +ENV PATH ${CONDA_PREFIX}/bin:$PATH +RUN install -d -o jovyan -g jovyan ${CONDA_PREFIX} + +WORKDIR /home/jovyan + +# prevent bibtex from interupting nbconvert +RUN update-alternatives --install /usr/bin/bibtex bibtex /bin/true 200 + +USER jovyan + +#################################################################### +# Download, install and configure the Conda environment + +RUN curl -o /tmp/miniconda.sh \ + https://repo.continuum.io/miniconda/Miniconda3-4.4.10-Linux-x86_64.sh + +# Install miniconda +RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} + +RUN conda config --set always_yes yes --set changeps1 no +RUN conda update -q conda + +# Encapsulate the environment info into its own yml file (which carries +# the name `data100` in it +COPY environment.yml /tmp/ +RUN conda env create -q -f /tmp/environment.yml + +# We modify the path directly since the `source activate data100` +# environment won't be preserved here. +ENV PATH ${CONDA_PREFIX}/envs/data100/bin:$PATH + +# Set bash as shell in terminado. +ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ +# Disable history. +ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ + +# Installed in conda environment +RUN jupyter serverextension enable --sys-prefix --py jupyterlab + +RUN jupyter labextension install @jupyterlab/hub-extension + +RUN jupyter serverextension enable --sys-prefix --py nbzip +RUN jupyter nbextension install --sys-prefix --py nbzip +RUN jupyter nbextension enable --sys-prefix --py nbzip + +# Useful for debugging any issues with conda +RUN conda info -a + +# Make JupyterHub ports visible +EXPOSE 8888 diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..13155c6 --- /dev/null +++ b/environment.yml @@ -0,0 +1,105 @@ +name: data100 +channels: +- defaults +dependencies: +- attrs=17.3.0 +- beautifulsoup4=4.6.0 +- bleach=2.1.2 +- chardet=3.0.4 +- coverage=4.4.2 +- cryptography=2.1.4 +- cycler=0.10.0 +- dask=0.16.1 +- decorator=4.2.1 +- entrypoints=0.2.3 +- expat=2.2.5 +- flake8=3.5.0 +- html5lib=1.0.1 +- hypothesis=3.38.5 +- ipykernel=4.7.0 +- ipython=7.2.0 +- ipython_genutils=0.2.0 +- ipywidgets=7.4.2 +- jedi=0.11.1 +- joblib=0.11 +- jsonschema=2.6.0 +- jupyter=1.0.0 +- jupyter_client=5.2.3 +- jupyter_console=6.0.0 +- jupyter_core=4.4.0 +- lxml=4.1.1 +- matplotlib=3.0.1 +- mistune=0.8.3 +- nbformat=4.4.0 +- networkx=2.0 +- nodejs=10.13.0 +- notebook=5.7.8 +- numpy=1.15.4 +- oauthlib=2.1.0 +- pandas=0.23.4 +- pandoc=1.19.2.1 +- pandocfilters=1.4.2 +- patsy=0.5.0 +- pcre=8.42 +- pep8=1.7.1 +- pickleshare=0.7.4 +- pip=9.0.1 +- plotly=2.2.2 +- pluggy=0.6.0 +- prompt_toolkit=2.0.7 +- protobuf=3.6.1 +- py=1.5.2 +- pyflakes=1.5.0 +- pygments=2.2.0 +- pylint=1.8.1 +- pyparsing=2.2.0 +- pytest=3.3.2 +- python=3.6.4 +- python-dateutil=2.6.1 +- pytz=2017.3 +- pyzmq=17.1.2 +- requests=2.20.1 +- requests-oauthlib=1.0.0 +- scikit-image=0.13.0 +- scikit-learn=0.19.1 +- scipy=1.1.0 +# see below for seaborn version +#- seaborn=0.9.0 +- setuptools=38.4.0 +- simplegeneric=0.8.1 +- sip=4.19.8 +- six=1.11.0 +- sqlalchemy=1.2.1 +- sqlite=3.25.3 +- statsmodels=0.8.0 +- sympy=1.1.1 +- tensorflow=1.12.0 +- toolz=0.9.0 +- tornado=5.1.1 +- traitlets=4.3.2 +- widgetsnbextension=3.4.2 +- xarray=0.11.0 +- xz=5.2.4 +- zeromq=4.2.5 +- graphviz=2.40.1 +- pip: + - colorlover==0.2.1 + - cufflinks==0.12.1 + - jupyterlab==0.35.4 + - tweepy==3.5.0 + - nbzip==0.1.0 + - nbgitpuller==0.6.1 + - jupyter-contrib-nbextensions==0.5.1 + - psycopg2==2.7.6.1 + - jupyterhub==0.9.2 + - jupyter-server-proxy==1.0.0 + - jassign==0.0.7 + - okpy==1.13.11 + - tqdm==4.29.1 + - nbresuse==0.3.1 + # nbconvert 5.4.0 with pdf export fix + - git+https://github.com/ryanlovett/nbconvert.git@16fe302 + # Install seaborn via pip, it will update pandas along the way + - seaborn==0.9.0 + - graphviz==0.10.1 + - ray[debug]==0.6.5 diff --git a/ipython_config.py b/ipython_config.py new file mode 100644 index 0000000..1c229b5 --- /dev/null +++ b/ipython_config.py @@ -0,0 +1,3 @@ +# Disable history manager, we don't really use it +# and by default it puts an sqlite file on NFS, which is not something we wanna do +c.Historymanager.enabled = False diff --git a/jupyter_nbconvert_config.py b/jupyter_nbconvert_config.py new file mode 100644 index 0000000..79afd57 --- /dev/null +++ b/jupyter_nbconvert_config.py @@ -0,0 +1 @@ +c.PDFExporter.bib_command = ['true'] diff --git a/jupyter_notebook_config.py b/jupyter_notebook_config.py new file mode 100644 index 0000000..f140340 --- /dev/null +++ b/jupyter_notebook_config.py @@ -0,0 +1,12 @@ +# This is sh (dash) by default, not $SHELL +c.NotebookApp.terminado_settings = { "shell_command": ["bash"] } + +c.ServerProxy.servers = { + 'http-server': { + 'command': ['python3', '-m', 'http.server', '{port}'], + 'absolute_url': False, + 'launcher_entry': { + 'title': "HTTP Server" + } + } +} From f6f0cbdd85b34ba82204eeca3fc5d6787c40d284 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 17 Jul 2019 10:39:35 -0700 Subject: [PATCH 005/184] Install joblib via pip. The conda package isn't available. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 13155c6..7b882a2 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,6 @@ dependencies: - ipython_genutils=0.2.0 - ipywidgets=7.4.2 - jedi=0.11.1 -- joblib=0.11 - jsonschema=2.6.0 - jupyter=1.0.0 - jupyter_client=5.2.3 @@ -91,6 +90,7 @@ dependencies: - nbgitpuller==0.6.1 - jupyter-contrib-nbextensions==0.5.1 - psycopg2==2.7.6.1 + - joblib==0.11 - jupyterhub==0.9.2 - jupyter-server-proxy==1.0.0 - jassign==0.0.7 From 4f9bb6351d11358b6c7f704decd983cdbafb594b Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Thu, 22 Aug 2019 16:42:39 -0700 Subject: [PATCH 006/184] Add jupyter-tree-download to data100. Update nbconvert. --- environment.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 7b882a2..5e7aa94 100644 --- a/environment.yml +++ b/environment.yml @@ -86,8 +86,9 @@ dependencies: - cufflinks==0.12.1 - jupyterlab==0.35.4 - tweepy==3.5.0 - - nbzip==0.1.0 - - nbgitpuller==0.6.1 + #- nbzip==0.1.0 + - git+https://github.com/ryanlovett/jupyter-tree-download.git@a3ad2a1 + - nbgitpuller==0.7.0 - jupyter-contrib-nbextensions==0.5.1 - psycopg2==2.7.6.1 - joblib==0.11 @@ -98,7 +99,8 @@ dependencies: - tqdm==4.29.1 - nbresuse==0.3.1 # nbconvert 5.4.0 with pdf export fix - - git+https://github.com/ryanlovett/nbconvert.git@16fe302 + #- git+https://github.com/ryanlovett/nbconvert.git@16fe302 + - nbconvert==5.6.0 # Install seaborn via pip, it will update pandas along the way - seaborn==0.9.0 - graphviz==0.10.1 From b1dcbb9c6a67e676359d48d638b819e891df0776 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Thu, 22 Aug 2019 17:05:15 -0700 Subject: [PATCH 007/184] Disable nbzip. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b16d6d6..4d4327b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,9 +100,9 @@ RUN jupyter serverextension enable --sys-prefix --py jupyterlab RUN jupyter labextension install @jupyterlab/hub-extension -RUN jupyter serverextension enable --sys-prefix --py nbzip -RUN jupyter nbextension install --sys-prefix --py nbzip -RUN jupyter nbextension enable --sys-prefix --py nbzip +#RUN jupyter serverextension enable --sys-prefix --py nbzip +#RUN jupyter nbextension install --sys-prefix --py nbzip +#RUN jupyter nbextension enable --sys-prefix --py nbzip # Useful for debugging any issues with conda RUN conda info -a From cccca1be21b07426d54e0f6e9c61e6754f36a49c Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Mon, 26 Aug 2019 19:17:45 -0700 Subject: [PATCH 008/184] Update jupyter-tree-download. --- environment.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/environment.yml b/environment.yml index 5e7aa94..2c2ee8c 100644 --- a/environment.yml +++ b/environment.yml @@ -62,8 +62,6 @@ dependencies: - scikit-image=0.13.0 - scikit-learn=0.19.1 - scipy=1.1.0 -# see below for seaborn version -#- seaborn=0.9.0 - setuptools=38.4.0 - simplegeneric=0.8.1 - sip=4.19.8 @@ -87,7 +85,7 @@ dependencies: - jupyterlab==0.35.4 - tweepy==3.5.0 #- nbzip==0.1.0 - - git+https://github.com/ryanlovett/jupyter-tree-download.git@a3ad2a1 + - git+https://github.com/ryanlovett/jupyter-tree-download.git@b11745c - nbgitpuller==0.7.0 - jupyter-contrib-nbextensions==0.5.1 - psycopg2==2.7.6.1 @@ -98,8 +96,6 @@ dependencies: - okpy==1.13.11 - tqdm==4.29.1 - nbresuse==0.3.1 - # nbconvert 5.4.0 with pdf export fix - #- git+https://github.com/ryanlovett/nbconvert.git@16fe302 - nbconvert==5.6.0 # Install seaborn via pip, it will update pandas along the way - seaborn==0.9.0 From c212f825f3617e9f74f042c9aa6bb7aa8b10578d Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 11 Sep 2019 14:23:03 -0700 Subject: [PATCH 009/184] Bump nbgitpuller version to v0.7.2 Brings in https://github.com/jupyterhub/nbgitpuller/pull/104 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2c2ee8c..a1d0af2 100644 --- a/environment.yml +++ b/environment.yml @@ -86,7 +86,7 @@ dependencies: - tweepy==3.5.0 #- nbzip==0.1.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git@b11745c - - nbgitpuller==0.7.0 + - nbgitpuller==0.7.2 - jupyter-contrib-nbextensions==0.5.1 - psycopg2==2.7.6.1 - joblib==0.11 From 4e8355ca9ed92d457fe89262ae938abe726485e7 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 7 Oct 2019 12:50:38 +0530 Subject: [PATCH 010/184] Bump nbresuse version everywhere --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index a1d0af2..4dfd008 100644 --- a/environment.yml +++ b/environment.yml @@ -95,7 +95,7 @@ dependencies: - jassign==0.0.7 - okpy==1.13.11 - tqdm==4.29.1 - - nbresuse==0.3.1 + - nbresuse==0.3.2 - nbconvert==5.6.0 # Install seaborn via pip, it will update pandas along the way - seaborn==0.9.0 From 07e50eae76d2a4002b684d235001cd81efbdb189 Mon Sep 17 00:00:00 2001 From: Suraj Rampure Date: Tue, 19 Nov 2019 13:14:08 -0800 Subject: [PATCH 011/184] scikit-learn to 0.15.0 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 4dfd008..b52bc66 100644 --- a/environment.yml +++ b/environment.yml @@ -59,7 +59,7 @@ dependencies: - pyzmq=17.1.2 - requests=2.20.1 - requests-oauthlib=1.0.0 -- scikit-image=0.13.0 +- scikit-image=0.15.0 - scikit-learn=0.19.1 - scipy=1.1.0 - setuptools=38.4.0 From 0fb46000ab36cdaba7c3d1621c4414e46f02e14f Mon Sep 17 00:00:00 2001 From: William Huang Date: Fri, 24 Jan 2020 14:11:58 -0800 Subject: [PATCH 012/184] updating a bunch of python pacakges, and installing dsassign and pytorch --- environment.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index b52bc66..d87ec2c 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - defaults dependencies: - attrs=17.3.0 -- beautifulsoup4=4.6.0 +- beautifulsoup4=4.8.2 - bleach=2.1.2 - chardet=3.0.4 - coverage=4.4.2 @@ -33,9 +33,9 @@ dependencies: - networkx=2.0 - nodejs=10.13.0 - notebook=5.7.8 -- numpy=1.15.4 +- numpy=1.18.1 - oauthlib=2.1.0 -- pandas=0.23.4 +- pandas=0.25.3 - pandoc=1.19.2.1 - pandocfilters=1.4.2 - patsy=0.5.0 @@ -43,7 +43,7 @@ dependencies: - pep8=1.7.1 - pickleshare=0.7.4 - pip=9.0.1 -- plotly=2.2.2 +- plotly=4.5.0 - pluggy=0.6.0 - prompt_toolkit=2.0.7 - protobuf=3.6.1 @@ -59,15 +59,15 @@ dependencies: - pyzmq=17.1.2 - requests=2.20.1 - requests-oauthlib=1.0.0 -- scikit-image=0.15.0 -- scikit-learn=0.19.1 -- scipy=1.1.0 +- scikit-image=0.16.2 +- scikit-learn=0.22.1 +- scipy=1.4.1 - setuptools=38.4.0 - simplegeneric=0.8.1 - sip=4.19.8 - six=1.11.0 -- sqlalchemy=1.2.1 -- sqlite=3.25.3 +- sqlalchemy=1.3.12 +- sqlite=3.30.1 - statsmodels=0.8.0 - sympy=1.1.1 - tensorflow=1.12.0 @@ -81,7 +81,7 @@ dependencies: - graphviz=2.40.1 - pip: - colorlover==0.2.1 - - cufflinks==0.12.1 + - cufflinks==0.17.0 - jupyterlab==0.35.4 - tweepy==3.5.0 #- nbzip==0.1.0 @@ -101,3 +101,6 @@ dependencies: - seaborn==0.9.0 - graphviz==0.10.1 - ray[debug]==0.6.5 + - dsassign = 0.0.8 + - altair = 4.0.1 + - torch = 1.4.0 From 78f6bf874a0c7aad31899e124247bea0f6cd156a Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 28 Jan 2020 12:28:30 -0800 Subject: [PATCH 013/184] Adding requests for #1239 and dsassign for #1233. --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index b52bc66..8d92506 100644 --- a/environment.yml +++ b/environment.yml @@ -93,6 +93,7 @@ dependencies: - jupyterhub==0.9.2 - jupyter-server-proxy==1.0.0 - jassign==0.0.7 + - dsassign==0.0.8 - okpy==1.13.11 - tqdm==4.29.1 - nbresuse==0.3.2 From 3411dff2ff4a357004b708973ec3dbef50aea553 Mon Sep 17 00:00:00 2001 From: William Huang Date: Tue, 28 Jan 2020 12:40:35 -0800 Subject: [PATCH 014/184] updating data100 libraries, moving some to pip to get latest version --- environment.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/environment.yml b/environment.yml index d87ec2c..e603917 100644 --- a/environment.yml +++ b/environment.yml @@ -36,14 +36,13 @@ dependencies: - numpy=1.18.1 - oauthlib=2.1.0 - pandas=0.25.3 -- pandoc=1.19.2.1 +- pandoc=2.9.1.1 - pandocfilters=1.4.2 - patsy=0.5.0 - pcre=8.42 - pep8=1.7.1 - pickleshare=0.7.4 - pip=9.0.1 -- plotly=4.5.0 - pluggy=0.6.0 - prompt_toolkit=2.0.7 - protobuf=3.6.1 @@ -59,9 +58,6 @@ dependencies: - pyzmq=17.1.2 - requests=2.20.1 - requests-oauthlib=1.0.0 -- scikit-image=0.16.2 -- scikit-learn=0.22.1 -- scipy=1.4.1 - setuptools=38.4.0 - simplegeneric=0.8.1 - sip=4.19.8 @@ -80,6 +76,10 @@ dependencies: - zeromq=4.2.5 - graphviz=2.40.1 - pip: + - scipy=1.4.1 + - scikit-learn=0.22.1 + - scikit-image=0.16.2 + - plotly=4.5.0 - colorlover==0.2.1 - cufflinks==0.17.0 - jupyterlab==0.35.4 @@ -96,11 +96,11 @@ dependencies: - okpy==1.13.11 - tqdm==4.29.1 - nbresuse==0.3.2 - - nbconvert==5.6.0 + - nbconvert==5.6.1 # Install seaborn via pip, it will update pandas along the way - - seaborn==0.9.0 - - graphviz==0.10.1 + - seaborn==0.10.0 + - graphviz==0.13.2 - ray[debug]==0.6.5 - dsassign = 0.0.8 - altair = 4.0.1 - - torch = 1.4.0 + - torch = 1.4.0 \ No newline at end of file From 2706971caaa24c46144ddd813ecdc253de22f75e Mon Sep 17 00:00:00 2001 From: William Huang Date: Tue, 28 Jan 2020 13:02:01 -0800 Subject: [PATCH 015/184] fixed pandoc version to the apt version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e603917..6eef667 100644 --- a/environment.yml +++ b/environment.yml @@ -36,7 +36,7 @@ dependencies: - numpy=1.18.1 - oauthlib=2.1.0 - pandas=0.25.3 -- pandoc=2.9.1.1 +- pandoc=1.19.2.4 - pandocfilters=1.4.2 - patsy=0.5.0 - pcre=8.42 From 35a3f06e61468d8b60f271b3683616a4e93d9b34 Mon Sep 17 00:00:00 2001 From: William Huang Date: Tue, 28 Jan 2020 13:25:21 -0800 Subject: [PATCH 016/184] Reverted pandoc version. --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 6eef667..2954d59 100644 --- a/environment.yml +++ b/environment.yml @@ -36,7 +36,7 @@ dependencies: - numpy=1.18.1 - oauthlib=2.1.0 - pandas=0.25.3 -- pandoc=1.19.2.4 +- pandoc=1.19.2.1 - pandocfilters=1.4.2 - patsy=0.5.0 - pcre=8.42 @@ -103,4 +103,4 @@ dependencies: - ray[debug]==0.6.5 - dsassign = 0.0.8 - altair = 4.0.1 - - torch = 1.4.0 \ No newline at end of file + - torch = 1.4.0 From 5b83bdf5ac068268466b2c7fd8b6a063c1d1d403 Mon Sep 17 00:00:00 2001 From: William Huang Date: Tue, 28 Jan 2020 13:49:03 -0800 Subject: [PATCH 017/184] Pip install typos, = to == --- environment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 2954d59..058fbe0 100644 --- a/environment.yml +++ b/environment.yml @@ -76,10 +76,10 @@ dependencies: - zeromq=4.2.5 - graphviz=2.40.1 - pip: - - scipy=1.4.1 - - scikit-learn=0.22.1 - - scikit-image=0.16.2 - - plotly=4.5.0 + - scipy==1.4.1 + - scikit-learn==0.22.1 + - scikit-image==0.16.2 + - plotly==4.5.0 - colorlover==0.2.1 - cufflinks==0.17.0 - jupyterlab==0.35.4 From 59bb9c8c7dc5b76c8620afc00790828651a3757e Mon Sep 17 00:00:00 2001 From: William Huang Date: Tue, 28 Jan 2020 13:51:31 -0800 Subject: [PATCH 018/184] More pip typos --- environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 058fbe0..ef0e0c3 100644 --- a/environment.yml +++ b/environment.yml @@ -101,6 +101,6 @@ dependencies: - seaborn==0.10.0 - graphviz==0.13.2 - ray[debug]==0.6.5 - - dsassign = 0.0.8 - - altair = 4.0.1 - - torch = 1.4.0 + - dsassign==0.0.8 + - altair==4.0.1 + - torch==1.4.0 From f90e71268c8efd37727c5285b3e515ebdaab277d Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 28 Jan 2020 15:01:46 -0800 Subject: [PATCH 019/184] What's up with dsassign? --- Dockerfile | 2 +- environment.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d4327b..bdd7889 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,7 +84,7 @@ RUN conda update -q conda # Encapsulate the environment info into its own yml file (which carries # the name `data100` in it COPY environment.yml /tmp/ -RUN conda env create -q -f /tmp/environment.yml +RUN conda env create -f /tmp/environment.yml # We modify the path directly since the `source activate data100` # environment won't be preserved here. diff --git a/environment.yml b/environment.yml index 84cc9e9..ef0e0c3 100644 --- a/environment.yml +++ b/environment.yml @@ -93,7 +93,6 @@ dependencies: - jupyterhub==0.9.2 - jupyter-server-proxy==1.0.0 - jassign==0.0.7 - - dsassign==0.0.8 - okpy==1.13.11 - tqdm==4.29.1 - nbresuse==0.3.2 From 9d5b07481175087fc7022e6c74912fd6c0aabe24 Mon Sep 17 00:00:00 2001 From: William Huang Date: Thu, 30 Jan 2020 16:03:08 -0800 Subject: [PATCH 020/184] upgrading jupyter stuff --- environment.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/environment.yml b/environment.yml index ef0e0c3..2884d9f 100644 --- a/environment.yml +++ b/environment.yml @@ -16,16 +16,16 @@ dependencies: - flake8=3.5.0 - html5lib=1.0.1 - hypothesis=3.38.5 -- ipykernel=4.7.0 -- ipython=7.2.0 +- ipykernel=5.1.4 +- ipython=7.9.0 - ipython_genutils=0.2.0 - ipywidgets=7.4.2 - jedi=0.11.1 - jsonschema=2.6.0 - jupyter=1.0.0 -- jupyter_client=5.2.3 +- jupyter_client=5.3.4 - jupyter_console=6.0.0 -- jupyter_core=4.4.0 +- jupyter_core=4.6.1 - lxml=4.1.1 - matplotlib=3.0.1 - mistune=0.8.3 @@ -82,7 +82,7 @@ dependencies: - plotly==4.5.0 - colorlover==0.2.1 - cufflinks==0.17.0 - - jupyterlab==0.35.4 + - jupyterlab==1.2.6 - tweepy==3.5.0 #- nbzip==0.1.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git@b11745c @@ -90,8 +90,8 @@ dependencies: - jupyter-contrib-nbextensions==0.5.1 - psycopg2==2.7.6.1 - joblib==0.11 - - jupyterhub==0.9.2 - - jupyter-server-proxy==1.0.0 + - jupyterhub==1.1.0 + - jupyter-server-proxy==1.2.0 - jassign==0.0.7 - okpy==1.13.11 - tqdm==4.29.1 @@ -104,3 +104,4 @@ dependencies: - dsassign==0.0.8 - altair==4.0.1 - torch==1.4.0 + - xlrd==1.2.0 From 99884e5c78afa29aa4fec1c1f23af48d29973feb Mon Sep 17 00:00:00 2001 From: William Huang Date: Thu, 30 Jan 2020 17:07:25 -0800 Subject: [PATCH 021/184] fixing versioning error --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 2884d9f..1180c77 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - ipykernel=5.1.4 - ipython=7.9.0 - ipython_genutils=0.2.0 -- ipywidgets=7.4.2 +- ipywidgets=7.5.1 - jedi=0.11.1 - jsonschema=2.6.0 - jupyter=1.0.0 @@ -70,7 +70,7 @@ dependencies: - toolz=0.9.0 - tornado=5.1.1 - traitlets=4.3.2 -- widgetsnbextension=3.4.2 +- widgetsnbextension=3.5.1 - xarray=0.11.0 - xz=5.2.4 - zeromq=4.2.5 From edf49cceefe39abda37cfa635657098600c5492b Mon Sep 17 00:00:00 2001 From: William Huang Date: Sat, 1 Feb 2020 14:00:25 -0800 Subject: [PATCH 022/184] upgrading data100 to python 3.7, and associated spring cleaning --- Dockerfile | 3 ++ environment.yml | 112 +++++++++++++++++++----------------------------- 2 files changed, 46 insertions(+), 69 deletions(-) diff --git a/Dockerfile b/Dockerfile index bdd7889..593e724 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN apt-get install --yes --no-install-recommends \ git \ htop \ less \ + libpq-dev \ man \ mc \ nano \ @@ -80,6 +81,7 @@ RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} RUN conda config --set always_yes yes --set changeps1 no RUN conda update -q conda +RUN conda config --add channels conda-forge # Encapsulate the environment info into its own yml file (which carries # the name `data100` in it @@ -99,6 +101,7 @@ ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ RUN jupyter serverextension enable --sys-prefix --py jupyterlab RUN jupyter labextension install @jupyterlab/hub-extension +RUN jupyter labextension install @jupyterlab/plotly-extension #RUN jupyter serverextension enable --sys-prefix --py nbzip #RUN jupyter nbextension install --sys-prefix --py nbzip diff --git a/environment.yml b/environment.yml index 1180c77..4839c05 100644 --- a/environment.yml +++ b/environment.yml @@ -1,107 +1,81 @@ name: data100 channels: - defaults +- pytorch dependencies: -- attrs=17.3.0 +- attrs=19.3.0 - beautifulsoup4=4.8.2 -- bleach=2.1.2 -- chardet=3.0.4 -- coverage=4.4.2 -- cryptography=2.1.4 -- cycler=0.10.0 -- dask=0.16.1 -- decorator=4.2.1 -- entrypoints=0.2.3 -- expat=2.2.5 -- flake8=3.5.0 +- bleach=3.1.0 +- coverage=5.0.3 +- decorator=4.4.1 - html5lib=1.0.1 -- hypothesis=3.38.5 - ipykernel=5.1.4 - ipython=7.9.0 - ipython_genutils=0.2.0 - ipywidgets=7.5.1 -- jedi=0.11.1 -- jsonschema=2.6.0 +- jsonschema=3.2.0 - jupyter=1.0.0 - jupyter_client=5.3.4 - jupyter_console=6.0.0 - jupyter_core=4.6.1 -- lxml=4.1.1 -- matplotlib=3.0.1 -- mistune=0.8.3 -- nbformat=4.4.0 -- networkx=2.0 -- nodejs=10.13.0 -- notebook=5.7.8 +- lxml=4.5.0 +- matplotlib=3.1.2 +- nbformat=5.0.4 +- networkx=2.4 +- nodejs=13.7.0 +- notebook=6.0.3 - numpy=1.18.1 -- oauthlib=2.1.0 +- oauthlib=3.0.1 - pandas=0.25.3 -- pandoc=1.19.2.1 +- pandoc=2.9.1.1 - pandocfilters=1.4.2 -- patsy=0.5.0 -- pcre=8.42 -- pep8=1.7.1 -- pickleshare=0.7.4 -- pip=9.0.1 -- pluggy=0.6.0 -- prompt_toolkit=2.0.7 -- protobuf=3.6.1 -- py=1.5.2 -- pyflakes=1.5.0 -- pygments=2.2.0 -- pylint=1.8.1 -- pyparsing=2.2.0 -- pytest=3.3.2 -- python=3.6.4 -- python-dateutil=2.6.1 -- pytz=2017.3 -- pyzmq=17.1.2 -- requests=2.20.1 -- requests-oauthlib=1.0.0 -- setuptools=38.4.0 -- simplegeneric=0.8.1 -- sip=4.19.8 -- six=1.11.0 -- sqlalchemy=1.3.12 +- pip=20.0.2 +- protobuf=3.11.2 +- pygments=2.5.2 +- pytest=5.3.5 # needed for jupyter-tree-download +- python=3.7.6 +- python-dateutil=2.8.1 +- pytz=2019.3 +- pyzmq=18.1.1 +- requests=2.22.0 +- requests-oauthlib=1.2.0 +- setuptools=45.1.0 +- sqlalchemy=1.3.13 - sqlite=3.30.1 -- statsmodels=0.8.0 -- sympy=1.1.1 -- tensorflow=1.12.0 -- toolz=0.9.0 -- tornado=5.1.1 -- traitlets=4.3.2 +- statsmodels=0.11.0 +- sympy=1.5.1 +- tensorflow=2.0.0 +- tornado=6.0.3 +- traitlets=4.3.3 - widgetsnbextension=3.5.1 -- xarray=0.11.0 +- xarray=0.14.1 - xz=5.2.4 -- zeromq=4.2.5 -- graphviz=2.40.1 +- zeromq=4.3.2 +- graphviz=2.42.3 +- pytorch=1.4.0 - pip: - scipy==1.4.1 - scikit-learn==0.22.1 - scikit-image==0.16.2 - plotly==4.5.0 - - colorlover==0.2.1 + - colorlover==0.3.0 - cufflinks==0.17.0 - jupyterlab==1.2.6 - - tweepy==3.5.0 - #- nbzip==0.1.0 - - git+https://github.com/ryanlovett/jupyter-tree-download.git@b11745c - - nbgitpuller==0.7.2 + - tweepy==3.8.0 + - nbgitpuller==0.8.0 - jupyter-contrib-nbextensions==0.5.1 - - psycopg2==2.7.6.1 - - joblib==0.11 + - psycopg2==2.8.4 - jupyterhub==1.1.0 - jupyter-server-proxy==1.2.0 - jassign==0.0.7 - okpy==1.13.11 - - tqdm==4.29.1 - - nbresuse==0.3.2 + - tqdm==4.42.0 + - nbresuse==0.3.3 - nbconvert==5.6.1 # Install seaborn via pip, it will update pandas along the way - seaborn==0.10.0 - - graphviz==0.13.2 - - ray[debug]==0.6.5 + - ray==0.8.1 - dsassign==0.0.8 - altair==4.0.1 - - torch==1.4.0 - xlrd==1.2.0 + - git+https://github.com/ryanlovett/jupyter-tree-download.git From 862e641886550dfa71c311b8b147663d3e99cc5f Mon Sep 17 00:00:00 2001 From: William Huang Date: Sat, 1 Feb 2020 14:32:55 -0800 Subject: [PATCH 023/184] fixing jupyterlab extension command --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 593e724..68504f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,7 +101,9 @@ ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ RUN jupyter serverextension enable --sys-prefix --py jupyterlab RUN jupyter labextension install @jupyterlab/hub-extension -RUN jupyter labextension install @jupyterlab/plotly-extension +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@1.1 +RUN jupyter labextension install jupyterlab-plotly@1.5.0 +RUN jupyter labextension install plotlywidget@1.5.0 #RUN jupyter serverextension enable --sys-prefix --py nbzip #RUN jupyter nbextension install --sys-prefix --py nbzip From d9227ab09548f4728dc068d6984ff4f326ec2ab9 Mon Sep 17 00:00:00 2001 From: William Huang Date: Sun, 12 Apr 2020 22:28:32 -0700 Subject: [PATCH 024/184] updating statustmodel --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 1180c77..399db39 100644 --- a/environment.yml +++ b/environment.yml @@ -64,7 +64,7 @@ dependencies: - six=1.11.0 - sqlalchemy=1.3.12 - sqlite=3.30.1 -- statsmodels=0.8.0 +- statsmodels=0.11.1 - sympy=1.1.1 - tensorflow=1.12.0 - toolz=0.9.0 From 278b1eda0b5baa33cfaa27a7436d221dd51ada28 Mon Sep 17 00:00:00 2001 From: William Huang Date: Sun, 12 Apr 2020 23:41:22 -0700 Subject: [PATCH 025/184] updated miniconda version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 68504f4..efef5bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ USER jovyan # Download, install and configure the Conda environment RUN curl -o /tmp/miniconda.sh \ - https://repo.continuum.io/miniconda/Miniconda3-4.4.10-Linux-x86_64.sh + https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh # Install miniconda RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} From 57188a0cecc6f36d62ba29844d0739f73235bc41 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 30 May 2020 18:24:08 +0530 Subject: [PATCH 026/184] Bump jupyterlab for data100 Ref #1468 --- Dockerfile | 11 +++-------- environment.yml | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index efef5bb..c310026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,14 +100,9 @@ ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ # Installed in conda environment RUN jupyter serverextension enable --sys-prefix --py jupyterlab -RUN jupyter labextension install @jupyterlab/hub-extension -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@1.1 -RUN jupyter labextension install jupyterlab-plotly@1.5.0 -RUN jupyter labextension install plotlywidget@1.5.0 - -#RUN jupyter serverextension enable --sys-prefix --py nbzip -#RUN jupyter nbextension install --sys-prefix --py nbzip -#RUN jupyter nbextension enable --sys-prefix --py nbzip +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@2.0.0 \ + jupyterlab-plotly@4.8.1 \ + plotlywidget@4.8.1 # Useful for debugging any issues with conda RUN conda info -a diff --git a/environment.yml b/environment.yml index 4839c05..09bd102 100644 --- a/environment.yml +++ b/environment.yml @@ -60,7 +60,7 @@ dependencies: - plotly==4.5.0 - colorlover==0.3.0 - cufflinks==0.17.0 - - jupyterlab==1.2.6 + - jupyterlab==2.1.3 - tweepy==3.8.0 - nbgitpuller==0.8.0 - jupyter-contrib-nbextensions==0.5.1 From 55cc3aed21c22c20a7b1cc5f2f429b383b4f5f4e Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 30 May 2020 18:34:49 +0530 Subject: [PATCH 027/184] Use spaces not tabs in data100 Dockerfile --- Dockerfile | 62 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index c310026..ac3e7ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,45 +13,45 @@ RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan RUN apt-get update --yes RUN apt-get install --yes \ - python3.6 \ - python3.6-venv \ - python3.6-dev \ - tar \ - vim \ - locales + python3.6 \ + python3.6-venv \ + python3.6-dev \ + tar \ + vim \ + locales # Other packages for user convenience and Data100 usage # Install these without 'recommended' packages to keep image smaller. RUN apt-get install --yes --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - default-jdk \ - emacs-nox \ - git \ - htop \ - less \ - libpq-dev \ - man \ - mc \ - nano \ - openssh-client \ - postgresql-client \ - screen \ - tar \ - tmux \ - wget + build-essential \ + ca-certificates \ + curl \ + default-jdk \ + emacs-nox \ + git \ + htop \ + less \ + libpq-dev \ + man \ + mc \ + nano \ + openssh-client \ + postgresql-client \ + screen \ + tar \ + tmux \ + wget RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ - locale-gen + locale-gen # for nbconvert RUN apt-get install --yes \ - # for nbconvert - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-generic-recommended + # for nbconvert + pandoc \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-generic-recommended # for pdf export RUN apt-get install --yes wkhtmltopdf @@ -74,7 +74,7 @@ USER jovyan # Download, install and configure the Conda environment RUN curl -o /tmp/miniconda.sh \ - https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh + https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh # Install miniconda RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} From 7adb2a783254e13080d8ee4fffb263d169c217ba Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 30 May 2020 18:37:49 +0530 Subject: [PATCH 028/184] Cleanup data100 docker image - Remove unused python3.6 apt-get install. We use python from conda - Remove duplicated apt-get package installs - Make apt-get lines slightly better - we still need to remove the lists and clean cache after each install, but this is slightly better than what we had before - Pipe output of apt-get install to /dev/null to hide lots of useless output --- Dockerfile | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac3e7ec..3127a54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,18 +11,10 @@ ENV DEBIAN_FRONTEND=noninteractive RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan -RUN apt-get update --yes -RUN apt-get install --yes \ - python3.6 \ - python3.6-venv \ - python3.6-dev \ - tar \ - vim \ - locales - # Other packages for user convenience and Data100 usage # Install these without 'recommended' packages to keep image smaller. -RUN apt-get install --yes --no-install-recommends \ +RUN apt-get update -qq --yes && \ + apt-get install --yes --no-install-recommends -qq \ build-essential \ ca-certificates \ curl \ @@ -40,24 +32,23 @@ RUN apt-get install --yes --no-install-recommends \ screen \ tar \ tmux \ - wget + wget \ + vim \ + locales > /dev/null RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ locale-gen -# for nbconvert -RUN apt-get install --yes \ +RUN apt-get update -qq --yes && \ + apt-get install --yes -qq \ # for nbconvert pandoc \ texlive-xetex \ texlive-fonts-recommended \ - texlive-generic-recommended - -# for pdf export -RUN apt-get install --yes wkhtmltopdf + texlive-generic-recommended \ + wkhtmltopdf # for pdf export \ + > /dev/null -# Keep image size at a minimum -RUN apt-get clean ENV CONDA_PREFIX /srv/conda ENV PATH ${CONDA_PREFIX}/bin:$PATH From b1e56dfc7db28aa9d9540a54672057a70ea484b8 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 1 Jun 2020 16:25:53 -0700 Subject: [PATCH 029/184] add otter-grader to data 100 hub --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 4839c05..f82cd03 100644 --- a/environment.yml +++ b/environment.yml @@ -79,3 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git + - git+https://github.com/ucbds-infra/otter-grader.git@8cf21571440e38302b08aa251f952d0c32e8baf4 From a375cd45f34158819c3e388f01aef12dff3110b1 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 1 Jun 2020 18:44:43 -0700 Subject: [PATCH 030/184] edit otter commit hash --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f82cd03..0344a80 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@8cf21571440e38302b08aa251f952d0c32e8baf4 + - git+https://github.com/ucbds-infra/otter-grader.git@80c7c7dc335f2a950576cec6be8eb5ad456d7a25 From f7ed9b1a11e0e601a1dba8d22cf74d901f8354d4 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 1 Jun 2020 18:50:38 -0700 Subject: [PATCH 031/184] updated otter version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 0344a80..9206dad 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@80c7c7dc335f2a950576cec6be8eb5ad456d7a25 + - git+https://github.com/ucbds-infra/otter-grader.git@1463f96f8ed6bb264f62feba8cce7d88bc6c76ae From bd9deb9b900073825d911f09345c2e456d0afcbd Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 1 Jun 2020 20:51:03 -0700 Subject: [PATCH 032/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 9206dad..007c46f 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@1463f96f8ed6bb264f62feba8cce7d88bc6c76ae + - git+https://github.com/ucbds-infra/otter-grader.git@30e206876badcd29d8a7b70d0e3d51eb2e0af97e From 2a7289e24168e4fb386bdb556edf432b1e5fe7df Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 8 Jun 2020 15:21:38 -0700 Subject: [PATCH 033/184] bump otter on data 100 hub --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index dd43a60..d9cd645 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@30e206876badcd29d8a7b70d0e3d51eb2e0af97e + - git+https://github.com/ucbds-infra/otter-grader.git@e2c99fc08c419111af90b33753e0e82c4be6e627 From 47a4951b0d1ee45b5e43ee35266bd53dfba302c0 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 9 Jun 2020 20:58:15 -0700 Subject: [PATCH 034/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d9cd645..6d940b9 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@e2c99fc08c419111af90b33753e0e82c4be6e627 + - git+https://github.com/ucbds-infra/otter-grader.git@7b997eb759b3581e49f8c9335fec2be9a57b7e78 From e3c5a11521d7e64cc8b09f2efd34015a46898b14 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Sun, 14 Jun 2020 21:28:26 -0700 Subject: [PATCH 035/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6d940b9..d202919 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@7b997eb759b3581e49f8c9335fec2be9a57b7e78 + - git+https://github.com/ucbds-infra/otter-grader.git@075a6bc956ebe162a16498458320192ed83e8ab5 From ced462e359362dd71bf030e496f7cae25e8f6fce Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sun, 14 Jun 2020 21:29:35 -0700 Subject: [PATCH 036/184] Revert "Update environment.yml" This reverts commit e3c5a11521d7e64cc8b09f2efd34015a46898b14. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d202919..6d940b9 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@075a6bc956ebe162a16498458320192ed83e8ab5 + - git+https://github.com/ucbds-infra/otter-grader.git@7b997eb759b3581e49f8c9335fec2be9a57b7e78 From d96c0247f7296ba9e25db16ba93048b8ecbdb721 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Sun, 14 Jun 2020 21:40:22 -0700 Subject: [PATCH 037/184] bump otter on data 100 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6d940b9..98683b1 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@7b997eb759b3581e49f8c9335fec2be9a57b7e78 + - git+https://github.com/ucbds-infra/otter-grader.git@d9ccaedabbf05a18deb019a3c134bda5bbeb9762 From 6420090c3c229f673bd4f925838b5aa0c9ec58b2 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 15 Jun 2020 11:19:49 -0700 Subject: [PATCH 038/184] fix commit hash --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 98683b1..d112d6b 100644 --- a/environment.yml +++ b/environment.yml @@ -79,4 +79,4 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@d9ccaedabbf05a18deb019a3c134bda5bbeb9762 + - git+https://github.com/ucbds-infra/otter-grader.git@01b8efd5fb47a772ca0d4be80868ae720cf95b3a From df71d88a9a75b9108901bb69a577ef73232b6585 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 16 Jun 2020 11:06:53 -0700 Subject: [PATCH 039/184] Add notetaker to Dockerfile for Data 100 hub study --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3127a54..15e1c7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,6 +88,11 @@ ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ # Disable history. ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ +# install and enable notetaker for Data 100 study +RUN git clone https://github.com/yifanwu/notetaker /tmp/notetaker +RUN jupyter nbextension install --sys-prefix /tmp/notetaker/notetaker +RUN jupyter nbextension enable --sys-prefix notetaker/main + # Installed in conda environment RUN jupyter serverextension enable --sys-prefix --py jupyterlab From 5029e5edf55420d7b1a04df4923443623f1c15a6 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Tue, 16 Jun 2020 11:30:49 -0700 Subject: [PATCH 040/184] Revert "Add notetaker to Dockerfile for Data 100 hub study" This reverts commit df71d88a9a75b9108901bb69a577ef73232b6585. --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 15e1c7c..3127a54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,11 +88,6 @@ ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ # Disable history. ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ -# install and enable notetaker for Data 100 study -RUN git clone https://github.com/yifanwu/notetaker /tmp/notetaker -RUN jupyter nbextension install --sys-prefix /tmp/notetaker/notetaker -RUN jupyter nbextension enable --sys-prefix notetaker/main - # Installed in conda environment RUN jupyter serverextension enable --sys-prefix --py jupyterlab From 72009d37692b3bfdd9500b529704e550d6b00eb9 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 16 Jun 2020 12:11:13 -0700 Subject: [PATCH 041/184] add notetaker to data 100 env --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index d112d6b..b555a36 100644 --- a/environment.yml +++ b/environment.yml @@ -80,3 +80,4 @@ dependencies: - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - git+https://github.com/ucbds-infra/otter-grader.git@01b8efd5fb47a772ca0d4be80868ae720cf95b3a + - notetaker==0.0.2 # for data 100 hub study From df25da97568fe620bedad64f2e50a9c00177d89c Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 16 Jun 2020 20:20:27 -0700 Subject: [PATCH 042/184] bump otter on data 100 last time for a while (hopefully) --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index b555a36..87c2bf2 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@01b8efd5fb47a772ca0d4be80868ae720cf95b3a + - git+https://github.com/ucbds-infra/otter-grader.git@a1431f1b595f3106193c112f91c474150547409a - notetaker==0.0.2 # for data 100 hub study From 22a9a2bc39154ba3cefa0d75b29a91e6bfe03577 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Thu, 18 Jun 2020 08:25:07 -0700 Subject: [PATCH 043/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 87c2bf2..08d5912 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@a1431f1b595f3106193c112f91c474150547409a + - git+https://github.com/ucbds-infra/otter-grader.git@1a50a9d39c356a60833a7d9a0ede4c1f357c5c3d - notetaker==0.0.2 # for data 100 hub study From 4cf02b158a31f906d117d4b6ced92b1b9b33da83 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Thu, 18 Jun 2020 22:25:23 -0700 Subject: [PATCH 044/184] bump otter on data 100 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 08d5912..a4ac592 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@1a50a9d39c356a60833a7d9a0ede4c1f357c5c3d + - git+https://github.com/ucbds-infra/otter-grader.git@d86f28a78c2e3e9d7026bde44baa3761ca646aac - notetaker==0.0.2 # for data 100 hub study From a3d88071b428a39986ff7e7ed7a4404ccea3de02 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 22 Jun 2020 12:59:50 -0700 Subject: [PATCH 045/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index a4ac592..a704609 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@d86f28a78c2e3e9d7026bde44baa3761ca646aac + - git+https://github.com/ucbds-infra/otter-grader.git@69dc771e557e4b6c9e7a75938fb4aace763d1dd0 - notetaker==0.0.2 # for data 100 hub study From faebf0c4ab136894db8c437e9e605497eeb901d7 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 23 Jun 2020 15:04:25 -0700 Subject: [PATCH 046/184] Update otter on data 100 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index a704609..f58db5e 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@69dc771e557e4b6c9e7a75938fb4aace763d1dd0 + - git+https://github.com/ucbds-infra/otter-grader.git@957a1d4474fe3c83d4d1357d63ba602cf6f912fc - notetaker==0.0.2 # for data 100 hub study From 30ba4f5b90b0bcf94e9aa8eaa7be031153885bee Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 29 Jun 2020 20:51:34 -0700 Subject: [PATCH 047/184] bump otter on data 100 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f58db5e..2deba45 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@957a1d4474fe3c83d4d1357d63ba602cf6f912fc + - git+https://github.com/ucbds-infra/otter-grader.git@ae101badebfa284ea0c5c6257a4eb2a4bc50b773 - notetaker==0.0.2 # for data 100 hub study From 08f1c760b1278b5deacda8d8e27f953d50840c63 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 30 Jun 2020 18:26:52 -0700 Subject: [PATCH 048/184] Bump otter on data 100 This will be done at some point, I swear. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2deba45..16baa02 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@ae101badebfa284ea0c5c6257a4eb2a4bc50b773 + - git+https://github.com/ucbds-infra/otter-grader.git@2c72c964bba0be9f91c7f146c8ac2ad8964c5260 - notetaker==0.0.2 # for data 100 hub study From a20f5f0b0f7b5f5f69fd051befcc3a1e6780452b Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Fri, 10 Jul 2020 19:07:56 -0700 Subject: [PATCH 049/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 16baa02..24ccfdb 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - git+https://github.com/ucbds-infra/otter-grader.git@2c72c964bba0be9f91c7f146c8ac2ad8964c5260 + - otter-grader==1.0.0.b3 - notetaker==0.0.2 # for data 100 hub study From fc197207d0259f55fa7a703c4dc309846f0d59b0 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Wed, 15 Jul 2020 14:20:52 -0700 Subject: [PATCH 050/184] Bump otter on data 100 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 24ccfdb..fe1e9eb 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - otter-grader==1.0.0.b3 + - otter-grader==1.0.0.b6 - notetaker==0.0.2 # for data 100 hub study From 72cba69a563f0bd17ae551645a5c552ca752a566 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Wed, 15 Jul 2020 17:23:57 -0700 Subject: [PATCH 051/184] bump notetaker --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index fe1e9eb..6f75817 100644 --- a/environment.yml +++ b/environment.yml @@ -80,4 +80,4 @@ dependencies: - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - otter-grader==1.0.0.b6 - - notetaker==0.0.2 # for data 100 hub study + - notetaker==0.0.3 # for data 100 hub study From d8743bcb9fad2caa4d30098c2ecd7e9728b9047c Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Fri, 31 Jul 2020 15:07:07 -0700 Subject: [PATCH 052/184] Bump otter on data 100 hub --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6f75817..cea9776 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - otter-grader==1.0.0.b6 + - otter-grader==1.0.0.b8 - notetaker==0.0.3 # for data 100 hub study From 808f5e58cddc350ec219c1aca548d93fe9ce6fc7 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Fri, 31 Jul 2020 15:28:37 -0700 Subject: [PATCH 053/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index cea9776..12e1f15 100644 --- a/environment.yml +++ b/environment.yml @@ -79,5 +79,5 @@ dependencies: - altair==4.0.1 - xlrd==1.2.0 - git+https://github.com/ryanlovett/jupyter-tree-download.git - - otter-grader==1.0.0.b8 + - otter-grader==1.0.0.b9 - notetaker==0.0.3 # for data 100 hub study From 10c0ec6f781e7f87da9d8c07c15695a396afac4b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 3 Aug 2020 17:42:17 +0530 Subject: [PATCH 054/184] data100: Put pip requirements in a requirements.txt file --- environment.yml | 28 +--------------------------- requirements.txt | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 requirements.txt diff --git a/environment.yml b/environment.yml index 6f75817..9c356a3 100644 --- a/environment.yml +++ b/environment.yml @@ -54,30 +54,4 @@ dependencies: - graphviz=2.42.3 - pytorch=1.4.0 - pip: - - scipy==1.4.1 - - scikit-learn==0.22.1 - - scikit-image==0.16.2 - - plotly==4.5.0 - - colorlover==0.3.0 - - cufflinks==0.17.0 - - jupyterlab==2.1.3 - - tweepy==3.8.0 - - nbgitpuller==0.8.0 - - jupyter-contrib-nbextensions==0.5.1 - - psycopg2==2.8.4 - - jupyterhub==1.1.0 - - jupyter-server-proxy==1.2.0 - - jassign==0.0.7 - - okpy==1.13.11 - - tqdm==4.42.0 - - nbresuse==0.3.3 - - nbconvert==5.6.1 - # Install seaborn via pip, it will update pandas along the way - - seaborn==0.10.0 - - ray==0.8.1 - - dsassign==0.0.8 - - altair==4.0.1 - - xlrd==1.2.0 - - git+https://github.com/ryanlovett/jupyter-tree-download.git - - otter-grader==1.0.0.b6 - - notetaker==0.0.3 # for data 100 hub study + - -r requirements.txt \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..51a8c33 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,27 @@ +scipy==1.4.1 +scikit-learn==0.22.1 +scikit-image==0.16.2 +plotly==4.5.0 +colorlover==0.3.0 +cufflinks==0.17.0 +jupyterlab==2.1.3 +tweepy==3.8.0 +nbgitpuller==0.8.0 +jupyter-contrib-nbextensions==0.5.1 +psycopg2==2.8.4 +jupyterhub==1.1.0 +jupyter-server-proxy==1.2.0 +jassign==0.0.7 +okpy==1.13.11 +tqdm==4.42.0 +nbresuse==0.3.3 +nbconvert==5.6.1 +# Install seaborn via pip, it will update pandas along the way +seaborn==0.10.0 +ray==0.8.1 +dsassign==0.0.8 +altair==4.0.1 +xlrd==1.2.0 +git+https://github.com/ryanlovett/jupyter-tree-download.git +otter-grader==1.0.0.b6 +notetaker==0.0.3 # for data 100 hub study From 33929d3053f91c6f334a7388bcd22b711eeba864 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 3 Aug 2020 18:09:17 +0530 Subject: [PATCH 055/184] data100: Copy requirements.txt into image too --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 3127a54..29fa4d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,6 +77,7 @@ RUN conda config --add channels conda-forge # Encapsulate the environment info into its own yml file (which carries # the name `data100` in it COPY environment.yml /tmp/ +COPY requirements.txt /tmp/ RUN conda env create -f /tmp/environment.yml # We modify the path directly since the `source activate data100` From e8e1da8fd763108fe3c519833ed4d414e0faf73f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 7 Aug 2020 14:20:33 +0530 Subject: [PATCH 056/184] Install 'infrastructure' packages from a common file We want to provide a consistent set of notebook UI related packages across all our hubs. When we upgrade something (like nbformat, for https://github.com/berkeley-dsep-infra/datahub/pull/1641), we want that to upgrade these packages everywhere. Currently, these were spread out in individual hubs, and were not consistent at all. This commit brings them up from a single source, sync'd with a script. This lets us upgrade them all together. Also lets us make sure a set of UX things are available consistently across all hubs --- environment.yml | 6 ------ infra-requirements.txt | 19 +++++++++++++++++++ requirements.txt | 6 ++---- 3 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 infra-requirements.txt diff --git a/environment.yml b/environment.yml index 7c347b5..8d900a5 100644 --- a/environment.yml +++ b/environment.yml @@ -14,16 +14,10 @@ dependencies: - ipython_genutils=0.2.0 - ipywidgets=7.5.1 - jsonschema=3.2.0 -- jupyter=1.0.0 -- jupyter_client=5.3.4 -- jupyter_console=6.0.0 -- jupyter_core=4.6.1 - lxml=4.5.0 - matplotlib=3.1.2 -- nbformat=5.0.4 - networkx=2.4 - nodejs=13.7.0 -- notebook=6.0.3 - numpy=1.18.1 - oauthlib=3.0.1 - pandas=0.25.3 diff --git a/infra-requirements.txt b/infra-requirements.txt new file mode 100644 index 0000000..1038e40 --- /dev/null +++ b/infra-requirements.txt @@ -0,0 +1,19 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +# FIXME: Automate finding out when there are new versions of these +notebook==6.1.1 +nbconvert==5.6.1 +nbformat==5.0.7 +jupyterlab==2.2.2 +nbgitpuller==0.9.0 +nbresuse==0.3.6 +jupyterhub==1.1.0 +appmode==0.8.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3bb64b3..ff24503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,19 @@ +-r infra-requirements.txt + scipy==1.4.1 scikit-learn==0.22.1 scikit-image==0.16.2 plotly==4.5.0 colorlover==0.3.0 cufflinks==0.17.0 -jupyterlab==2.1.3 tweepy==3.8.0 nbgitpuller==0.8.0 jupyter-contrib-nbextensions==0.5.1 psycopg2==2.8.4 -jupyterhub==1.1.0 jupyter-server-proxy==1.2.0 jassign==0.0.7 okpy==1.13.11 tqdm==4.42.0 -nbresuse==0.3.3 -nbconvert==5.6.1 # Install seaborn via pip, it will update pandas along the way seaborn==0.10.0 ray==0.8.1 From eee7775e0b1794fb8457dae12f629a1b704175ee Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 7 Aug 2020 14:30:59 +0530 Subject: [PATCH 057/184] Copy infra-requirements.txt files explicitly into Docker builds --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 29fa4d1..bd221fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ RUN conda config --add channels conda-forge # the name `data100` in it COPY environment.yml /tmp/ COPY requirements.txt /tmp/ +COPY infra-requirements.txt /tmp/ RUN conda env create -f /tmp/environment.yml # We modify the path directly since the `source activate data100` From f0b8927bda9c83f0025c47a451f3912fc67240d9 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 7 Aug 2020 14:48:53 +0530 Subject: [PATCH 058/184] Install requests with requirements.txt Must install *everything* with requirements.txt soon --- environment.yml | 2 -- requirements.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 8d900a5..2e66be3 100644 --- a/environment.yml +++ b/environment.yml @@ -31,8 +31,6 @@ dependencies: - python-dateutil=2.8.1 - pytz=2019.3 - pyzmq=18.1.1 -- requests=2.22.0 -- requests-oauthlib=1.2.0 - setuptools=45.1.0 - sqlalchemy=1.3.13 - sqlite=3.30.1 diff --git a/requirements.txt b/requirements.txt index ff24503..c3055d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,13 +7,13 @@ plotly==4.5.0 colorlover==0.3.0 cufflinks==0.17.0 tweepy==3.8.0 -nbgitpuller==0.8.0 jupyter-contrib-nbextensions==0.5.1 psycopg2==2.8.4 jupyter-server-proxy==1.2.0 jassign==0.0.7 okpy==1.13.11 tqdm==4.42.0 +requests-oauthlib=1.2.0 # Install seaborn via pip, it will update pandas along the way seaborn==0.10.0 ray==0.8.1 From 1a57a6eec9aa40e76787e9996989195d9d301028 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 7 Aug 2020 15:23:32 +0530 Subject: [PATCH 059/184] Install nbpdfexport everywhere Should move to notebook-to-pdf, but one step at a time --- infra-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 1038e40..6126216 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -16,4 +16,5 @@ jupyterlab==2.2.2 nbgitpuller==0.9.0 nbresuse==0.3.6 jupyterhub==1.1.0 -appmode==0.8.0 \ No newline at end of file +appmode==0.8.0 +nbpdfexport==0.2.1 From ba6a4e602b4e74e03dc5e08fdfbd7870a3dc3a08 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 11 Aug 2020 11:27:20 +0530 Subject: [PATCH 060/184] data100: Fix typo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c3055d7..346e250 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ jupyter-server-proxy==1.2.0 jassign==0.0.7 okpy==1.13.11 tqdm==4.42.0 -requests-oauthlib=1.2.0 +requests-oauthlib==1.2.0 # Install seaborn via pip, it will update pandas along the way seaborn==0.10.0 ray==0.8.1 From ec408e02d73f55cd75df379c7ff13b0303ae93a9 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 13 Aug 2020 16:46:42 +0530 Subject: [PATCH 061/184] template: Fix typo that made ipython_config.py useless grr aargh. Fixes it in all images too. --- ipython_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython_config.py b/ipython_config.py index 1c229b5..ab5c5b8 100644 --- a/ipython_config.py +++ b/ipython_config.py @@ -1,3 +1,3 @@ # Disable history manager, we don't really use it # and by default it puts an sqlite file on NFS, which is not something we wanna do -c.Historymanager.enabled = False +c.HistoryManager.enabled = False From 846654646800e89e4de95fd48c805dd6ddac2863 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 17 Aug 2020 13:19:23 +0530 Subject: [PATCH 062/184] Remove all tornado pins Not an issue anymore, I think --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2e66be3..a45b1a2 100644 --- a/environment.yml +++ b/environment.yml @@ -37,7 +37,6 @@ dependencies: - statsmodels=0.11.0 - sympy=1.5.1 - tensorflow=2.0.0 -- tornado=6.0.3 - traitlets=4.3.3 - widgetsnbextension=3.5.1 - xarray=0.14.1 From ac09c7202899fb9be5f3a00a9a6eb5102b0eec12 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 17 Aug 2020 13:35:04 +0530 Subject: [PATCH 063/184] Upgrade notebook & lab infra versions --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6126216..67e9e2f 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,10 +9,10 @@ # FIXME: Freeze this to get exact versions of all dependencies # FIXME: Automate finding out when there are new versions of these -notebook==6.1.1 +notebook==6.1.3 nbconvert==5.6.1 nbformat==5.0.7 -jupyterlab==2.2.2 +jupyterlab==2.2.4 nbgitpuller==0.9.0 nbresuse==0.3.6 jupyterhub==1.1.0 From a7dbbe2da8788dd1d11e5075a73c9f9114d0d1b8 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 17 Aug 2020 14:06:28 +0530 Subject: [PATCH 064/184] Standardize ipywidgets versions across hubs It's definitely part of 'infrastructure' --- environment.yml | 1 - infra-requirements.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index a45b1a2..f87c4b1 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,6 @@ dependencies: - ipykernel=5.1.4 - ipython=7.9.0 - ipython_genutils=0.2.0 -- ipywidgets=7.5.1 - jsonschema=3.2.0 - lxml=4.5.0 - matplotlib=3.1.2 diff --git a/infra-requirements.txt b/infra-requirements.txt index 67e9e2f..74ca0d9 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -18,3 +18,4 @@ nbresuse==0.3.6 jupyterhub==1.1.0 appmode==0.8.0 nbpdfexport==0.2.1 +ipywidgets==7.5.1 \ No newline at end of file From 934c6303d09a4efadfe644883ee099926ad9debe Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 18 Aug 2020 12:01:54 +0530 Subject: [PATCH 065/184] Replace nbpdfexport with notebook-as-pdf Is actually maintained! --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 74ca0d9..306086e 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -17,5 +17,5 @@ nbgitpuller==0.9.0 nbresuse==0.3.6 jupyterhub==1.1.0 appmode==0.8.0 -nbpdfexport==0.2.1 -ipywidgets==7.5.1 \ No newline at end of file +ipywidgets==7.5.1 +notebook-as-pdf==0.3.1 \ No newline at end of file From 3aecb9e25d3121712de1c8de87c52e3fc3b7c779 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 19 Aug 2020 11:20:04 +0530 Subject: [PATCH 066/184] data100: Bump to python 3.8 Let's try this out --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f87c4b1..0ea0345 100644 --- a/environment.yml +++ b/environment.yml @@ -26,7 +26,7 @@ dependencies: - protobuf=3.11.2 - pygments=2.5.2 - pytest=5.3.5 # needed for jupyter-tree-download -- python=3.7.6 +- python=3.8.* - python-dateutil=2.8.1 - pytz=2019.3 - pyzmq=18.1.1 From 45a57fdc307d846166461eee9fcf9ca73c5c8236 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 20 Aug 2020 11:47:20 +0530 Subject: [PATCH 067/184] data100: Use miniforge, not miniconda To be consistent with other hubs --- Dockerfile | 40 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd221fd..fd13fc5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,13 @@ ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 ENV DEBIAN_FRONTEND=noninteractive +ENV NB_USER jovyan +ENV NB_UID 1000 -RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan +ENV CONDA_DIR /opt/conda +ENV PATH ${CONDA_DIR}/bin:$PATH + +RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER} # Other packages for user convenience and Data100 usage # Install these without 'recommended' packages to keep image smaller. @@ -49,41 +54,22 @@ RUN apt-get update -qq --yes && \ wkhtmltopdf # for pdf export \ > /dev/null - -ENV CONDA_PREFIX /srv/conda -ENV PATH ${CONDA_PREFIX}/bin:$PATH -RUN install -d -o jovyan -g jovyan ${CONDA_PREFIX} - WORKDIR /home/jovyan # prevent bibtex from interupting nbconvert RUN update-alternatives --install /usr/bin/bibtex bibtex /bin/true 200 -USER jovyan - -#################################################################### -# Download, install and configure the Conda environment +COPY install-miniforge.bash /tmp/install-miniforge.bash +RUN /tmp/install-miniforge.bash -RUN curl -o /tmp/miniconda.sh \ - https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh +USER ${NB_USER} -# Install miniconda -RUN bash /tmp/miniconda.sh -b -u -p ${CONDA_PREFIX} -RUN conda config --set always_yes yes --set changeps1 no -RUN conda update -q conda -RUN conda config --add channels conda-forge - -# Encapsulate the environment info into its own yml file (which carries -# the name `data100` in it COPY environment.yml /tmp/ COPY requirements.txt /tmp/ COPY infra-requirements.txt /tmp/ -RUN conda env create -f /tmp/environment.yml -# We modify the path directly since the `source activate data100` -# environment won't be preserved here. -ENV PATH ${CONDA_PREFIX}/envs/data100/bin:$PATH +RUN conda env update -p ${CONDA_DIR} -f /tmp/environment.yml # Set bash as shell in terminado. ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ @@ -96,9 +82,3 @@ RUN jupyter serverextension enable --sys-prefix --py jupyterlab RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@2.0.0 \ jupyterlab-plotly@4.8.1 \ plotlywidget@4.8.1 - -# Useful for debugging any issues with conda -RUN conda info -a - -# Make JupyterHub ports visible -EXPOSE 8888 From 657d624ea877b5ef723def071cb16b9ea55e600b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 20 Aug 2020 11:47:52 +0530 Subject: [PATCH 068/184] data100: Remove wkhtmltopdf pyppetteer is available instead now, and doesn't bring in basically all of linux desktop --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fd13fc5..3dcfffa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,6 @@ RUN apt-get update -qq --yes && \ texlive-xetex \ texlive-fonts-recommended \ texlive-generic-recommended \ - wkhtmltopdf # for pdf export \ > /dev/null WORKDIR /home/jovyan From d8db1605194258cb1b04e4e87af4e233db80a870 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 20 Aug 2020 11:48:15 +0530 Subject: [PATCH 069/184] data100: Don't explicitly enable jupyterlab serverextension Hasn't been needed for a while --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3dcfffa..35341eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,9 +75,6 @@ ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ # Disable history. ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ -# Installed in conda environment -RUN jupyter serverextension enable --sys-prefix --py jupyterlab - RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@2.0.0 \ jupyterlab-plotly@4.8.1 \ plotlywidget@4.8.1 From 894c3bdc87fdddba48cca9898763a14cc5dd6b36 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 20 Aug 2020 11:48:49 +0530 Subject: [PATCH 070/184] data100: Add miniforge-installer script Was missing in earlier commit --- install-miniforge.bash | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 install-miniforge.bash diff --git a/install-miniforge.bash b/install-miniforge.bash new file mode 100755 index 0000000..7b2441a --- /dev/null +++ b/install-miniforge.bash @@ -0,0 +1,42 @@ +#!/bin/bash +# This downloads and installs a pinned version of miniconda +set -ex + +cd $(dirname $0) +MINIFORGE_VERSION=4.8.3-5 + +URL="https://github.com/conda-forge/miniforge/releases/download/4.8.3-5/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh" +INSTALLER_PATH=/tmp/miniforge-installer.sh + +# make sure we don't do anything funky with user's $HOME +# since this is run as root +unset HOME + +wget --quiet $URL -O ${INSTALLER_PATH} +chmod +x ${INSTALLER_PATH} + +bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} +export PATH="${CONDA_DIR}/bin:$PATH" + +# Do not attempt to auto update conda or dependencies +conda config --system --set auto_update_conda false +conda config --system --set show_channel_urls true + +# empty conda history file, +# which seems to result in some effective pinning of packages in the initial env, +# which we don't intend. +# this file must not be *removed*, however +echo '' > ${CONDA_DIR}/conda-meta/history + +# Clean things out! +conda clean --all -f -y + +# Remove the big installer so we don't increase docker image size too much +rm ${INSTALLER_PATH} + +# Remove the pip cache created as part of installing miniconda +rm -rf /root/.cache + +chown -R $NB_USER:$NB_USER ${CONDA_DIR} + +conda list -n root From 18e2c1bb5592adaa099fd2d3ece5d3faad8760a9 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 20 Aug 2020 11:49:05 +0530 Subject: [PATCH 071/184] data100: Upgrade & cleanup packages - Install most things via pip, not conda. Earlier there was a complex mix of two. Now almost everything comes from pip. - Update packages where necessary, but keep it minimal --- environment.yml | 39 +-------------------------------------- requirements.txt | 45 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 49 deletions(-) diff --git a/environment.yml b/environment.yml index 0ea0345..90f94ae 100644 --- a/environment.yml +++ b/environment.yml @@ -1,47 +1,10 @@ -name: data100 channels: -- defaults -- pytorch +- conda-forge dependencies: -- attrs=19.3.0 -- beautifulsoup4=4.8.2 -- bleach=3.1.0 -- coverage=5.0.3 -- decorator=4.4.1 -- html5lib=1.0.1 -- ipykernel=5.1.4 -- ipython=7.9.0 -- ipython_genutils=0.2.0 -- jsonschema=3.2.0 -- lxml=4.5.0 -- matplotlib=3.1.2 -- networkx=2.4 - nodejs=13.7.0 -- numpy=1.18.1 -- oauthlib=3.0.1 -- pandas=0.25.3 -- pandoc=2.9.1.1 -- pandocfilters=1.4.2 - pip=20.0.2 - protobuf=3.11.2 -- pygments=2.5.2 -- pytest=5.3.5 # needed for jupyter-tree-download - python=3.8.* -- python-dateutil=2.8.1 -- pytz=2019.3 -- pyzmq=18.1.1 -- setuptools=45.1.0 -- sqlalchemy=1.3.13 -- sqlite=3.30.1 -- statsmodels=0.11.0 -- sympy=1.5.1 -- tensorflow=2.0.0 -- traitlets=4.3.3 -- widgetsnbextension=3.5.1 -- xarray=0.14.1 -- xz=5.2.4 -- zeromq=4.3.2 - graphviz=2.42.3 -- pytorch=1.4.0 - pip: - -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 346e250..fb3ded4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,25 +1,48 @@ -r infra-requirements.txt +# base scientific packages +numpy==1.18.1 scipy==1.4.1 scikit-learn==0.22.1 scikit-image==0.16.2 +networkx==2.4 +xarray==0.14.1 +torch==1.4.0 +pandas==0.25.3 +ray==0.8.7 +tensorflow==2.2.0 +sympy==1.5.1 +statsmodels==0.11.0 +pandas==0.25.3 +xlrd==1.2.0 + +# visualization +matplotlib==3.1.2 plotly==4.5.0 colorlover==0.3.0 cufflinks==0.17.0 +altair==4.0.1 +seaborn==0.10.0 + +# web access +beautifulsoup4==4.8.2 +lxml==4.5.0 tweepy==3.8.0 -jupyter-contrib-nbextensions==0.5.1 + +# utils +tqdm==4.42.0 +pytz==2019.3 + psycopg2==2.8.4 -jupyter-server-proxy==1.2.0 + +# teaching jassign==0.0.7 okpy==1.13.11 -tqdm==4.42.0 -requests-oauthlib==1.2.0 -# Install seaborn via pip, it will update pandas along the way -seaborn==0.10.0 -ray==0.8.1 dsassign==0.0.8 -altair==4.0.1 -xlrd==1.2.0 -git+https://github.com/ryanlovett/jupyter-tree-download.git -otter-grader==1.0.0.b9 notetaker==0.0.3 # for data 100 hub study +otter-grader==1.0.0.b9 + +# notebook stuff +jupyter-contrib-nbextensions==0.5.1 +jupyter-server-proxy==1.2.0 +git+https://github.com/ryanlovett/jupyter-tree-download.git From 0048d980f2e1f179d5a104217f811e0751efca31 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Sun, 23 Aug 2020 09:28:09 -0700 Subject: [PATCH 072/184] Bump otter to stable version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fb3ded4..ad47a30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 notetaker==0.0.3 # for data 100 hub study -otter-grader==1.0.0.b9 +otter-grader==1.0.0 # notebook stuff jupyter-contrib-nbextensions==0.5.1 From ad3761b80b36a7b3a84439d004d24beb6aab06d3 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Wed, 26 Aug 2020 16:36:22 -0700 Subject: [PATCH 073/184] Bump pandas to 1.1.1 --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb3ded4..9ce0d71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,12 +8,11 @@ scikit-image==0.16.2 networkx==2.4 xarray==0.14.1 torch==1.4.0 -pandas==0.25.3 ray==0.8.7 tensorflow==2.2.0 sympy==1.5.1 statsmodels==0.11.0 -pandas==0.25.3 +pandas==1.1.1 xlrd==1.2.0 # visualization From c6d8c5a726a933d8993f81d76883fed469c9355e Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Fri, 28 Aug 2020 19:40:01 -0700 Subject: [PATCH 074/184] Bump otter --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0ef205..ca4125e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 notetaker==0.0.3 # for data 100 hub study -otter-grader==1.0.0 +otter-grader==1.0.1 # notebook stuff jupyter-contrib-nbextensions==0.5.1 From 5eeb318a8dec57e9f80ae1e3ff867fe0017ef276 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 7 Sep 2020 19:58:46 -0700 Subject: [PATCH 075/184] Bump otter --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ca4125e..50c4cbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 notetaker==0.0.3 # for data 100 hub study -otter-grader==1.0.1 +otter-grader==1.1.0 # notebook stuff jupyter-contrib-nbextensions==0.5.1 From 8f664110e63de97b3d1fb508efface5cd7f5484f Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 8 Sep 2020 11:54:35 -0700 Subject: [PATCH 076/184] Bump otter (again) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50c4cbb..abb8ad6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 notetaker==0.0.3 # for data 100 hub study -otter-grader==1.1.0 +otter-grader==1.1.1 # notebook stuff jupyter-contrib-nbextensions==0.5.1 From 080e3a2d603606e161468fd00092532390793693 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Wed, 23 Sep 2020 18:42:24 -0700 Subject: [PATCH 077/184] Bump otter --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index abb8ad6..c2900cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 notetaker==0.0.3 # for data 100 hub study -otter-grader==1.1.1 +otter-grader==1.1.2 # notebook stuff jupyter-contrib-nbextensions==0.5.1 From e50fb89856b928b639297cbe50305a2af65cd1a9 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Wed, 23 Sep 2020 18:43:08 -0700 Subject: [PATCH 078/184] Bump seaborn --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2900cc..4b055d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ plotly==4.5.0 colorlover==0.3.0 cufflinks==0.17.0 altair==4.0.1 -seaborn==0.10.0 +seaborn==0.11.0 # web access beautifulsoup4==4.8.2 From d6f05f8782c85f6460d6b0e5a1545dc522694e60 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 1 Oct 2020 15:34:45 +0530 Subject: [PATCH 079/184] Add packages for IB-105 Add otter-grader to *all* hubs as an infrastructure package. Everyone seems to want to use it :) Fixes #1846 --- infra-requirements.txt | 3 ++- requirements.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 306086e..6d4f9aa 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -18,4 +18,5 @@ nbresuse==0.3.6 jupyterhub==1.1.0 appmode==0.8.0 ipywidgets==7.5.1 -notebook-as-pdf==0.3.1 \ No newline at end of file +notebook-as-pdf==0.3.1 +otter-grader==1.1.3 diff --git a/requirements.txt b/requirements.txt index 4b055d1..cea1f1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,6 @@ jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 notetaker==0.0.3 # for data 100 hub study -otter-grader==1.1.2 # notebook stuff jupyter-contrib-nbextensions==0.5.1 From 1411b5ba1a2c6ce090adde4fa2ef8813396c1b3f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 9 Oct 2020 11:20:08 +0530 Subject: [PATCH 080/184] Add jupyter-tree-download to all hubs Currently some have nbzip, which is deprecated. --- infra-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6d4f9aa..5858416 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,3 +20,4 @@ appmode==0.8.0 ipywidgets==7.5.1 notebook-as-pdf==0.3.1 otter-grader==1.1.3 +jupyter-tree-download==1.0.1 From 44c3358c2f786214d57b4f002fc92dcbbdc4981a Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 17 Oct 2020 18:43:39 +0530 Subject: [PATCH 081/184] Set SHELL to /bin/bash for all hubs Was inconsistently set via jupyter_notebook_config.py in some hubs, and only specific to Jupyter Notebooks. This sets it more generally, and does it for all hubs. Fixes #1890 --- jupyter_notebook_config.py | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 jupyter_notebook_config.py diff --git a/jupyter_notebook_config.py b/jupyter_notebook_config.py deleted file mode 100644 index f140340..0000000 --- a/jupyter_notebook_config.py +++ /dev/null @@ -1,12 +0,0 @@ -# This is sh (dash) by default, not $SHELL -c.NotebookApp.terminado_settings = { "shell_command": ["bash"] } - -c.ServerProxy.servers = { - 'http-server': { - 'command': ['python3', '-m', 'http.server', '{port}'], - 'absolute_url': False, - 'launcher_entry': { - 'title': "HTTP Server" - } - } -} From e256cb07c1112f08770ee0c26231149d8d08274a Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 17 Oct 2020 20:29:39 +0530 Subject: [PATCH 082/184] Don't copy non-existent jupyter_notebook_config.py --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 35341eb..e278040 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,6 @@ COPY infra-requirements.txt /tmp/ RUN conda env update -p ${CONDA_DIR} -f /tmp/environment.yml # Set bash as shell in terminado. -ADD jupyter_notebook_config.py ${CONDA_PREFIX}/envs/data100/etc/jupyter/ # Disable history. ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ From bf9e1fea524167dfa632c14d933ef897f1d5d089 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Thu, 12 Nov 2020 11:52:56 -0800 Subject: [PATCH 083/184] Updated notetaker --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cea1f1b..6afcab6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ psycopg2==2.8.4 jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 -notetaker==0.0.3 # for data 100 hub study +notetaker==0.0.5 # for data 100 hub study # notebook stuff jupyter-contrib-nbextensions==0.5.1 From 206120dd8467dc7bf42ec7439ce92f2144830261 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 17 Nov 2020 13:20:25 +0530 Subject: [PATCH 084/184] Upgrade nbresuse to not shadow /metrics Brings in https://github.com/yuvipanda/nbresuse/pull/68 so we can move towards getting prometheus metrics from notebooks. --- infra-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 5858416..f782460 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -14,7 +14,8 @@ nbconvert==5.6.1 nbformat==5.0.7 jupyterlab==2.2.4 nbgitpuller==0.9.0 -nbresuse==0.3.6 +# Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged +git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 jupyterhub==1.1.0 appmode==0.8.0 ipywidgets==7.5.1 From a64aa9a6eef214f2f4af8d8d77e74fdc22f9447f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 18 Nov 2020 01:53:41 +0530 Subject: [PATCH 085/184] Bump notebook package version Brings in https://github.com/jupyter/notebook/pull/5870, working towards getting notebook request metrics into prometheus. --- infra-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index f782460..22a7eee 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,8 @@ # FIXME: Freeze this to get exact versions of all dependencies # FIXME: Automate finding out when there are new versions of these -notebook==6.1.3 +# Temporary until https://github.com/jupyter/notebook/pull/5870 is merged +git+https://github.com/yuvipanda/notebook@c0ab18a53472372e31dee293fefd66f218107c98 nbconvert==5.6.1 nbformat==5.0.7 jupyterlab==2.2.4 From 70613cd699e806089c31186685b7dea4d789e617 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 18 Nov 2020 12:43:40 +0530 Subject: [PATCH 086/184] Install 'notebook' package from jupyter repo My PR got merged! --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 22a7eee..be1e90b 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,8 +9,8 @@ # FIXME: Freeze this to get exact versions of all dependencies # FIXME: Automate finding out when there are new versions of these -# Temporary until https://github.com/jupyter/notebook/pull/5870 is merged -git+https://github.com/yuvipanda/notebook@c0ab18a53472372e31dee293fefd66f218107c98 +# Temporary until https://github.com/jupyter/notebook/pull/5870 is released +git+https://github.com/jupyter/notebook@5a73a8e4026ccc8d7437f788bf0bc2758a73567d nbconvert==5.6.1 nbformat==5.0.7 jupyterlab==2.2.4 From 3d96b71f9f904fab5a3cfc423dc70099fb0d1bc2 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Sun, 22 Nov 2020 18:56:15 -0800 Subject: [PATCH 087/184] Update data 100 to otter beta --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 5858416..8647eff 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -19,5 +19,5 @@ jupyterhub==1.1.0 appmode==0.8.0 ipywidgets==7.5.1 notebook-as-pdf==0.3.1 -otter-grader==1.1.3 +otter-grader==2.0.0.b0 jupyter-tree-download==1.0.1 From 9749c5ff014adbcdf60237e0065ac9af35922490 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sun, 22 Nov 2020 18:57:14 -0800 Subject: [PATCH 088/184] Revert "Update data 100 to otter beta" This reverts commit 3d96b71f9f904fab5a3cfc423dc70099fb0d1bc2. --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 8647eff..5858416 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -19,5 +19,5 @@ jupyterhub==1.1.0 appmode==0.8.0 ipywidgets==7.5.1 notebook-as-pdf==0.3.1 -otter-grader==2.0.0.b0 +otter-grader==1.1.3 jupyter-tree-download==1.0.1 From b96d95c83f86bc27ca7b44910c4a75cddf4d026b Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sun, 22 Nov 2020 19:00:57 -0800 Subject: [PATCH 089/184] upgrade to otter v2.0.0.b0 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 5858416..8647eff 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -19,5 +19,5 @@ jupyterhub==1.1.0 appmode==0.8.0 ipywidgets==7.5.1 notebook-as-pdf==0.3.1 -otter-grader==1.1.3 +otter-grader==2.0.0.b0 jupyter-tree-download==1.0.1 From c2a752be21190cc66572a42536b03d02b999c8a1 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 23 Nov 2020 18:04:48 -0800 Subject: [PATCH 090/184] Bump notetaker --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6afcab6..58227a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ psycopg2==2.8.4 jassign==0.0.7 okpy==1.13.11 dsassign==0.0.8 -notetaker==0.0.5 # for data 100 hub study +notetaker==0.0.6 # for data 100 hub study # notebook stuff jupyter-contrib-nbextensions==0.5.1 From 562995a7baf3db13242a40e8a98a7a64efb6450c Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Mon, 23 Nov 2020 19:09:09 -0800 Subject: [PATCH 091/184] Revert "upgrade to otter v2.0.0.b0" This reverts commit b96d95c83f86bc27ca7b44910c4a75cddf4d026b. --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 0345976..be1e90b 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -21,5 +21,5 @@ jupyterhub==1.1.0 appmode==0.8.0 ipywidgets==7.5.1 notebook-as-pdf==0.3.1 -otter-grader==2.0.0.b0 +otter-grader==1.1.3 jupyter-tree-download==1.0.1 From 0f89271706778ee894b3edcc189a04da27163459 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Sun, 29 Nov 2020 10:28:48 -0800 Subject: [PATCH 092/184] Add nltk to data 100 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 58227a0..524cea9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ sympy==1.5.1 statsmodels==0.11.0 pandas==1.1.1 xlrd==1.2.0 +nltk==3.5 # visualization matplotlib==3.1.2 From d750e01291bb02b36a2a758aca065a5d1629d7d6 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 22 Dec 2020 18:00:32 +0530 Subject: [PATCH 093/184] Add git-credential-helpers to all images Part of trying to get private GitHub repositories to work on datahub. --- infra-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index be1e90b..abd9ce5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -23,3 +23,4 @@ ipywidgets==7.5.1 notebook-as-pdf==0.3.1 otter-grader==1.1.3 jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 From 5a7ba2b67bcbacf5261dfc25ddce972399bc31c6 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 4 Jan 2021 14:04:53 +0530 Subject: [PATCH 094/184] Pin pip to 20.2.* 20.3.* has a strict resolver, and allensdk does *not* like that. Let's not try to solve this problem now too. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 90f94ae..f1d2021 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ channels: - conda-forge dependencies: - nodejs=13.7.0 -- pip=20.0.2 +- pip=20.2.* - protobuf=3.11.2 - python=3.8.* - graphviz=2.42.3 From ee512834842186f4ced8aa08cbbcc7c5305462e4 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 4 Jan 2021 21:18:47 +0530 Subject: [PATCH 095/184] Move data100 base image to focal Simpler to move. --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e278040..86ede47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps:bionic-scm +FROM buildpack-deps:focal-scm # Set up common env variables ENV TZ=America/Los_Angeles @@ -50,14 +50,11 @@ RUN apt-get update -qq --yes && \ pandoc \ texlive-xetex \ texlive-fonts-recommended \ - texlive-generic-recommended \ + texlive-plain-generic \ > /dev/null WORKDIR /home/jovyan -# prevent bibtex from interupting nbconvert -RUN update-alternatives --install /usr/bin/bibtex bibtex /bin/true 200 - COPY install-miniforge.bash /tmp/install-miniforge.bash RUN /tmp/install-miniforge.bash From d303b4fe1beb7e2384b0ffbf27e065040799169c Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 8 Jan 2021 12:32:15 +0530 Subject: [PATCH 096/184] Upgrade infra-packages - Install packages from PyPI instead of git, as they have been released - Upgrade packages where we can. No jupyterlab 3.x yet, since a few lab extensions haven't been upgraded yet. --- infra-requirements.txt | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index abd9ce5..c1d4e03 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,18 +8,13 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -# FIXME: Automate finding out when there are new versions of these -# Temporary until https://github.com/jupyter/notebook/pull/5870 is released -git+https://github.com/jupyter/notebook@5a73a8e4026ccc8d7437f788bf0bc2758a73567d -nbconvert==5.6.1 -nbformat==5.0.7 -jupyterlab==2.2.4 +notebook==6.1.6 +jupyterlab==2.2.9 nbgitpuller==0.9.0 -# Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged -git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 -jupyterhub==1.1.0 +jupyter-resource-usage==0.5.1 +jupyterhub==1.3.0 appmode==0.8.0 -ipywidgets==7.5.1 +ipywidgets==7.6.3 notebook-as-pdf==0.3.1 otter-grader==1.1.3 jupyter-tree-download==1.0.1 From 19027ba5313bc6a9d33dc0fbfe54c61f57f100c5 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 9 Jan 2021 15:32:51 +0530 Subject: [PATCH 097/184] Revert "Upgrade infra-packages" Ref #2037 ``` Traceback (most recent call last): File "/opt/conda/lib/python3.8/site-packages/tornado/web.py", line 1704, in _execute result = await result File "/opt/conda/lib/python3.8/site-packages/tornado/gen.py", line 769, in run yielded = self.gen.throw(*exc_info) # type: ignore File "/opt/conda/lib/python3.8/site-packages/notebook/services/sessions/handlers.py", line 69, in post model = yield maybe_future( File "/opt/conda/lib/python3.8/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/opt/conda/lib/python3.8/site-packages/tornado/gen.py", line 769, in run yielded = self.gen.throw(*exc_info) # type: ignore File "/opt/conda/lib/python3.8/site-packages/notebook/services/sessions/sessionmanager.py", line 88, in create_session kernel_id = yield self.start_kernel_for_session(session_id, path, name, type, kernel_name) File "/opt/conda/lib/python3.8/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/opt/conda/lib/python3.8/site-packages/tornado/gen.py", line 769, in run yielded = self.gen.throw(*exc_info) # type: ignore File "/opt/conda/lib/python3.8/site-packages/notebook/services/sessions/sessionmanager.py", line 100, in start_kernel_for_session kernel_id = yield maybe_future( File "/opt/conda/lib/python3.8/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/opt/conda/lib/python3.8/site-packages/notebook/services/kernels/kernelmanager.py", line 176, in start_kernel kernel_id = await maybe_future(self.pinned_superclass.start_kernel(self, **kwargs)) File "/opt/conda/lib/python3.8/site-packages/jupyter_client/multikernelmanager.py", line 184, in start_kernel km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) File "/opt/conda/lib/python3.8/site-packages/jupyter_client/multikernelmanager.py", line 169, in pre_start_kernel km = self.kernel_manager_factory(connection_file=os.path.join( File "/opt/conda/lib/python3.8/site-packages/jupyter_client/multikernelmanager.py", line 83, in create_kernel_manager km = kernel_manager_ctor(*args, **kwargs) File "/opt/conda/lib/python3.8/site-packages/traitlets/config/configurable.py", line 82, in __init__ super(Configurable, self).__init__(**kwargs) File "/opt/conda/lib/python3.8/site-packages/traitlets/traitlets.py", line 1080, in __init__ super_kwargs[key] = value File "/opt/conda/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/opt/conda/lib/python3.8/site-packages/traitlets/traitlets.py", line 1214, in hold_trait_notifications self.notify_change(change) File "/opt/conda/lib/python3.8/site-packages/traitlets/traitlets.py", line 1227, in notify_change return self._notify_observers(change) File "/opt/conda/lib/python3.8/site-packages/traitlets/traitlets.py", line 1264, in _notify_observers c(event) File "/opt/conda/lib/python3.8/site-packages/traitlets/traitlets.py", line 888, in compatible_observer return func(self, change) TypeError: _kernel_spec_manager_changed() takes 1 positional argument but 2 were given ``` This reverts commit d303b4fe1beb7e2384b0ffbf27e065040799169c. --- infra-requirements.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index c1d4e03..abd9ce5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,13 +8,18 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.1.6 -jupyterlab==2.2.9 +# FIXME: Automate finding out when there are new versions of these +# Temporary until https://github.com/jupyter/notebook/pull/5870 is released +git+https://github.com/jupyter/notebook@5a73a8e4026ccc8d7437f788bf0bc2758a73567d +nbconvert==5.6.1 +nbformat==5.0.7 +jupyterlab==2.2.4 nbgitpuller==0.9.0 -jupyter-resource-usage==0.5.1 -jupyterhub==1.3.0 +# Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged +git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 +jupyterhub==1.1.0 appmode==0.8.0 -ipywidgets==7.6.3 +ipywidgets==7.5.1 notebook-as-pdf==0.3.1 otter-grader==1.1.3 jupyter-tree-download==1.0.1 From 9a07cf3c43a34b4dd790c0346ea7ebba545c0a6d Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 11 Jan 2021 17:19:41 +0530 Subject: [PATCH 098/184] Actually update jupyterhub in all images Forgot to run sync script earlier! --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index abd9ce5..e741a33 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -17,7 +17,7 @@ jupyterlab==2.2.4 nbgitpuller==0.9.0 # Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 -jupyterhub==1.1.0 +jupyterhub==1.3.0 appmode==0.8.0 ipywidgets==7.5.1 notebook-as-pdf==0.3.1 From 68159d49ece3326e582ef422c0b25d1df33e8d96 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 13 Jan 2021 12:18:24 +0530 Subject: [PATCH 099/184] Install jupyterlab 3.0 - A bunch of extensions are now using the new python based extension system! We stop using npm to install these. - datahub: install the plotly jupyterlab extension, which doesn't seem to have been installed earlier. --- Dockerfile | 4 ---- infra-requirements.txt | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 86ede47..c871577 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,3 @@ RUN conda env update -p ${CONDA_DIR} -f /tmp/environment.yml # Set bash as shell in terminado. # Disable history. ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ - -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@2.0.0 \ - jupyterlab-plotly@4.8.1 \ - plotlywidget@4.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index e741a33..e0fa3c7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,13 +13,13 @@ git+https://github.com/jupyter/notebook@5a73a8e4026ccc8d7437f788bf0bc2758a73567d nbconvert==5.6.1 nbformat==5.0.7 -jupyterlab==2.2.4 +jupyterlab==3.0.4 nbgitpuller==0.9.0 # Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 jupyterhub==1.3.0 appmode==0.8.0 -ipywidgets==7.5.1 +ipywidgets==7.6.3 notebook-as-pdf==0.3.1 otter-grader==1.1.3 jupyter-tree-download==1.0.1 From 80380d811c13f9c57c2f6b2f75a160b78fde9613 Mon Sep 17 00:00:00 2001 From: Yuvi Panda Date: Wed, 13 Jan 2021 17:15:39 +0530 Subject: [PATCH 100/184] Revert "Install jupyterlab 3.0" --- Dockerfile | 4 ++++ infra-requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c871577..86ede47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,3 +70,7 @@ RUN conda env update -p ${CONDA_DIR} -f /tmp/environment.yml # Set bash as shell in terminado. # Disable history. ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ + +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@2.0.0 \ + jupyterlab-plotly@4.8.1 \ + plotlywidget@4.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index e0fa3c7..e741a33 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,13 +13,13 @@ git+https://github.com/jupyter/notebook@5a73a8e4026ccc8d7437f788bf0bc2758a73567d nbconvert==5.6.1 nbformat==5.0.7 -jupyterlab==3.0.4 +jupyterlab==2.2.4 nbgitpuller==0.9.0 # Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 jupyterhub==1.3.0 appmode==0.8.0 -ipywidgets==7.6.3 +ipywidgets==7.5.1 notebook-as-pdf==0.3.1 otter-grader==1.1.3 jupyter-tree-download==1.0.1 From 5bfd6c2aca27083e2523bb3ad3ba502e5d1c7a8b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 15 Jan 2021 13:05:56 +0530 Subject: [PATCH 101/184] Install jupyter-contrib-nbextension everywhere Everyone likes it --- infra-requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index e741a33..38f3b79 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -15,8 +15,7 @@ nbconvert==5.6.1 nbformat==5.0.7 jupyterlab==2.2.4 nbgitpuller==0.9.0 -# Temporary fix until https://github.com/yuvipanda/nbresuse/pull/68/commits/87c306f7a604e866e83c6b1f6d4cc8b4015143c6 is merged -git+https://github.com/yuvipanda/nbresuse.git@87c306f7a604e866e83c6b1f6d4cc8b4015143c6 +jupyter-resource-usage==0.5.1 jupyterhub==1.3.0 appmode==0.8.0 ipywidgets==7.5.1 @@ -24,3 +23,5 @@ notebook-as-pdf==0.3.1 otter-grader==1.1.3 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 +# Enough people like this, let's load it in. +jupyter-contrib-nbextensions==0.5.1 From 90081f056400a44975c48d5a151389a6789f8989 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 15 Jan 2021 13:49:45 +0530 Subject: [PATCH 102/184] Add data100 packages to datahub image Let's move data100 to the datahub image. data100 hub is primarily to limit admin access. Using the same image reduces our maintenance burden. We couldn't do this early on because the datahub image did not use conda. It does now. Installs the jupyter_nbextensions_configurator on all hubs as well, since it was in the data100 hub. Seems generally useful. --- Dockerfile | 2 +- infra-requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 86ede47..4594a66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,11 +28,11 @@ RUN apt-get update -qq --yes && \ git \ htop \ less \ - libpq-dev \ man \ mc \ nano \ openssh-client \ + libpq-dev \ postgresql-client \ screen \ tar \ diff --git a/infra-requirements.txt b/infra-requirements.txt index 38f3b79..81b3468 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -25,3 +25,4 @@ jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. jupyter-contrib-nbextensions==0.5.1 +jupyter_nbextensions_configurator==0.4.1 From 7166bd9c7b263eecd75fc83c55cf915ccfeed35a Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 15 Jan 2021 19:03:53 +0530 Subject: [PATCH 103/184] Remove data100 image! Follow-up to https://github.com/berkeley-dsep-infra/datahub/pull/2072 --- Dockerfile | 76 ------------------------------------- environment.yml | 10 ----- infra-requirements.txt | 28 -------------- install-miniforge.bash | 42 -------------------- ipython_config.py | 3 -- jupyter_nbconvert_config.py | 1 - requirements.txt | 47 ----------------------- 7 files changed, 207 deletions(-) delete mode 100644 Dockerfile delete mode 100644 environment.yml delete mode 100644 infra-requirements.txt delete mode 100755 install-miniforge.bash delete mode 100644 ipython_config.py delete mode 100644 jupyter_nbconvert_config.py delete mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4594a66..0000000 --- a/Dockerfile +++ /dev/null @@ -1,76 +0,0 @@ -FROM buildpack-deps:focal-scm - -# Set up common env variables -ENV TZ=America/Los_Angeles -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEBIAN_FRONTEND=noninteractive -ENV NB_USER jovyan -ENV NB_UID 1000 - -ENV CONDA_DIR /opt/conda -ENV PATH ${CONDA_DIR}/bin:$PATH - -RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER} - -# Other packages for user convenience and Data100 usage -# Install these without 'recommended' packages to keep image smaller. -RUN apt-get update -qq --yes && \ - apt-get install --yes --no-install-recommends -qq \ - build-essential \ - ca-certificates \ - curl \ - default-jdk \ - emacs-nox \ - git \ - htop \ - less \ - man \ - mc \ - nano \ - openssh-client \ - libpq-dev \ - postgresql-client \ - screen \ - tar \ - tmux \ - wget \ - vim \ - locales > /dev/null - -RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ - locale-gen - -RUN apt-get update -qq --yes && \ - apt-get install --yes -qq \ - # for nbconvert - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-plain-generic \ - > /dev/null - -WORKDIR /home/jovyan - -COPY install-miniforge.bash /tmp/install-miniforge.bash -RUN /tmp/install-miniforge.bash - -USER ${NB_USER} - - -COPY environment.yml /tmp/ -COPY requirements.txt /tmp/ -COPY infra-requirements.txt /tmp/ - -RUN conda env update -p ${CONDA_DIR} -f /tmp/environment.yml - -# Set bash as shell in terminado. -# Disable history. -ADD ipython_config.py ${CONDA_PREFIX}/envs/data100/etc/ipython/ - -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager@2.0.0 \ - jupyterlab-plotly@4.8.1 \ - plotlywidget@4.8.1 diff --git a/environment.yml b/environment.yml deleted file mode 100644 index f1d2021..0000000 --- a/environment.yml +++ /dev/null @@ -1,10 +0,0 @@ -channels: -- conda-forge -dependencies: -- nodejs=13.7.0 -- pip=20.2.* -- protobuf=3.11.2 -- python=3.8.* -- graphviz=2.42.3 -- pip: - - -r requirements.txt diff --git a/infra-requirements.txt b/infra-requirements.txt deleted file mode 100644 index 81b3468..0000000 --- a/infra-requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -# WARNING: Original source at scripts/infra-packages/requirements.txt -# PLEASE DO NOT EDIT ELSEWHERE -# After editing scripts/infra-packages/requirements.txt, please run -# scripts/infra-packages/sync.bash. - -# This file pins versions of notebook related python packages we want -# across all hubs. This makes sure we don't need to upgrade them -# everwhere one by one. - -# FIXME: Freeze this to get exact versions of all dependencies -# FIXME: Automate finding out when there are new versions of these -# Temporary until https://github.com/jupyter/notebook/pull/5870 is released -git+https://github.com/jupyter/notebook@5a73a8e4026ccc8d7437f788bf0bc2758a73567d -nbconvert==5.6.1 -nbformat==5.0.7 -jupyterlab==2.2.4 -nbgitpuller==0.9.0 -jupyter-resource-usage==0.5.1 -jupyterhub==1.3.0 -appmode==0.8.0 -ipywidgets==7.5.1 -notebook-as-pdf==0.3.1 -otter-grader==1.1.3 -jupyter-tree-download==1.0.1 -git-credential-helpers==0.2 -# Enough people like this, let's load it in. -jupyter-contrib-nbextensions==0.5.1 -jupyter_nbextensions_configurator==0.4.1 diff --git a/install-miniforge.bash b/install-miniforge.bash deleted file mode 100755 index 7b2441a..0000000 --- a/install-miniforge.bash +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# This downloads and installs a pinned version of miniconda -set -ex - -cd $(dirname $0) -MINIFORGE_VERSION=4.8.3-5 - -URL="https://github.com/conda-forge/miniforge/releases/download/4.8.3-5/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/miniforge-installer.sh - -# make sure we don't do anything funky with user's $HOME -# since this is run as root -unset HOME - -wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} - -bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" - -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history - -# Clean things out! -conda clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} - -# Remove the pip cache created as part of installing miniconda -rm -rf /root/.cache - -chown -R $NB_USER:$NB_USER ${CONDA_DIR} - -conda list -n root diff --git a/ipython_config.py b/ipython_config.py deleted file mode 100644 index ab5c5b8..0000000 --- a/ipython_config.py +++ /dev/null @@ -1,3 +0,0 @@ -# Disable history manager, we don't really use it -# and by default it puts an sqlite file on NFS, which is not something we wanna do -c.HistoryManager.enabled = False diff --git a/jupyter_nbconvert_config.py b/jupyter_nbconvert_config.py deleted file mode 100644 index 79afd57..0000000 --- a/jupyter_nbconvert_config.py +++ /dev/null @@ -1 +0,0 @@ -c.PDFExporter.bib_command = ['true'] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 524cea9..0000000 --- a/requirements.txt +++ /dev/null @@ -1,47 +0,0 @@ --r infra-requirements.txt - -# base scientific packages -numpy==1.18.1 -scipy==1.4.1 -scikit-learn==0.22.1 -scikit-image==0.16.2 -networkx==2.4 -xarray==0.14.1 -torch==1.4.0 -ray==0.8.7 -tensorflow==2.2.0 -sympy==1.5.1 -statsmodels==0.11.0 -pandas==1.1.1 -xlrd==1.2.0 -nltk==3.5 - -# visualization -matplotlib==3.1.2 -plotly==4.5.0 -colorlover==0.3.0 -cufflinks==0.17.0 -altair==4.0.1 -seaborn==0.11.0 - -# web access -beautifulsoup4==4.8.2 -lxml==4.5.0 -tweepy==3.8.0 - -# utils -tqdm==4.42.0 -pytz==2019.3 - -psycopg2==2.8.4 - -# teaching -jassign==0.0.7 -okpy==1.13.11 -dsassign==0.0.8 -notetaker==0.0.6 # for data 100 hub study - -# notebook stuff -jupyter-contrib-nbextensions==0.5.1 -jupyter-server-proxy==1.2.0 -git+https://github.com/ryanlovett/jupyter-tree-download.git From 4f29459434788569600aa48039abcb1ecb073194 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Fri, 20 Jan 2023 14:24:26 -0800 Subject: [PATCH 104/184] Splitting data100 into a separate image. --- apt.txt | 94 ++++++++++++++++++++++++++++++++++++++++++ environment.yml | 92 +++++++++++++++++++++++++++++++++++++++++ infra-requirements.txt | 32 ++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 apt.txt create mode 100644 environment.yml create mode 100644 infra-requirements.txt diff --git a/apt.txt b/apt.txt new file mode 100644 index 0000000..8edfcc4 --- /dev/null +++ b/apt.txt @@ -0,0 +1,94 @@ +# Some linux packages for basic terminal work, particularly +# oriented at users new to Unix/cmd line environments. + +# Basic unix tools +man +man-db +manpages-posix +manpages-dev +manpages-posix-dev + +# Download tools +curl +wget + +# Core text editors on a *nix box: vim and emacs +vim +emacs-nox +emacs-goodies-el +python-mode + +# A couple of CLI editors that are easier than vim +# micro # currently not working on 18.04 +nano +jed +jed-extra + +# powerful terminal-based file manager, better than the one in JLab +mc + +# for easily managing multiple repositories with one command (perl-doc +# is needed for its help pages to work) +mr +perl-doc + +# Regular build tools for compiling common stuff +build-essential +gfortran + +# Dependencies for nbconvert +texlive-xetex +texlive-fonts-recommended +texlive-plain-generic +# https://github.com/berkeley-dsep-infra/datahub/issues/3719 +texlive-lang-chinese +lmodern + +# Other useful document-related tools +pandoc +latexdiff + +# Some useful git utilities use basic Ruby +ruby + +# Other niceties for command-line work and life +ack # powerful grep-like tool +pydf # colorized disk usage +tmux +screen +htop +nnn # cmd line file manager +zsh +rsync +tig # console UI for git +multitail + +# For later, these are not available in 18.04 +#browsh # text-based web browser, occasionally handy +#dasel # json/yml/csv/etc data wrangling at the terminal +#fzf # fuzzy file finder + +## This section adds tools for desktop environment usage +dbus-x11 +xorg +xubuntu-icon-theme +xfce4 +xfce4-goodies +xclip +xsel +firefox +chromium-browser + +# GUI text editors +emacs +vim-gtk3 +gedit + +# Git clients and tools +git-gui +gitg +qgit +meld + +# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 +zip \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..84cd0bc --- /dev/null +++ b/environment.yml @@ -0,0 +1,92 @@ +name: data100-s23 + +channels: +- conda-forge + +dependencies: +- python==3.9 +- syncthing==1.20.4 +- git +- altair==4.2.0 +- beautifulsoup4==4.11.1 +- black==22.6.0 +- bokeh==2.4.3 +- bqplot==0.12.34 +- cartopy==0.21.0 +- coverage==6.4.4 +- cython==0.29.32 +- dask==2022.8.1 +- dask-labextension==5.3.0 +- fortran-magic==0.7 +- h5netcdf==1.0.2 +- h5py==3.7.0 +- hdf4==4.2.15 +- hdf5==1.12.2 +- intake==0.6.5 +- intake-esm==2021.8.17 +- intake-xarray==0.6.0 +- ipycanvas==0.12.1 +- ipydatagrid==1.1.12 +- ipympl==0.9.2 +- ipyparallel==8.4.1 +- jupyter-book==0.13.1 +- jupyter-repo2docker==2022.2.0 +- jupyter-resource-usage==0.6.1 +- jupyter_bokeh==3.0.4 +- jupyterlab==3.4.5 +- jupyterlab-drawio==0.9.0 +- jupyterlab-favorites==3.0.0 +- jupyterlab-geojson==3.2.0 +- jupyterlab-git==0.39.0 +- jupyterlab-link-share==0.2.4 +- jupyterlab-variableinspector==3.0.9 +- jupyterlab_pygments==0.2.2 +- jupyterlab_widgets==3.0.2 +- jupytext==1.14.0 +- matplotlib==3.5.3 +- matplotlib-inline==0.1.6 +- mock==4.0.3 +- nbdime==3.1.1 +- networkx==2.8.6 +- numba +- numpy==1.23.2 +- pandas==1.4.3 +- pandoc==2.19.2 +- pandocfilters==1.5.0 +- pep8==1.7.1 +- pillow==9.2.0 +- plotly==5.10.0 +- pooch==1.6.0 +- prettytable==3.4.1 +- pyarrow==9.0.0 +- pypdf2==2.10.4 +- pytables==3.7.0 +- pytest==7.1.2 +- pytest-cov==3.0.0 +- pytest-notebook==0.6.1 +- python-pdfkit==1.0.0 +- requests==2.28.1 +- scikit-image==0.19.3 +- scikit-learn==1.1.2 +- scipy==1.9.0 +- seaborn==0.11.2 +- sphinx-jupyterbook-latex==0.4.6 +- sqlparse==0.4.3 +- statsmodels==0.13.2 +- sympy==1.10.1 +- tqdm==4.64.0 +- xarray==2022.6.0 +- xlrd==2.0.1 +- micro==2.0.8 +- websockify +- pip +- pip: + - -r infra-requirements.txt + - jupyter-desktop-server + # For push authentication to GitHub + - gh-scoped-creds==4.1 + # Upgrade separate from what everyone else uses for now + # https://github.com/berkeley-dsep-infra/datahub/issues/3693 + - otter-grader==4.0.1 + - ipython-sql==0.4.1 + - geopandas==0.12.1 diff --git a/infra-requirements.txt b/infra-requirements.txt new file mode 100644 index 0000000..15b23ad --- /dev/null +++ b/infra-requirements.txt @@ -0,0 +1,32 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +notebook==6.4.12 +jupyterlab==3.4.5 +retrolab==0.3.21 +nbgitpuller==1.1.0 +jupyter-resource-usage==0.6.1 +otter-grader==4.0.1 +# Matches version in images/hub/Dockerfile +jupyterhub==3.1.0 +appmode==0.8.0 +ipywidgets==7.7.2 +jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 +# Enough people like this, let's load it in. +jupyter-contrib-nbextensions==0.5.1 +jupyter_nbextensions_configurator==0.4.1 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +popularity-contest==0.4.1 +# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 +RISE==5.7.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.3 From 3df28848475a097d3f40e9e5529e3a4f11ce5b7e Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Fri, 20 Jan 2023 14:24:26 -0800 Subject: [PATCH 105/184] Splitting data100 into a separate image. --- apt.txt | 94 ++++++++++++++++++++++++++++++++++++++++++ environment.yml | 92 +++++++++++++++++++++++++++++++++++++++++ infra-requirements.txt | 32 ++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 apt.txt create mode 100644 environment.yml create mode 100644 infra-requirements.txt diff --git a/apt.txt b/apt.txt new file mode 100644 index 0000000..8edfcc4 --- /dev/null +++ b/apt.txt @@ -0,0 +1,94 @@ +# Some linux packages for basic terminal work, particularly +# oriented at users new to Unix/cmd line environments. + +# Basic unix tools +man +man-db +manpages-posix +manpages-dev +manpages-posix-dev + +# Download tools +curl +wget + +# Core text editors on a *nix box: vim and emacs +vim +emacs-nox +emacs-goodies-el +python-mode + +# A couple of CLI editors that are easier than vim +# micro # currently not working on 18.04 +nano +jed +jed-extra + +# powerful terminal-based file manager, better than the one in JLab +mc + +# for easily managing multiple repositories with one command (perl-doc +# is needed for its help pages to work) +mr +perl-doc + +# Regular build tools for compiling common stuff +build-essential +gfortran + +# Dependencies for nbconvert +texlive-xetex +texlive-fonts-recommended +texlive-plain-generic +# https://github.com/berkeley-dsep-infra/datahub/issues/3719 +texlive-lang-chinese +lmodern + +# Other useful document-related tools +pandoc +latexdiff + +# Some useful git utilities use basic Ruby +ruby + +# Other niceties for command-line work and life +ack # powerful grep-like tool +pydf # colorized disk usage +tmux +screen +htop +nnn # cmd line file manager +zsh +rsync +tig # console UI for git +multitail + +# For later, these are not available in 18.04 +#browsh # text-based web browser, occasionally handy +#dasel # json/yml/csv/etc data wrangling at the terminal +#fzf # fuzzy file finder + +## This section adds tools for desktop environment usage +dbus-x11 +xorg +xubuntu-icon-theme +xfce4 +xfce4-goodies +xclip +xsel +firefox +chromium-browser + +# GUI text editors +emacs +vim-gtk3 +gedit + +# Git clients and tools +git-gui +gitg +qgit +meld + +# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 +zip \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..84cd0bc --- /dev/null +++ b/environment.yml @@ -0,0 +1,92 @@ +name: data100-s23 + +channels: +- conda-forge + +dependencies: +- python==3.9 +- syncthing==1.20.4 +- git +- altair==4.2.0 +- beautifulsoup4==4.11.1 +- black==22.6.0 +- bokeh==2.4.3 +- bqplot==0.12.34 +- cartopy==0.21.0 +- coverage==6.4.4 +- cython==0.29.32 +- dask==2022.8.1 +- dask-labextension==5.3.0 +- fortran-magic==0.7 +- h5netcdf==1.0.2 +- h5py==3.7.0 +- hdf4==4.2.15 +- hdf5==1.12.2 +- intake==0.6.5 +- intake-esm==2021.8.17 +- intake-xarray==0.6.0 +- ipycanvas==0.12.1 +- ipydatagrid==1.1.12 +- ipympl==0.9.2 +- ipyparallel==8.4.1 +- jupyter-book==0.13.1 +- jupyter-repo2docker==2022.2.0 +- jupyter-resource-usage==0.6.1 +- jupyter_bokeh==3.0.4 +- jupyterlab==3.4.5 +- jupyterlab-drawio==0.9.0 +- jupyterlab-favorites==3.0.0 +- jupyterlab-geojson==3.2.0 +- jupyterlab-git==0.39.0 +- jupyterlab-link-share==0.2.4 +- jupyterlab-variableinspector==3.0.9 +- jupyterlab_pygments==0.2.2 +- jupyterlab_widgets==3.0.2 +- jupytext==1.14.0 +- matplotlib==3.5.3 +- matplotlib-inline==0.1.6 +- mock==4.0.3 +- nbdime==3.1.1 +- networkx==2.8.6 +- numba +- numpy==1.23.2 +- pandas==1.4.3 +- pandoc==2.19.2 +- pandocfilters==1.5.0 +- pep8==1.7.1 +- pillow==9.2.0 +- plotly==5.10.0 +- pooch==1.6.0 +- prettytable==3.4.1 +- pyarrow==9.0.0 +- pypdf2==2.10.4 +- pytables==3.7.0 +- pytest==7.1.2 +- pytest-cov==3.0.0 +- pytest-notebook==0.6.1 +- python-pdfkit==1.0.0 +- requests==2.28.1 +- scikit-image==0.19.3 +- scikit-learn==1.1.2 +- scipy==1.9.0 +- seaborn==0.11.2 +- sphinx-jupyterbook-latex==0.4.6 +- sqlparse==0.4.3 +- statsmodels==0.13.2 +- sympy==1.10.1 +- tqdm==4.64.0 +- xarray==2022.6.0 +- xlrd==2.0.1 +- micro==2.0.8 +- websockify +- pip +- pip: + - -r infra-requirements.txt + - jupyter-desktop-server + # For push authentication to GitHub + - gh-scoped-creds==4.1 + # Upgrade separate from what everyone else uses for now + # https://github.com/berkeley-dsep-infra/datahub/issues/3693 + - otter-grader==4.0.1 + - ipython-sql==0.4.1 + - geopandas==0.12.1 diff --git a/infra-requirements.txt b/infra-requirements.txt new file mode 100644 index 0000000..15b23ad --- /dev/null +++ b/infra-requirements.txt @@ -0,0 +1,32 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +notebook==6.4.12 +jupyterlab==3.4.5 +retrolab==0.3.21 +nbgitpuller==1.1.0 +jupyter-resource-usage==0.6.1 +otter-grader==4.0.1 +# Matches version in images/hub/Dockerfile +jupyterhub==3.1.0 +appmode==0.8.0 +ipywidgets==7.7.2 +jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 +# Enough people like this, let's load it in. +jupyter-contrib-nbextensions==0.5.1 +jupyter_nbextensions_configurator==0.4.1 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +popularity-contest==0.4.1 +# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 +RISE==5.7.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.3 From 9da9aa76b08476ccdc1bd815adf90d46002df75c Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Fri, 27 Jan 2023 15:57:34 -0800 Subject: [PATCH 106/184] Update environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 84cd0bc..05fd6ba 100644 --- a/environment.yml +++ b/environment.yml @@ -79,6 +79,7 @@ dependencies: - xlrd==2.0.1 - micro==2.0.8 - websockify +- folium==0.14.0 - pip - pip: - -r infra-requirements.txt From fd9cb08f66090fb6fea006ae558ad35ddceae1eb Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Wed, 15 Feb 2023 12:22:27 -0800 Subject: [PATCH 107/184] added sqlalchemy to requirements.txt --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 05fd6ba..19d7d83 100644 --- a/environment.yml +++ b/environment.yml @@ -80,6 +80,7 @@ dependencies: - micro==2.0.8 - websockify - folium==0.14.0 +- sqlalchemy==1.4.29 - pip - pip: - -r infra-requirements.txt From 3476c9c8620c7c0052eec8580963c0940930d842 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 28 Mar 2023 12:01:13 -0700 Subject: [PATCH 108/184] upgrade jupyterhub to 4.0.0b2, normalize python env files --- infra-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 15b23ad..4a6e199 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,9 +13,8 @@ jupyterlab==3.4.5 retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 -otter-grader==4.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==3.1.0 +jupyterhub==4.0.0b2 appmode==0.8.0 ipywidgets==7.7.2 jupyter-tree-download==1.0.1 From 9ad0a5cfa1ab9cc3aa9c9360fe512eee89f5eda5 Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Wed, 12 Apr 2023 23:03:04 -0700 Subject: [PATCH 109/184] Update environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 19d7d83..b413526 100644 --- a/environment.yml +++ b/environment.yml @@ -92,3 +92,4 @@ dependencies: - otter-grader==4.0.1 - ipython-sql==0.4.1 - geopandas==0.12.1 + - iwut==0.0.4 From 03fab800f7af1c1959e20d3ba8c9ffb041faa61f Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 18 Apr 2023 12:50:39 -0700 Subject: [PATCH 110/184] Update environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index b413526..2af3c86 100644 --- a/environment.yml +++ b/environment.yml @@ -81,6 +81,7 @@ dependencies: - websockify - folium==0.14.0 - sqlalchemy==1.4.29 +- tensorflow==2.11.1 - pip - pip: - -r infra-requirements.txt From 742836b586f697456bae84ebcebb349162c92df5 Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 18 Apr 2023 16:37:14 -0700 Subject: [PATCH 111/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2af3c86..c979c46 100644 --- a/environment.yml +++ b/environment.yml @@ -81,7 +81,7 @@ dependencies: - websockify - folium==0.14.0 - sqlalchemy==1.4.29 -- tensorflow==2.11.1 +- tensorflow-mkl==2.5.0 - pip - pip: - -r infra-requirements.txt From 655145e80f87577fd4dd4c4df9da4529eab46d7a Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 18 Apr 2023 20:59:09 -0700 Subject: [PATCH 112/184] Update environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index c979c46..8da5582 100644 --- a/environment.yml +++ b/environment.yml @@ -82,6 +82,7 @@ dependencies: - folium==0.14.0 - sqlalchemy==1.4.29 - tensorflow-mkl==2.5.0 +- _tflow_select==2.2.0 - pip - pip: - -r infra-requirements.txt From 77c4f5ae27499814adaae040f7a1ec515a2715a5 Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 18 Apr 2023 21:26:55 -0700 Subject: [PATCH 113/184] Update environment.yml --- environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 8da5582..e668130 100644 --- a/environment.yml +++ b/environment.yml @@ -81,8 +81,6 @@ dependencies: - websockify - folium==0.14.0 - sqlalchemy==1.4.29 -- tensorflow-mkl==2.5.0 -- _tflow_select==2.2.0 - pip - pip: - -r infra-requirements.txt @@ -95,3 +93,4 @@ dependencies: - ipython-sql==0.4.1 - geopandas==0.12.1 - iwut==0.0.4 + - tensorflow-cpu==2.12.0 From 3d03f2cbc8d328079c1c5bc68e5718e421440669 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 18 May 2023 16:17:57 -0700 Subject: [PATCH 114/184] upgrading data100, eecs, stat159 to jammy --- apt.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apt.txt b/apt.txt index 8edfcc4..c4bb4d0 100644 --- a/apt.txt +++ b/apt.txt @@ -16,7 +16,6 @@ wget vim emacs-nox emacs-goodies-el -python-mode # A couple of CLI editors that are easier than vim # micro # currently not working on 18.04 @@ -91,4 +90,4 @@ qgit meld # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 -zip \ No newline at end of file +zip From 77dbc2b510e5253ec0d50c0c2350c60c3765e41d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 13 Jun 2023 09:45:23 -0700 Subject: [PATCH 115/184] add tornado, comment out some infra-reqs.txt --- environment.yml | 1 + infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e668130..acd0af2 100644 --- a/environment.yml +++ b/environment.yml @@ -74,6 +74,7 @@ dependencies: - sqlparse==0.4.3 - statsmodels==0.13.2 - sympy==1.10.1 +- tornado==6.2.0 - tqdm==4.64.0 - xarray==2022.6.0 - xlrd==2.0.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index 4a6e199..4aed6a7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 -jupyterlab==3.4.5 +# jupyterlab==3.4.5 retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 From e14aed27d7cdfd6776ff32064a0de5faa4d7568e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 13 Jun 2023 09:48:23 -0700 Subject: [PATCH 116/184] derp dont change data100 --- environment.yml | 1 - infra-requirements.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index acd0af2..e668130 100644 --- a/environment.yml +++ b/environment.yml @@ -74,7 +74,6 @@ dependencies: - sqlparse==0.4.3 - statsmodels==0.13.2 - sympy==1.10.1 -- tornado==6.2.0 - tqdm==4.64.0 - xarray==2022.6.0 - xlrd==2.0.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index 4aed6a7..4a6e199 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 -# jupyterlab==3.4.5 +jupyterlab==3.4.5 retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 From ee808edd53ef10fdc9e7e0a8302b115cf8cd5cec Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Jun 2023 12:02:59 -0700 Subject: [PATCH 117/184] removed pinned versions of repo2docker --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e668130..8ad7215 100644 --- a/environment.yml +++ b/environment.yml @@ -30,7 +30,7 @@ dependencies: - ipympl==0.9.2 - ipyparallel==8.4.1 - jupyter-book==0.13.1 -- jupyter-repo2docker==2022.2.0 +- jupyter-repo2docker==2023.06.0 - jupyter-resource-usage==0.6.1 - jupyter_bokeh==3.0.4 - jupyterlab==3.4.5 From 74f70f7c092cd531d2b3ae699c22eb2a80340c4c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Jun 2023 16:52:22 -0700 Subject: [PATCH 118/184] pull repo2docker from singleuser images --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8ad7215..1002c48 100644 --- a/environment.yml +++ b/environment.yml @@ -30,7 +30,6 @@ dependencies: - ipympl==0.9.2 - ipyparallel==8.4.1 - jupyter-book==0.13.1 -- jupyter-repo2docker==2023.06.0 - jupyter-resource-usage==0.6.1 - jupyter_bokeh==3.0.4 - jupyterlab==3.4.5 From 55253e3d4f9f39a91c1342d80a31d405fbf74608 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 10 Aug 2023 17:33:13 -0700 Subject: [PATCH 119/184] initial commit to upgrade data100 to jl 4/notebook 7 --- apt.txt | 5 +-- environment.yml | 82 ++++++++++++++++++++++++++----------------------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/apt.txt b/apt.txt index c4bb4d0..bfc2a10 100644 --- a/apt.txt +++ b/apt.txt @@ -12,10 +12,8 @@ manpages-posix-dev curl wget -# Core text editors on a *nix box: vim and emacs +# Core text editors on a *nix box: vim vim -emacs-nox -emacs-goodies-el # A couple of CLI editors that are easier than vim # micro # currently not working on 18.04 @@ -79,7 +77,6 @@ firefox chromium-browser # GUI text editors -emacs vim-gtk3 gedit diff --git a/environment.yml b/environment.yml index 1002c48..e67ff03 100644 --- a/environment.yml +++ b/environment.yml @@ -1,23 +1,24 @@ -name: data100-s23 +name: data100-jl4 channels: - conda-forge dependencies: -- python==3.9 +- python==3.11.0 - syncthing==1.20.4 -- git -- altair==4.2.0 +- git==2.39.1 +- altair==5.0.1 - beautifulsoup4==4.11.1 - black==22.6.0 - bokeh==2.4.3 - bqplot==0.12.34 - cartopy==0.21.0 -- coverage==6.4.4 +- coverage==7.2.2 - cython==0.29.32 -- dask==2022.8.1 -- dask-labextension==5.3.0 +- distributed==2023.6.0 - fortran-magic==0.7 +- graphviz==8.0.3 +- python-graphviz==0.20.1 - h5netcdf==1.0.2 - h5py==3.7.0 - hdf4==4.2.15 @@ -29,32 +30,34 @@ dependencies: - ipydatagrid==1.1.12 - ipympl==0.9.2 - ipyparallel==8.4.1 -- jupyter-book==0.13.1 -- jupyter-resource-usage==0.6.1 -- jupyter_bokeh==3.0.4 -- jupyterlab==3.4.5 -- jupyterlab-drawio==0.9.0 +- jsonschema==4.17.3 +- jupyter-book==0.15.1 +- jupyter-resource-usage==1.0.0 +- jupyter_bokeh +- jupyterlab==4.0.4 - jupyterlab-favorites==3.0.0 - jupyterlab-geojson==3.2.0 -- jupyterlab-git==0.39.0 -- jupyterlab-link-share==0.2.4 - jupyterlab-variableinspector==3.0.9 - jupyterlab_pygments==0.2.2 -- jupyterlab_widgets==3.0.2 -- jupytext==1.14.0 -- matplotlib==3.5.3 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- matplotlib==3.7.1 - matplotlib-inline==0.1.6 - mock==4.0.3 +- nbclassic==1.0.0 - nbdime==3.1.1 +- nbgitpuller==1.1.1 - networkx==2.8.6 -- numba -- numpy==1.23.2 -- pandas==1.4.3 -- pandoc==2.19.2 +- notebook==7.0.2 +- numba==0.57.0 +- numpy==1.24.2 +- pandas==2.0.2 +- pandoc==2.12 - pandocfilters==1.5.0 - pep8==1.7.1 - pillow==9.2.0 -- plotly==5.10.0 +- plotly==5.13.1 - pooch==1.6.0 - prettytable==3.4.1 - pyarrow==9.0.0 @@ -62,34 +65,37 @@ dependencies: - pytables==3.7.0 - pytest==7.1.2 - pytest-cov==3.0.0 -- pytest-notebook==0.6.1 -- python-pdfkit==1.0.0 -- requests==2.28.1 +- python-pdfkit==1.0.0 +- requests==2.28.2 - scikit-image==0.19.3 -- scikit-learn==1.1.2 -- scipy==1.9.0 -- seaborn==0.11.2 -- sphinx-jupyterbook-latex==0.4.6 +- scikit-learn==1.2.2 +- scipy==1.10.1 +- seaborn==0.12.2 +- sphinx-jupyterbook-latex==0.5.2 - sqlparse==0.4.3 -- statsmodels==0.13.2 +- statsmodels==0.14.0 - sympy==1.10.1 +- tornado==6.2.0 - tqdm==4.64.0 -- xarray==2022.6.0 +- xarray==2023.5.0 - xlrd==2.0.1 - micro==2.0.8 -- websockify +- websockify==0.11.0 - folium==0.14.0 -- sqlalchemy==1.4.29 +- sqlalchemy==2.0.16 - pip - pip: - - -r infra-requirements.txt + # - -r infra-requirements.txt - jupyter-desktop-server - # For push authentication to GitHub - gh-scoped-creds==4.1 - # Upgrade separate from what everyone else uses for now - # https://github.com/berkeley-dsep-infra/datahub/issues/3693 - otter-grader==4.0.1 - - ipython-sql==0.4.1 + - jupysql==0.8.0 - geopandas==0.12.1 - iwut==0.0.4 - tensorflow-cpu==2.12.0 + - ipywidgets==8.0.7 + # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) + # - jupyter_collaboration==1.0.1 + - jupyterhub==4.0.1 + - nbconvert==7.6.0 + - pytest-notebook==0.8.1 From 09ea25a970af6ec9d7b3ccf4fab9f7a6be715d1f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 11 Aug 2023 09:40:34 -0700 Subject: [PATCH 120/184] update the conda env name --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e67ff03..f7f8a64 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: data100-jl4 +name: data100-FA23 channels: - conda-forge From ae3e2d11a248289d2c5e00900f964fa925d6ec34 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 14 Aug 2023 11:41:59 -0700 Subject: [PATCH 121/184] through enough stuff at the wall... --- environment.yml | 142 ++++++++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 65 deletions(-) diff --git a/environment.yml b/environment.yml index f7f8a64..286a708 100644 --- a/environment.yml +++ b/environment.yml @@ -5,94 +5,106 @@ channels: dependencies: - python==3.11.0 -- syncthing==1.20.4 +# - syncthing==1.20.4 - git==2.39.1 -- altair==5.0.1 -- beautifulsoup4==4.11.1 -- black==22.6.0 -- bokeh==2.4.3 -- bqplot==0.12.34 -- cartopy==0.21.0 -- coverage==7.2.2 -- cython==0.29.32 -- distributed==2023.6.0 -- fortran-magic==0.7 -- graphviz==8.0.3 -- python-graphviz==0.20.1 -- h5netcdf==1.0.2 -- h5py==3.7.0 -- hdf4==4.2.15 -- hdf5==1.12.2 -- intake==0.6.5 -- intake-esm==2021.8.17 -- intake-xarray==0.6.0 -- ipycanvas==0.12.1 -- ipydatagrid==1.1.12 -- ipympl==0.9.2 -- ipyparallel==8.4.1 -- jsonschema==4.17.3 -- jupyter-book==0.15.1 +# - altair==5.0.1 +# - beautifulsoup4==4.11.1 +# - black==22.6.0 +# - bokeh==2.4.3 +# - bqplot==0.12.34 +# - cartopy==0.21.0 +# - coverage==7.2.2 +# - cython==0.29.32 +# - distributed==2023.6.0 +# - fortran-magic==0.7 +# - graphviz==8.0.3 +# - python-graphviz==0.20.1 +# - h5netcdf==1.0.2 +# - h5py==3.7.0 +# - hdf4==4.2.15 +# - hdf5==1.12.2 +# - intake==0.6.5 +# - intake-esm==2021.8.17 +# - intake-xarray==0.6.0 +# - ipycanvas==0.12.1 +# - ipydatagrid==1.1.12 +# - ipympl==0.9.2 +# - ipyparallel==8.4.1 +# - jsonschema==4.17.3 +# - jupyter-book==0.15.1 - jupyter-resource-usage==1.0.0 -- jupyter_bokeh +# - jupyter_bokeh - jupyterlab==4.0.4 - jupyterlab-favorites==3.0.0 -- jupyterlab-geojson==3.2.0 -- jupyterlab-variableinspector==3.0.9 -- jupyterlab_pygments==0.2.2 +# - jupyterlab-geojson==3.2.0 +# - jupyterlab-variableinspector==3.0.9 +# - jupyterlab_pygments==0.2.2 - jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 - jupyter_server==2.7.0 -- matplotlib==3.7.1 -- matplotlib-inline==0.1.6 -- mock==4.0.3 -- nbclassic==1.0.0 -- nbdime==3.1.1 +# - matplotlib==3.7.1 +# - matplotlib-inline==0.1.6 +# - mock==4.0.3 +# - nbclassic==1.0.0 +# - nbdime==3.1.1 - nbgitpuller==1.1.1 -- networkx==2.8.6 +# - networkx==2.8.6 - notebook==7.0.2 -- numba==0.57.0 +# - numba==0.57.0 +# - numpy==1.24.2 +# - pandas==2.0.2 +# - pandoc==2.12 +# - pandocfilters==1.5.0 +# - pep8==1.7.1 +# - pillow==9.2.0 +# - plotly==5.13.1 +# - pooch==1.6.0 +# - prettytable==3.4.1 +# - pyarrow==9.0.0 +# - pypdf2==2.10.4 +# - pytables==3.7.0 +# - pytest==7.1.2 +# - pytest-cov==3.0.0 +# - python-pdfkit==1.0.0 +# - requests==2.28.2 +# - scikit-image==0.19.3 +# - scikit-learn==1.2.2 +# - scipy==1.10.1 +# - seaborn==0.12.2 +# - sphinx-jupyterbook-latex==0.5.2 +# - sqlparse==0.4.3 +# - statsmodels==0.14.0 +# - sympy==1.10.1 +# - tornado==6.2.0 +# - tqdm==4.64.0 +# - xarray==2023.5.0 +# - xlrd==2.0.1 +# - micro==2.0.8 +# - websockify==0.11.0 +# - folium==0.14.0 +- folium==0.14.0 +- h5netcdf==1.0.2 +- ipywidgets==8.0.7 +- jupysql==0.8.0 +- jupyterlab==4.0.4 +- matplotlib==3.7.1 +- mdit-py-plugins==0.4.0 - numpy==1.24.2 +- otter-grader==5.1.1 - pandas==2.0.2 -- pandoc==2.12 -- pandocfilters==1.5.0 -- pep8==1.7.1 -- pillow==9.2.0 - plotly==5.13.1 -- pooch==1.6.0 -- prettytable==3.4.1 -- pyarrow==9.0.0 -- pypdf2==2.10.4 -- pytables==3.7.0 -- pytest==7.1.2 -- pytest-cov==3.0.0 -- python-pdfkit==1.0.0 - requests==2.28.2 - scikit-image==0.19.3 - scikit-learn==1.2.2 - scipy==1.10.1 - seaborn==0.12.2 -- sphinx-jupyterbook-latex==0.5.2 -- sqlparse==0.4.3 - statsmodels==0.14.0 -- sympy==1.10.1 -- tornado==6.2.0 -- tqdm==4.64.0 -- xarray==2023.5.0 -- xlrd==2.0.1 -- micro==2.0.8 -- websockify==0.11.0 -- folium==0.14.0 +- tensorflow-cpu==2.12.0 - sqlalchemy==2.0.16 - pip - pip: # - -r infra-requirements.txt - - jupyter-desktop-server - - gh-scoped-creds==4.1 - - otter-grader==4.0.1 - jupysql==0.8.0 - - geopandas==0.12.1 - - iwut==0.0.4 - - tensorflow-cpu==2.12.0 - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 From 8e52416d281a68672731d5b4f3408e894d7e4b7b Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 14 Aug 2023 11:47:21 -0700 Subject: [PATCH 122/184] ...and sometimes it sticks? --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 286a708..3e4206f 100644 --- a/environment.yml +++ b/environment.yml @@ -99,7 +99,7 @@ dependencies: - scipy==1.10.1 - seaborn==0.12.2 - statsmodels==0.14.0 -- tensorflow-cpu==2.12.0 +- tensorflow-cpu==2.12.1 - sqlalchemy==2.0.16 - pip - pip: From e4d22e0dc1896d96120b60469b6c3dd190a95b38 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 14 Aug 2023 11:58:41 -0700 Subject: [PATCH 123/184] forgot to remove commented out entries --- environment.yml | 68 ------------------------------------------------- 1 file changed, 68 deletions(-) diff --git a/environment.yml b/environment.yml index 3e4206f..f47da76 100644 --- a/environment.yml +++ b/environment.yml @@ -5,83 +5,15 @@ channels: dependencies: - python==3.11.0 -# - syncthing==1.20.4 - git==2.39.1 -# - altair==5.0.1 -# - beautifulsoup4==4.11.1 -# - black==22.6.0 -# - bokeh==2.4.3 -# - bqplot==0.12.34 -# - cartopy==0.21.0 -# - coverage==7.2.2 -# - cython==0.29.32 -# - distributed==2023.6.0 -# - fortran-magic==0.7 -# - graphviz==8.0.3 -# - python-graphviz==0.20.1 -# - h5netcdf==1.0.2 -# - h5py==3.7.0 -# - hdf4==4.2.15 -# - hdf5==1.12.2 -# - intake==0.6.5 -# - intake-esm==2021.8.17 -# - intake-xarray==0.6.0 -# - ipycanvas==0.12.1 -# - ipydatagrid==1.1.12 -# - ipympl==0.9.2 -# - ipyparallel==8.4.1 -# - jsonschema==4.17.3 -# - jupyter-book==0.15.1 - jupyter-resource-usage==1.0.0 -# - jupyter_bokeh - jupyterlab==4.0.4 - jupyterlab-favorites==3.0.0 -# - jupyterlab-geojson==3.2.0 -# - jupyterlab-variableinspector==3.0.9 -# - jupyterlab_pygments==0.2.2 - jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 - jupyter_server==2.7.0 -# - matplotlib==3.7.1 -# - matplotlib-inline==0.1.6 -# - mock==4.0.3 -# - nbclassic==1.0.0 -# - nbdime==3.1.1 - nbgitpuller==1.1.1 -# - networkx==2.8.6 - notebook==7.0.2 -# - numba==0.57.0 -# - numpy==1.24.2 -# - pandas==2.0.2 -# - pandoc==2.12 -# - pandocfilters==1.5.0 -# - pep8==1.7.1 -# - pillow==9.2.0 -# - plotly==5.13.1 -# - pooch==1.6.0 -# - prettytable==3.4.1 -# - pyarrow==9.0.0 -# - pypdf2==2.10.4 -# - pytables==3.7.0 -# - pytest==7.1.2 -# - pytest-cov==3.0.0 -# - python-pdfkit==1.0.0 -# - requests==2.28.2 -# - scikit-image==0.19.3 -# - scikit-learn==1.2.2 -# - scipy==1.10.1 -# - seaborn==0.12.2 -# - sphinx-jupyterbook-latex==0.5.2 -# - sqlparse==0.4.3 -# - statsmodels==0.14.0 -# - sympy==1.10.1 -# - tornado==6.2.0 -# - tqdm==4.64.0 -# - xarray==2023.5.0 -# - xlrd==2.0.1 -# - micro==2.0.8 -# - websockify==0.11.0 -# - folium==0.14.0 - folium==0.14.0 - h5netcdf==1.0.2 - ipywidgets==8.0.7 From cc18964993d528c91d0ef7c6198946f34f0ae1cf Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Mon, 14 Aug 2023 20:56:39 -0700 Subject: [PATCH 124/184] Bump jupyterhub to 4.0.2 to get off of betas. Version 4.0.2 is also found in version 3.0.0 of the helm chart. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f47da76..bb6122d 100644 --- a/environment.yml +++ b/environment.yml @@ -40,6 +40,6 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.0.1 + - jupyterhub==4.0.2 - nbconvert==7.6.0 - pytest-notebook==0.8.1 From 6c68bf4bfcbe78ed5a8d52fe78377d1dd84f3bfb Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Mon, 14 Aug 2023 21:04:02 -0700 Subject: [PATCH 125/184] Run infra-packages script. --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 4a6e199..afa0268 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -14,7 +14,7 @@ retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.0.0b2 +jupyterhub==4.0.2 appmode==0.8.0 ipywidgets==7.7.2 jupyter-tree-download==1.0.1 From 816b6fadea7c6a0cd315b926303e492704fd75e5 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 16 Aug 2023 09:25:12 -0700 Subject: [PATCH 126/184] debump otter --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index bb6122d..5c2c919 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==5.1.1 +- otter-grader==4.4.3 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 From 78f1044915891b13f324466acbf4d15f97da9531 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 16 Aug 2023 09:32:35 -0700 Subject: [PATCH 127/184] s/3/1 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 5c2c919..39d848f 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==4.4.3 +- otter-grader==4.4.1 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 From afd0bad8331c62b0d4c86757f9669c2ae6ca240c Mon Sep 17 00:00:00 2001 From: mihranmiroyan Date: Thu, 17 Aug 2023 10:32:13 +0400 Subject: [PATCH 128/184] github credentials package --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 39d848f..769517f 100644 --- a/environment.yml +++ b/environment.yml @@ -43,3 +43,4 @@ dependencies: - jupyterhub==4.0.2 - nbconvert==7.6.0 - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 From d746747e33125c1bd0000c9e95b447f8787e51f4 Mon Sep 17 00:00:00 2001 From: mihranmiroyan Date: Fri, 18 Aug 2023 09:40:54 +0400 Subject: [PATCH 129/184] otter upgrade --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 769517f..79fa7a8 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==4.4.1 +- otter-grader==5.1.1 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 From 246cf72954a9d32767394e4e8928fd1546e8d0bb Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Wed, 23 Aug 2023 16:06:20 -0700 Subject: [PATCH 130/184] Debump otter to v4.4.1 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 79fa7a8..769517f 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==5.1.1 +- otter-grader==4.4.1 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 From 0cde735d68731a097ba7154a4dcafbe7d11693a0 Mon Sep 17 00:00:00 2001 From: mihranmiroyan Date: Thu, 24 Aug 2023 08:03:29 -0700 Subject: [PATCH 131/184] otter bump --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 769517f..a612e32 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==4.4.1 +- otter-grader==5.1.3 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 From 269b42f860cd4ce4a956a53c372e0e18e84ab8fd Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 12 Sep 2023 15:33:19 -0700 Subject: [PATCH 132/184] duplicate jupysql entry --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index a612e32..8b3a722 100644 --- a/environment.yml +++ b/environment.yml @@ -36,7 +36,6 @@ dependencies: - pip - pip: # - -r infra-requirements.txt - - jupysql==0.8.0 - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 From a0f0a06c620b0e30bb71c1ebd89bb788f34e211f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 9 Oct 2023 09:38:47 -0700 Subject: [PATCH 133/184] move python popcon from infra-reqs to environment.yaml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 8b3a722..7cb2ad8 100644 --- a/environment.yml +++ b/environment.yml @@ -43,3 +43,4 @@ dependencies: - nbconvert==7.6.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 + - popularity-contest==0.4.1 From 94c173daa804f262de6a2a6ee1e4f3d55ef181bd Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 31 Oct 2023 11:58:04 -0700 Subject: [PATCH 134/184] update popcon to the hash of my PR --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 7cb2ad8..9ce421a 100644 --- a/environment.yml +++ b/environment.yml @@ -43,4 +43,4 @@ dependencies: - nbconvert==7.6.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - - popularity-contest==0.4.1 + - popularity-contest==git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 diff --git a/infra-requirements.txt b/infra-requirements.txt index afa0268..75e9ab7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -24,7 +24,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.4.1 +popularity-contest==git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From a0b345d823dab8dfdfd1ca91a6985ba3dda45cbe Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 31 Oct 2023 12:21:40 -0700 Subject: [PATCH 135/184] s/python-popularity==// --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 9ce421a..de44dbd 100644 --- a/environment.yml +++ b/environment.yml @@ -43,4 +43,4 @@ dependencies: - nbconvert==7.6.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - - popularity-contest==git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 + - git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 diff --git a/infra-requirements.txt b/infra-requirements.txt index 75e9ab7..611bfb5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -24,7 +24,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 +git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From 4d797c334c231c92a1737f88055cee0c9522d2d5 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 7 Nov 2023 13:47:47 -0800 Subject: [PATCH 136/184] update infra-reqs, all hubs etc to run latest popcon with better error handling --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index de44dbd..32e744a 100644 --- a/environment.yml +++ b/environment.yml @@ -43,4 +43,4 @@ dependencies: - nbconvert==7.6.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - - git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling diff --git a/infra-requirements.txt b/infra-requirements.txt index 611bfb5..ca42d16 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -24,7 +24,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 +git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From b75c9809e515e291a79c3f3f8314731085a75b3a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 16 Nov 2023 14:40:08 -0800 Subject: [PATCH 137/184] less means more (images built) --- apt.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apt.txt b/apt.txt index bfc2a10..a6c4589 100644 --- a/apt.txt +++ b/apt.txt @@ -1,6 +1,9 @@ # Some linux packages for basic terminal work, particularly # oriented at users new to Unix/cmd line environments. +# installing less as more just isn't enough +less + # Basic unix tools man man-db From 657a9827ac39f3821fc1929f82c3ef8de226704f Mon Sep 17 00:00:00 2001 From: mihranmiroyan Date: Wed, 29 Nov 2023 13:32:35 -0800 Subject: [PATCH 138/184] mlxtend --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 32e744a..9008484 100644 --- a/environment.yml +++ b/environment.yml @@ -33,6 +33,7 @@ dependencies: - statsmodels==0.14.0 - tensorflow-cpu==2.12.1 - sqlalchemy==2.0.16 +- mlxtend==0.23.0 - pip - pip: # - -r infra-requirements.txt From 811d3cc0f55e1f12143b7a738b2ae0a84d9ff0d6 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 21 Dec 2023 15:11:54 -0800 Subject: [PATCH 139/184] testing out archive extension --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 9008484..c1777ea 100644 --- a/environment.yml +++ b/environment.yml @@ -18,6 +18,7 @@ dependencies: - h5netcdf==1.0.2 - ipywidgets==8.0.7 - jupysql==0.8.0 +- jupyter-archive==3.4.0 - jupyterlab==4.0.4 - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 From cf47e3a2b1af0c3dffeffbd162cd7fba4a5fadae Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 21 Dec 2023 15:20:16 -0800 Subject: [PATCH 140/184] might as well bump notebook while were in here --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c1777ea..d19be11 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - jupyterlab_widgets==3.0.8 - jupyter_server==2.7.0 - nbgitpuller==1.1.1 -- notebook==7.0.2 +- notebook==7.0.6 - folium==0.14.0 - h5netcdf==1.0.2 - ipywidgets==8.0.7 From 6a794850c1194a25734a8b9c59bb90c09fbda20d Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Wed, 10 Jan 2024 13:10:32 +0400 Subject: [PATCH 141/184] added packages for econ 148 --- environment.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/environment.yml b/environment.yml index d19be11..48ebacd 100644 --- a/environment.yml +++ b/environment.yml @@ -35,6 +35,10 @@ dependencies: - tensorflow-cpu==2.12.1 - sqlalchemy==2.0.16 - mlxtend==0.23.0 +- geopandas==0.14.2 +- geopy==2.4.1 +- lifelines==0.27.8 +- pycountry==22.3.5 - pip - pip: # - -r infra-requirements.txt @@ -46,3 +50,4 @@ dependencies: - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 From 5768cf523829b23c1ca0d3abe5aaeed3efb7b60d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:09:37 -0800 Subject: [PATCH 142/184] jupyterlab 4.0.10 redux --- environment.yml | 3 +-- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 48ebacd..2dfee2c 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ dependencies: - python==3.11.0 - git==2.39.1 - jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.4 +- jupyterlab==4.0.10 - jupyterlab-favorites==3.0.0 - jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 @@ -19,7 +19,6 @@ dependencies: - ipywidgets==8.0.7 - jupysql==0.8.0 - jupyter-archive==3.4.0 -- jupyterlab==4.0.4 - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 diff --git a/infra-requirements.txt b/infra-requirements.txt index ca42d16..6493f7d 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 -jupyterlab==3.4.5 +jupyterlab==4.0.10 retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 From 24b48e9596366ab47e197982f8c75f8e096a6f0b Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:20:15 -0800 Subject: [PATCH 143/184] remove retrolab --- infra-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6493f7d..664bf97 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,7 +10,6 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 jupyterlab==4.0.10 -retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile From 754bbfe9559025308a43c0c12d82c4cb2167a86a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:29:04 -0800 Subject: [PATCH 144/184] bump jupyter-resource-usage --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 664bf97..ffe5c10 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,7 +11,7 @@ notebook==6.4.12 jupyterlab==4.0.10 nbgitpuller==1.1.0 -jupyter-resource-usage==0.6.1 +jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile jupyterhub==4.0.2 appmode==0.8.0 From 57c34dba9ce038b41a36d0563de96590e77111ca Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:41:37 -0800 Subject: [PATCH 145/184] adding jupyter-archive to all hubs --- infra-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index ffe5c10..f36a59a 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -28,3 +28,5 @@ git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handli RISE==5.7.1 # syncthing for dropbox-like functionality jupyter-syncthing-proxy==1.0.3 +# jupyter archival tool for easy user downloads +jupyter-archive==3.4.0 From a740c9bbf29aeb6e14dee5abd790470e0a0f3709 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Sun, 14 Jan 2024 09:48:12 -0800 Subject: [PATCH 146/184] fix datahub build --- infra-requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index f36a59a..ef16a52 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,17 +10,17 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 jupyterlab==4.0.10 -nbgitpuller==1.1.0 +nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile jupyterhub==4.0.2 appmode==0.8.0 -ipywidgets==7.7.2 +ipywidgets==8.0.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. -jupyter-contrib-nbextensions==0.5.1 -jupyter_nbextensions_configurator==0.4.1 +jupyter-contrib-nbextensions==0.7.0 +jupyter_nbextensions_configurator==0.6.3 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 749da833bed1f9d3abe962fe0838b576678b82e9 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Sun, 14 Jan 2024 12:49:20 -0800 Subject: [PATCH 147/184] bump notebook to 7 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index ef16a52..e08dea3 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,7 +8,7 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.12 +notebook==7.0.6 jupyterlab==4.0.10 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 From c7348a0566bdd140c00c0af335c72f87e7cae983 Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Sun, 14 Jan 2024 15:52:31 -0800 Subject: [PATCH 148/184] [edit] package updates draft --- environment.yml | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/environment.yml b/environment.yml index 48ebacd..818e18f 100644 --- a/environment.yml +++ b/environment.yml @@ -7,33 +7,33 @@ dependencies: - python==3.11.0 - git==2.39.1 - jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.4 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 +- jupyterlab==4.0.10 +- jupyterlab-favorites==3.2.1 +- jupyterlab_server==2.25.2 +- jupyterlab_widgets==3.0.9 +- jupyter_server==2.12.4 - nbgitpuller==1.1.1 - notebook==7.0.6 -- folium==0.14.0 -- h5netcdf==1.0.2 +- folium==0.15.1 +- h5netcdf==1.3.0 - ipywidgets==8.0.7 -- jupysql==0.8.0 +- jupysql==0.10.7 - jupyter-archive==3.4.0 -- jupyterlab==4.0.4 -- matplotlib==3.7.1 +- matplotlib==3.8.2 - mdit-py-plugins==0.4.0 -- numpy==1.24.2 -- otter-grader==5.1.3 -- pandas==2.0.2 -- plotly==5.13.1 -- requests==2.28.2 -- scikit-image==0.19.3 -- scikit-learn==1.2.2 -- scipy==1.10.1 +- numpy==1.26.3 +- otter-grader==5.2.2 +- pandas==2.1.4 +- plotly==5.18.0 +- requests==2.31.0 +- scikit-image==0.22.0 +- scikit-learn==1.3.2 +- scipy==1.11.4 - seaborn==0.12.2 -- statsmodels==0.14.0 -- tensorflow-cpu==2.12.1 -- sqlalchemy==2.0.16 +- statsmodels==0.14.1 +- tensorflow==2.15.0 +- tensorflow-cpu==2.15.0 +- sqlalchemy==2.0.25 - mlxtend==0.23.0 - geopandas==0.14.2 - geopy==2.4.1 @@ -46,8 +46,8 @@ dependencies: # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - jupyterhub==4.0.2 - - nbconvert==7.6.0 - - pytest-notebook==0.8.1 + - nbconvert==7.14.1 + - pytest-notebook==0.10.0 - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 From e2bdd661559d825f7f79dcead7d042ca00c7602d Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Sun, 14 Jan 2024 23:48:50 -0800 Subject: [PATCH 149/184] [revert change] revert package change for resolving prev PR --- environment.yml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/environment.yml b/environment.yml index 818e18f..140ae46 100644 --- a/environment.yml +++ b/environment.yml @@ -7,33 +7,33 @@ dependencies: - python==3.11.0 - git==2.39.1 - jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.10 -- jupyterlab-favorites==3.2.1 -- jupyterlab_server==2.25.2 -- jupyterlab_widgets==3.0.9 -- jupyter_server==2.12.4 +- jupyterlab==4.0.4 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 - nbgitpuller==1.1.1 - notebook==7.0.6 -- folium==0.15.1 -- h5netcdf==1.3.0 +- folium==0.14.0 +- h5netcdf==1.0.2 - ipywidgets==8.0.7 -- jupysql==0.10.7 +- jupysql==0.8.0 - jupyter-archive==3.4.0 -- matplotlib==3.8.2 +- jupyterlab==4.0.4 +- matplotlib==3.7.1 - mdit-py-plugins==0.4.0 -- numpy==1.26.3 -- otter-grader==5.2.2 -- pandas==2.1.4 -- plotly==5.18.0 -- requests==2.31.0 -- scikit-image==0.22.0 -- scikit-learn==1.3.2 -- scipy==1.11.4 +- numpy==1.24.2 +- otter-grader==5.1.3 +- pandas==2.0.2 +- plotly==5.13.1 +- requests==2.28.2 +- scikit-image==0.19.3 +- scikit-learn==1.2.2 +- scipy==1.10.1 - seaborn==0.12.2 -- statsmodels==0.14.1 -- tensorflow==2.15.0 -- tensorflow-cpu==2.15.0 -- sqlalchemy==2.0.25 +- statsmodels==0.14.0 +- tensorflow-cpu==2.12.1 +- sqlalchemy==2.0.16 - mlxtend==0.23.0 - geopandas==0.14.2 - geopy==2.4.1 @@ -46,8 +46,8 @@ dependencies: # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - jupyterhub==4.0.2 - - nbconvert==7.14.1 - - pytest-notebook==0.10.0 + - nbconvert==7.6.0 + - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 + - ydata-profiling==4.6.4 \ No newline at end of file From a97c12af72264222fe8a328bf0cf6f7cc2f83fbb Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Tue, 16 Jan 2024 12:02:33 -0800 Subject: [PATCH 150/184] [fixed] add issue comment, revert changes --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 140ae46..49c7233 100644 --- a/environment.yml +++ b/environment.yml @@ -49,5 +49,5 @@ dependencies: - nbconvert==7.6.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 \ No newline at end of file From 453999a360abd4a79001176e4adf9d882dd4704e Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Tue, 16 Jan 2024 12:08:32 -0800 Subject: [PATCH 151/184] [fixed] comment with issue, revert back env --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 7b59081..2dfee2c 100644 --- a/environment.yml +++ b/environment.yml @@ -48,5 +48,5 @@ dependencies: - nbconvert==7.6.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 \ No newline at end of file + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 From 73e37139925c00bda3b664a45c590fe8e2e27341 Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 16 Jan 2024 21:33:29 -0800 Subject: [PATCH 152/184] bumping otter --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2dfee2c..584d6b7 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==5.1.3 +- otter-grader==5.2.3 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 From a6bb336ca20ae1683cfe93190b8eda63294c6adb Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 16 Jan 2024 22:38:41 -0800 Subject: [PATCH 153/184] small commit to rerun checks --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 584d6b7..c906ed7 100644 --- a/environment.yml +++ b/environment.yml @@ -50,3 +50,4 @@ dependencies: - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 + From 92cbd5d4f4d959e1fcd032c0641bbb835ef1ae65 Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Tue, 16 Jan 2024 22:54:32 -0800 Subject: [PATCH 154/184] testing pip installing rather than conda --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c906ed7..6fe5c01 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,6 @@ dependencies: - matplotlib==3.7.1 - mdit-py-plugins==0.4.0 - numpy==1.24.2 -- otter-grader==5.2.3 - pandas==2.0.2 - plotly==5.13.1 - requests==2.28.2 @@ -50,4 +49,5 @@ dependencies: - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 + - otter-grader==5.2.3 From b98e008bf5757c175fd25a1ddbb9867e4970f50a Mon Sep 17 00:00:00 2001 From: Rohan Jha Date: Sat, 27 Jan 2024 13:25:47 -0800 Subject: [PATCH 155/184] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6fe5c01..d1ce4b3 100644 --- a/environment.yml +++ b/environment.yml @@ -49,5 +49,5 @@ dependencies: - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 - - otter-grader==5.2.3 + - otter-grader==5.3.0 From 451a29f1d8916bef8fea1c1aa471b6038bcf0439 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 29 Jan 2024 11:30:56 -0800 Subject: [PATCH 156/184] remove deprecated nbconvert packages --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index e08dea3..0b14278 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -19,8 +19,8 @@ ipywidgets==8.0.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. -jupyter-contrib-nbextensions==0.7.0 -jupyter_nbextensions_configurator==0.6.3 +# jupyter-contrib-nbextensions==0.7.0 +# jupyter_nbextensions_configurator==0.6.3 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 314b19ca7d2a0885796ef5f05b91df70eee9ae1d Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 29 Jan 2024 11:34:57 -0800 Subject: [PATCH 157/184] remove rather than comment out --- infra-requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 0b14278..d029ba5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -18,9 +18,6 @@ appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 -# Enough people like this, let's load it in. -# jupyter-contrib-nbextensions==0.7.0 -# jupyter_nbextensions_configurator==0.6.3 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 2e09c4b729775dde7cbcac2ba2e05ddadd03966e Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Tue, 30 Jan 2024 13:32:26 -0800 Subject: [PATCH 158/184] [add] new packages for HW3 and lecture demos --- environment.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/environment.yml b/environment.yml index d1ce4b3..e7f8d03 100644 --- a/environment.yml +++ b/environment.yml @@ -33,6 +33,13 @@ dependencies: - tensorflow-cpu==2.12.1 - sqlalchemy==2.0.16 - mlxtend==0.23.0 +# Spring 2024 data 100 +- pytorch==2.1.2 +- cpuonly==2.0 +- transformers==4.37.1 +# Spring 2024 table demos +- lxml==5.1.0 +# Spring 2024 Econ 148 Packages - geopandas==0.14.2 - geopy==2.4.1 - lifelines==0.27.8 From b0aa858fefdd1ab3d6431e071189fdf09493f5b0 Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Tue, 30 Jan 2024 13:52:46 -0800 Subject: [PATCH 159/184] [add] pytorch channel --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index e7f8d03..b03c090 100644 --- a/environment.yml +++ b/environment.yml @@ -2,6 +2,7 @@ name: data100-FA23 channels: - conda-forge +- pytorch dependencies: - python==3.11.0 From 460601e49383300b85ac1eecf4fe06043c022b01 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 6 Feb 2024 10:05:17 -0800 Subject: [PATCH 160/184] notebook 7.0.7 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index d029ba5..52a0479 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,7 +8,7 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.6 +notebook==7.0.7 jupyterlab==4.0.10 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 From 815e5428b225bae110eba49003a98f0026786281 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 6 Feb 2024 10:07:49 -0800 Subject: [PATCH 161/184] non infra-reqs notebook to 7.0.7 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index b03c090..8a1e781 100644 --- a/environment.yml +++ b/environment.yml @@ -14,7 +14,7 @@ dependencies: - jupyterlab_widgets==3.0.8 - jupyter_server==2.7.0 - nbgitpuller==1.1.1 -- notebook==7.0.6 +- notebook==7.0.7 - folium==0.14.0 - h5netcdf==1.0.2 - ipywidgets==8.0.7 From 4e2dc55dc0e68e22259f9494d7bc20b11820197a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 6 Feb 2024 10:09:05 -0800 Subject: [PATCH 162/184] jupyterlab via infra-reqs to 4.0.11 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 52a0479..af781f4 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==7.0.7 -jupyterlab==4.0.10 +jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile From c57991025958573db9ceba27a9f9b90d453b7e26 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 6 Feb 2024 10:11:32 -0800 Subject: [PATCH 163/184] jupyterlab via environment.yaml to 4.0.11 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8a1e781..c8bc29e 100644 --- a/environment.yml +++ b/environment.yml @@ -8,7 +8,7 @@ dependencies: - python==3.11.0 - git==2.39.1 - jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.10 +- jupyterlab==4.0.11 - jupyterlab-favorites==3.0.0 - jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 From b5e23f7ccf7fa9ec279f13370b16a8fc377f822c Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Wed, 21 Feb 2024 22:07:41 -0800 Subject: [PATCH 164/184] bump to 5.4.0 otter --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c8bc29e..3ef7527 100644 --- a/environment.yml +++ b/environment.yml @@ -57,5 +57,5 @@ dependencies: - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 - - otter-grader==5.3.0 + - otter-grader==5.4.0 From 10d382d686feffa50e12fece8e7470ba1c1d794e Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Wed, 21 Feb 2024 22:07:41 -0800 Subject: [PATCH 165/184] bump to 5.4.0 otter --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c8bc29e..3ef7527 100644 --- a/environment.yml +++ b/environment.yml @@ -57,5 +57,5 @@ dependencies: - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 - - otter-grader==5.3.0 + - otter-grader==5.4.0 From aca013846ce37be6a50373966d595519e45e3305 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 15 Mar 2024 15:36:36 -0700 Subject: [PATCH 166/184] install nbconvert webpdf and pyppeteer --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 3ef7527..77f19cd 100644 --- a/environment.yml +++ b/environment.yml @@ -52,7 +52,8 @@ dependencies: # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - jupyterhub==4.0.2 - - nbconvert==7.6.0 + - nbconvert[webpdf] + - pyppeteer==2.0.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 281293a067fea3265b056396cf44260e3809e640 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Mon, 18 Mar 2024 13:41:37 -0700 Subject: [PATCH 167/184] Enable webpdf conversion in Data 100 Hub --- environment.yml | 2 +- postBuild | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 postBuild diff --git a/environment.yml b/environment.yml index 77f19cd..f150ed2 100644 --- a/environment.yml +++ b/environment.yml @@ -53,7 +53,7 @@ dependencies: # - jupyter_collaboration==1.0.1 - jupyterhub==4.0.2 - nbconvert[webpdf] - - pyppeteer==2.0.0 + # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling diff --git a/postBuild b/postBuild new file mode 100644 index 0000000..9514de4 --- /dev/null +++ b/postBuild @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +npm install -g playwright +playwright install chromium From 9e72d02789bb190891ca2382fb4dc3c5f46921db Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Wed, 20 Mar 2024 11:23:11 -0700 Subject: [PATCH 168/184] Removing webpdf related changes in Data 100 Hub --- postBuild | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 postBuild diff --git a/postBuild b/postBuild deleted file mode 100644 index 9514de4..0000000 --- a/postBuild +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# installing chromium browser to enable webpdf conversion using nbconvert -export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -npm install -g playwright -playwright install chromium From 9423ed51e95a5dd1ab19a8a612169440a81ada82 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Thu, 21 Mar 2024 15:07:58 -0700 Subject: [PATCH 169/184] Last try: Make webpdf work in containerized env --- postBuild | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 postBuild diff --git a/postBuild b/postBuild new file mode 100644 index 0000000..9514de4 --- /dev/null +++ b/postBuild @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# installing chromium browser to enable webpdf conversion using nbconvert +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +npm install -g playwright +playwright install chromium From 69d834655c1044d31b7ccdebf2e52c3fe0ffd10b Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Sat, 23 Mar 2024 16:39:23 -0700 Subject: [PATCH 170/184] [add] duckdb pip install --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index f150ed2..02e83ca 100644 --- a/environment.yml +++ b/environment.yml @@ -59,4 +59,5 @@ dependencies: - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 - otter-grader==5.4.0 + - duckdb==0.10.1 From f655ba625bad32325949a12fb4ac8858093b48a3 Mon Sep 17 00:00:00 2001 From: Charlie Cheng-Jie Ji Date: Tue, 26 Mar 2024 21:11:44 -0700 Subject: [PATCH 171/184] add duckdb_engine --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 02e83ca..d366f9c 100644 --- a/environment.yml +++ b/environment.yml @@ -60,4 +60,4 @@ dependencies: - ydata-profiling==4.6.4 - otter-grader==5.4.0 - duckdb==0.10.1 - + - duckdb_engine==0.11.2 From 1f3a2fcf13f3d6ebdb0bc95c7c50a40061139786 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Sat, 30 Mar 2024 16:18:22 -0700 Subject: [PATCH 172/184] bump jupyterhub --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index d366f9c..0cccb88 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.0.2 + - jupyterhub==4.1.0 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index af781f4..62504c7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.0.2 +jupyterhub==4.1.0 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 869fcc0f242fd6e04587e43e10ff29b4079015f8 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 1 Apr 2024 14:28:47 -0700 Subject: [PATCH 173/184] more bumping --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 0cccb88..caecddf 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.0 + - jupyterhub==4.1.4 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index 62504c7..d4feb19 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.0 +jupyterhub==4.1.4 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From bf55f591abc64bd2633f086156af161507637214 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 4 Apr 2024 22:27:05 -0700 Subject: [PATCH 174/184] debump jh 4.1.4 --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index caecddf..91f916f 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.4 + - jupyterhub==4.0.1 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index d4feb19..96b9029 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.4 +jupyterhub==4.0.1 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 9d90866ce06cd44323f27cdb8cc8c5fe4382a2eb Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 5 Apr 2024 13:52:42 -0700 Subject: [PATCH 175/184] upgrading is fun --- environment.yml | 4 ++-- infra-requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 91f916f..614cff1 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 - jupyter_server==2.7.0 -- nbgitpuller==1.1.1 +- nbgitpuller==1.2.1 - notebook==7.0.7 - folium==0.14.0 - h5netcdf==1.0.2 @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.0.1 + - jupyterhub==4.1.5 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index 96b9029..b01c9d8 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,10 +10,10 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==7.0.7 jupyterlab==4.0.11 -nbgitpuller==1.2.0 +nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.0.1 +jupyterhub==4.1.5 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 25c0e49f10a3f6f560259f14301f1b2a8194eabb Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 7 May 2024 11:47:53 -0700 Subject: [PATCH 176/184] normalize python installs across all hubs --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 614cff1..2f72be2 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - pytorch dependencies: -- python==3.11.0 +- python==3.11.* - git==2.39.1 - jupyter-resource-usage==1.0.0 - jupyterlab==4.0.11 From b13395d437ec74334bf991a4ffc34af6b5a0ef47 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 11 Jun 2024 13:26:32 -0700 Subject: [PATCH 177/184] adding deps for playwright --- apt.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apt.txt b/apt.txt index a6c4589..3852658 100644 --- a/apt.txt +++ b/apt.txt @@ -48,6 +48,10 @@ lmodern pandoc latexdiff +# for playwright +libnss3 +libnspr4 + # Some useful git utilities use basic Ruby ruby From c0d3276f192a61ac788c9e40605f0a1594566566 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 11 Jun 2024 16:38:58 -0700 Subject: [PATCH 178/184] [DH-305] make webpdf conversion work --- apt.txt | 4 ++++ postBuild | 1 - start | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 start diff --git a/apt.txt b/apt.txt index a6c4589..10e53b0 100644 --- a/apt.txt +++ b/apt.txt @@ -91,3 +91,7 @@ meld # For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979 zip + +# playwright deps https://jira-secure.berkeley.edu/browse/DH-305 +libnss3 +libnspr4 diff --git a/postBuild b/postBuild index 9514de4..1b6bcc9 100644 --- a/postBuild +++ b/postBuild @@ -2,5 +2,4 @@ # installing chromium browser to enable webpdf conversion using nbconvert export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} -npm install -g playwright playwright install chromium diff --git a/start b/start new file mode 100644 index 0000000..c3a978b --- /dev/null +++ b/start @@ -0,0 +1,5 @@ +#!/bin/bash + +# See https://jira-secure.berkeley.edu/browse/DH-305 +export PLAYWRIGHT_BROWSERS_PATH=${CONDA_DIR} +exec "$@" From 7077a20ebfd804b7f8dc91c037a9b7e024cdca6d Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 11 Jun 2024 17:26:33 -0700 Subject: [PATCH 179/184] [DH-305] removing duplicate apt packages --- apt.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apt.txt b/apt.txt index 076314a..10e53b0 100644 --- a/apt.txt +++ b/apt.txt @@ -48,10 +48,6 @@ lmodern pandoc latexdiff -# for playwright -libnss3 -libnspr4 - # Some useful git utilities use basic Ruby ruby From be4be5f583d82fb07a9662a0d986d92eccc6bf9f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 15 Jul 2024 10:46:16 -0700 Subject: [PATCH 180/184] pin jupytext on hubs that need it --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 2f72be2..ffbd747 100644 --- a/environment.yml +++ b/environment.yml @@ -61,3 +61,5 @@ dependencies: - otter-grader==5.4.0 - duckdb==0.10.1 - duckdb_engine==0.11.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 From e23b9ce3e559d6cdeb166ce41197f18d0203c450 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:17:07 -0700 Subject: [PATCH 181/184] bumping jupyterhub --- environment.yml | 2 +- infra-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index ffbd747..3d0da5b 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 + - jupyterhub==4.1.6 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 diff --git a/infra-requirements.txt b/infra-requirements.txt index b01c9d8..0fb0bd9 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 5ebb327ec35fe8cd2b63387e7408dd2bfb080109 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 19 Sep 2024 15:25:45 -0700 Subject: [PATCH 182/184] bumping jupyter core packages, moving some from pip to conda --- environment.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/environment.yml b/environment.yml index 3d0da5b..722417f 100644 --- a/environment.yml +++ b/environment.yml @@ -6,15 +6,20 @@ channels: dependencies: - python==3.11.* -- git==2.39.1 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 +- git==2.46.0 +- gh-scoped-creds==4.1 +- jupyter-resource-usage==1.1.0 +- jupyterhub==4.1.6 +- jupyterlab==4.2.5 - jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 +- jupyterlab-git==0.50.1 +- jupyterlab_server==2.27.3 - jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 +- jupyter_server==2.27.3 +# pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 +- jupytext==1.16.2 - nbgitpuller==1.2.1 -- notebook==7.0.7 +- notebook==7.2.2 - folium==0.14.0 - h5netcdf==1.0.2 - ipywidgets==8.0.7 @@ -47,19 +52,14 @@ dependencies: - pycountry==22.3.5 - pip - pip: - # - -r infra-requirements.txt - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.6 - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 - otter-grader==5.4.0 - duckdb==0.10.1 - duckdb_engine==0.11.2 - # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 - - jupytext==1.16.2 From 3f24c9d58ae7f69c394aef18e2c690451297582e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 19 Sep 2024 15:54:37 -0700 Subject: [PATCH 183/184] fixing some package defs --- environment.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/environment.yml b/environment.yml index 722417f..984afab 100644 --- a/environment.yml +++ b/environment.yml @@ -15,9 +15,8 @@ dependencies: - jupyterlab-git==0.50.1 - jupyterlab_server==2.27.3 - jupyterlab_widgets==3.0.8 -- jupyter_server==2.27.3 -# pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 -- jupytext==1.16.2 +- jupyter_server==2.14.2 +- jupytext==1.16.4 - nbgitpuller==1.2.1 - notebook==7.2.2 - folium==0.14.0 @@ -30,13 +29,13 @@ dependencies: - numpy==1.24.2 - pandas==2.0.2 - plotly==5.13.1 -- requests==2.28.2 +- requests==2.32.2 - scikit-image==0.19.3 - scikit-learn==1.2.2 - scipy==1.10.1 -- seaborn==0.12.2 +- seaborn==0.13.2 - statsmodels==0.14.0 -- tensorflow-cpu==2.12.1 +- tensorflow-cpu==2.17.0 - sqlalchemy==2.0.16 - mlxtend==0.23.0 # Spring 2024 data 100 @@ -55,7 +54,7 @@ dependencies: - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - nbconvert[webpdf] + - nbconvert[webpdf]==7.16.4 # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 3a60e15533a0634aff9b186ca38014d69e05d669 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 20 Sep 2024 14:40:31 -0700 Subject: [PATCH 184/184] debumping a bunch of packages to unbreak things --- environment.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/environment.yml b/environment.yml index 984afab..3d0da5b 100644 --- a/environment.yml +++ b/environment.yml @@ -6,19 +6,15 @@ channels: dependencies: - python==3.11.* -- git==2.46.0 -- gh-scoped-creds==4.1 -- jupyter-resource-usage==1.1.0 -- jupyterhub==4.1.6 -- jupyterlab==4.2.5 +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 - jupyterlab-favorites==3.0.0 -- jupyterlab-git==0.50.1 -- jupyterlab_server==2.27.3 +- jupyterlab_server==2.23.0 - jupyterlab_widgets==3.0.8 -- jupyter_server==2.14.2 -- jupytext==1.16.4 +- jupyter_server==2.7.0 - nbgitpuller==1.2.1 -- notebook==7.2.2 +- notebook==7.0.7 - folium==0.14.0 - h5netcdf==1.0.2 - ipywidgets==8.0.7 @@ -29,13 +25,13 @@ dependencies: - numpy==1.24.2 - pandas==2.0.2 - plotly==5.13.1 -- requests==2.32.2 +- requests==2.28.2 - scikit-image==0.19.3 - scikit-learn==1.2.2 - scipy==1.10.1 -- seaborn==0.13.2 +- seaborn==0.12.2 - statsmodels==0.14.0 -- tensorflow-cpu==2.17.0 +- tensorflow-cpu==2.12.1 - sqlalchemy==2.0.16 - mlxtend==0.23.0 # Spring 2024 data 100 @@ -51,14 +47,19 @@ dependencies: - pycountry==22.3.5 - pip - pip: + # - -r infra-requirements.txt - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - nbconvert[webpdf]==7.16.4 + - jupyterhub==4.1.6 + - nbconvert[webpdf] # - pyppeteer==2.0.0 - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - ydata-profiling==4.6.4 - otter-grader==5.4.0 - duckdb==0.10.1 - duckdb_engine==0.11.2 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2