Merge pull request #5118 from ryanlovett/datahub-integrate-reqs

Pull {infra-,}requirements into environment.yml.
berkeley-dsep-infra · Oct 10, 2023 · 85783be · 85783be
2 parents 9289eef + 3c54c9c
commit 85783be
Show file tree

Hide file tree

Showing 3 changed files with 216 additions and 17 deletions.
diff --git a/deployments/datahub/images/default/Dockerfile b/deployments/datahub/images/default/Dockerfile
@@ -128,17 +128,17 @@ RUN /tmp/install-mambaforge.bash
 # Install conda environment as our user
 USER ${NB_USER}
 
+COPY infra-requirements.txt /tmp/infra-requirements.txt
 COPY environment.yml /tmp/environment.yml
 
-RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && mamba clean -afy
+RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \
+    mamba clean -afy
 
-COPY infra-requirements.txt /tmp/infra-requirements.txt
-RUN pip install --no-cache -r /tmp/infra-requirements.txt
 RUN jupyter contrib nbextensions install --sys-prefix --symlink && \
     jupyter nbextensions_configurator enable --sys-prefix
 
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache -r /tmp/requirements.txt
+# Used by MCB32, but incompatible with ipywidgets 8.x
+RUN jupyter nbextension enable --py --sys-prefix qgrid
 
 # Set up nbpdf dependencies
 ENV PYPPETEER_HOME ${CONDA_DIR}
@@ -161,10 +161,6 @@ RUN /usr/local/sbin/connector-text.bash
 #COPY connectors/2021-fall-phys-188-288.bash /usr/local/sbin/
 #RUN /usr/local/sbin/2021-fall-phys-188-288.bash
 
-# Used by MCB32, but incompatible with ipywidgets 8.x
-RUN pip install --no-cache qgrid==1.3.1
-RUN jupyter nbextension enable --py --sys-prefix qgrid
-
 # clear out /tmp
 USER root
 RUN rm -rf /tmp/*

diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml
@@ -15,7 +15,7 @@ dependencies:
 
 # Base scientific packages that other conda packages we install depend on
 # We don't want to have conda packages depend on pip packages if possible
-- numpy=1.21.*
+- numpy=1.22.*
 - matplotlib=3.7.*
 - scipy=1.10.0
 - ipympl=0.9.*
@@ -46,13 +46,13 @@ dependencies:
 # EPS88, data100
 # https://github.com/berkeley-dsep-infra/datahub/issues/1796
 # https://github.com/berkeley-dsep-infra/datahub/issues/2824
-- shapely=1.8.*
-- cartopy=0.21.*
+- shapely==1.8.0
+- cartopy==0.22.0
 
 # ls 88-3; neuro
 - pillow=8.3.*
 
-# music 30, spring 2022, #3180
+# music 30, spring 2022, https://github.com/berkeley-dsep-infra/datahub/issues/3180
 - pysoundfile==0.10.3.post1
 
 # ds100, spring 2022, #3208
@@ -66,8 +66,8 @@ dependencies:
 - gdown==4.4.0
 
 # LBL summer research internship in materials informatics. Jupyter Book curriculum at https://enze-chen.github.io/mi-book/
-- pymatgen=2022.0.11
-- matminer=0.7.4
+- pymatgen==2023.10.4
+- matminer=0.9.0
 
 # data101, Fall 2022, https://github.com/berkeley-dsep-infra/datahub/issues/3646
 - pgspecial=2.0.1
@@ -91,10 +91,213 @@ dependencies:
 # error converting to PDF https://github.com/berkeley-dsep-infra/datahub/issues/5062
 - pyppeteer==1.0.2
 
+# data8; foundation
+- datascience==0.17.6
+
+- jupyter-server-proxy==3.2.1
+- jupyter-rsession-proxy==2.0.1
+
+- folium==0.12.1.post1
+
+# cogsci88;
+# disabled 2023-10-07 due to unresolved dependencies
+#- ggplot==0.11.5
+
+# cogsci131; spring 2018
+- nose==1.3.7
+
+# modules
+- beautifulsoup4==4.9.3
+
+# PS88 https://github.com/berkeley-dsep-infra/datahub/issues/2925
+- linearmodels==4.27
+
+# JupyterLab pypi extensions
+- jupyterlab-geojson==3.1.2
+
+# INDENG 142 Spring 2021 (future semesters as well) - https://github.com/berkeley-dsep-infra/datahub/issues/2314
+- fancyimpute==0.7.0
+
+# ESPM 167 - https://github.com/berkeley-dsep-infra/datahub/issues/2278
+- contextily==1.1.0
+
+# EPS 256, https://github.com/berkeley-dsep-infra/datahub/issues/1775
+- obspy==1.3.0
+
+- otter-grader==4.2.0
+
+- gh-scoped-creds==4.1
+
 - pip:
+  - -r /tmp/infra-requirements.txt
   # Econ 148, Spring 2023 https://github.com/berkeley-dsep-infra/datahub/issues/4093
   - pycountry-convert==0.7.2
-  - otter-grader==4.2.0
-  - gh-scoped-creds==4.1
   # error converting to PDF https://github.com/berkeley-dsep-infra/datahub/issues/5062
   - nbconvert[webpdf]
+  #
+  # Former contents of requirements.txt
+  # Some things were moved up to conda above.
+  #
+  # Until https://github.com/okpy/ok-client/pull/473 is merged
+  - git+https://github.com/yuvipanda/ok-client@6961d778741fe61911be4d00beff9bd8afc1edf7
+  #
+  - jupyter-shiny-proxy==1.1
+  #
+  # nb2pdf==0.6.2 commented out by sknapp 06.10.2023 to unblock https://github.com/berkeley-dsep-infra/datahub/issues/5062
+  #
+  # ls 88-3; neuro
+  - lxml==4.9.1
+  - tqdm==4.62.1
+  - mne==0.23.0
+  - nibabel==3.2.1
+  - h5py==3.7.0
+  - numexpr==2.7.3
+  - openpyxl==3.0.7
+  - nilearn==0.8.0
+
+  # data100, geog88
+  # https://github.com/berkeley-dsep-infra/datahub/issues/2838
+  - geopandas==0.10.*
+  - geopy==2.2.*
+  - pysal==2.5.*
+  - rtree==0.9.*
+  - netcdf4==1.6.*
+  - mplleaflet==0.0.5
+  # phys 151;
+  - emcee==3.1.0
+  - daft==0.1.2
+  - corner==2.2.1
+  #
+  # ce88;
+  - pymdptoolbox==4.0-b3
+  #
+  # data-x; DL
+  - tensorflow==2.11.1
+  - scikit-image==0.19.*
+  - tables==3.7.0
+  - opencv-python==4.6.0.66
+
+  # astr 128/256; spring 2021
+  - astroquery==0.4.6
+  - astropy==5.1
+  - dustmaps==1.0.9
+  - george==0.4.0
+  - exoplanet==0.5.2
+  - torch==1.13.1
+  - torchvision==0.14.1
+  - pyvo==1.1
+  - joblib==1.3.1
+  - theano-pymc==1.1.2
+  - pymc3==3.11.2
+
+  # eep 153; spring 2019
+  - requests==2.31.0
+  - Pint==0.17
+  # Google spreadsheets, Eric Van Dusen / Keeley Takimoto / Modules
+  - gspread-pandas==2.3.0
+  - gspread==4.0.1
+
+  # eps 109; fall 2019
+  - ffmpeg-python==0.2.0
+
+  # data 88e; spring 2021
+  - plotly==5.2.1
+  - mpmath==1.3.0
+  # sympy==1.6.2
+  - chart-studio==1.1.0
+  - csaps==1.1.0
+  - nbforms==0.5.1
+
+  # issue #875, global 150Q/pacs 190 - fall 2019
+  - wordcloud==1.8.1
+
+  # issue #929, SW 282 - fall 2019
+  - pyreadstat==1.1.2
+
+  # issue 954, EPS24 - fall 2019
+  - xarray==0.19.0
+
+  # issue 1001, Physics 188/288 - fall 2019
+  - umap-learn==0.5.1
+  - hdbscan==0.8.31
+
+  # espm 125/bio 105; fall 2019
+  # see https://github.com/berkeley-dsep-infra/datahub/issues/1796
+  - bitarray==2.3.0
+  - nlmpy==1.0.1
+
+  # physics 188/288 fall, 2019
+  - getdist==1.3.1
+  - tensorflow-hub==0.12.0
+  - tensorflow-probability==0.13.0
+
+  # cs16A/B, spring 2020
+  - lcapy==0.96
+
+  # ds198 mch infodemiology, fall 2020/spring 2021
+  # google apis
+  - google-api-python-client==2.15.0
+  - google-auth-httplib2==0.1.0
+  - google-auth-oauthlib==0.4.5
+  - google==3.0.0
+
+  # issue 1847; LS 22 spring 2021
+  - graphviz==0.17
+
+  # issue #1903, data h195A fall 2020
+  - habanero==0.7.4
+
+  # https://github.com/berkeley-dsep-infra/datahub/issues/1981
+  - ipycanvas==0.9.0
+
+  # data100 scientific packages
+  - ray==1.13.*
+  - xlrd==2.0.1
+
+  # data100 visualization
+  - colorlover==0.3.0
+  - cufflinks==0.17.3
+  - altair==4.1.0
+
+  # data100 tools to access things
+  - tweepy==3.10.0
+  - pytz==2021.1
+  - psycopg2==2.9.1
+
+  # data100 teaching
+  - jassign==0.0.7
+  - dsassign==0.0.8
+
+  # data102
+  - dask==2021.10.0
+  - distributed==2021.10.0
+  - keras-applications==1.0.8
+  - keras-preprocessing==1.1.2
+  - keras==2.11.0
+  - keras-vis==0.4.1
+  - plotly-express==0.4.1
+  - cytoolz==0.11.0
+  - pyro-ppl==1.7.0
+  - lime==0.2.0.1
+  - shap==0.39.0
+
+  # prob140 2021 Spring
+  - prob140==0.4.1.5
+  - sympy==1.8
+
+  # IB 105 / ESPM 125, Fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2696
+  - geonomics==1.3.0
+
+  # data100, fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2767
+  - ipython-sql==0.4.0
+
+  # data100, fall 2021 - https://github.com/berkeley-dsep-infra/datahub/issues/2875
+  - metpy==1.1.0
+  - pooch==1.5.2
+
+  # https://github.com/berkeley-dsep-infra/datahub/issues/2950
+  # Needed to work with a new enough version of httplib2
+  - httplib2>=0.20.2
+
+  # Used by MCB32, but incompatible with ipywidgets 8.x
+  - qgrid==1.3.1
diff --git a/...s/datahub/images/default/requirements.txt → .../images/default/requirements.txt.disabled b/...s/datahub/images/default/requirements.txt → .../images/default/requirements.txt.disabled