From d55fb5874023d1dc0e2da54b7e5ad3d33bcd7b90 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 13:44:34 -0700 Subject: [PATCH 1/8] env updates for fall2024; change course number and mem reqs --- deployments/astro/config/common.yaml | 6 +- deployments/astro/image/environment.yml | 116 +++++++++++------------- 2 files changed, 57 insertions(+), 65 deletions(-) diff --git a/deployments/astro/config/common.yaml b/deployments/astro/config/common.yaml index eb8caee98..c97af25a8 100644 --- a/deployments/astro/config/common.yaml +++ b/deployments/astro/config/common.yaml @@ -92,9 +92,9 @@ jupyterhub: custom: group_profiles: # Astro Lab 128, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5596 - course::1532117: # Temporarily grant 8G RAM to all students - mem_limit: 8G - mem_guarantee: 8G + course::1537100: # Temporarily bump down to 4G RAM to all students + mem_limit: 4G + mem_guarantee: 4G admin: extraVolumeMounts: - name: home diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index ce81f1033..353b406a4 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -6,97 +6,89 @@ channels: dependencies: - python=3.11.* -- jupyter-server-proxy==4.2.0 +- jupyter-server-proxy>=4.3 # A linux desktop environment - websockify - -- syncthing==1.18.6 - -- numpy==1.26.* -- matplotlib==3.8.* +- numpy>=2.1.0 +- numexpr>=2.10.0 +- matplotlib==3.9.* - graphviz>=2.50.0 - psycopg2==2.9.* -- pytorch==2.3.0 -- torchvision==0.18.* -- pytorch-lightning==2.2.1 -- pyaudio==0.2.11 -- cython +- pytorch +- torchvision +- pytorch-lightning - cartopy - protobuf -- mkl -- mkl-service +- setuptools>=65.3.0 - mpich - mpi4py -- scikit-learn==1.4.* -- scikit-image==0.19.* -- scipy==1.11.3 +- mkl +- mkl-service +- netcdf4>=1.6 +- scikit-learn==1.5.* +- scikit-image==0.24.* +- scipy==1.14.* +- jax>=0.4.31 +- jaxlib>=0.4.31 +- cython==3.0.11 - pip - pip: - - jupyter-desktop-server==0.1.3 + - syncthing - -r infra-requirements.txt - - tensorflow + - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 - sympy - beautifulsoup4 - tqdm - - h5py - - netcdf4 - - numexpr - corner==2.2.* - - dask>=2021.12.0 - - dask-ml>=2021.11.30 - - distributed>=2021.12.0 - - jaxlib>=0.1.75 - - SQLAlchemy>=1.4.29 - - xarray>=0.20.2 - - nltk>=3.6.7 - - seaborn>=0.11.2 - - bqplot>=0.12.31 - - astroquery>=0.4.5 - - astropy==5.3.3 - - dustmaps>=1.0.9 - - pyvo>=1.2 - - joblib==1.2.0 - - aesara>=2.3.4 - - pymc3>=3.11.4 - - requests>=2.27.1 + - dask>=2024.8.1 + - dask-ml>=2024.4.4 + - distributed>=2024.8.1 + - SQLAlchemy>=2.0.32 + - xarray>=2024.7.0 + - nltk>=3.9.1 + - seaborn>=0.13.2 + - bqplot>=0.12.43 + - astroquery>=0.4.7 + - astropy>=6.1.2 + - dustmaps>=1.0.13 + - pyvo>=1.5.2 + - joblib==1.4.2 + - pymc>=5.16.2 + - requests>=2.32.3 - ipycanvas - - altair>=4.2.0 + - altair>=5.4.0 - vega3>=0.13.0 - - pytz>=2021.3 - - keras - - keras-preprocessing>=1.1.0 - - tensorflow-io-gcs-filesystem>=0.23.0 + - pytz>=2024.1 - lightkurve - - ipyparallel>=8.1.0 - - jupyter-tensorboard>=0.2.0 - - line_profiler>=3.4.0 - - snakeviz>=2.1.1 - - memory_profiler>=0.60.0 - - flask>=2.0.2 - - stsci.tools>=4.0.0 - - gensim>=4.1.2 + - ipyparallel + - line_profiler>=4.1.3 + - snakeviz>=2.2.0 + - memory_profiler>=0.61.0 + - flask>=3.0.3 + - stsci.tools>=4.1.0 + - gensim>=4.3.3 - tweet-preprocessor - pyLDAvis==3.4.1 - - umap-learn>=0.5.2 - - pydot>=1.4.0 - - TPOT>=0.11.7 - - tables>=3.7.0 - - aiohttp==3.8.5 - - jax + - umap-learn>=0.5.6 + - pydot>=3.0.1 + - TPOT>=0.12.2 + - tables>=3.10.1 + - aiohttp>=3.10.4 - watermark - autopep8 - - datashader==0.13.0 - - holoviews==1.14.7 - - hvplot==0.7.3 - vega_datasets - vega - pandas-bokeh - pythreejs - ipywidgets - ipyvolume + - urllib3 + - six + - ipython + - notebook - click - fire - pycodestyle @@ -104,8 +96,8 @@ dependencies: # - nb_black - pycodestyle_magic - twine - - otter-grader==3.1.4 + - otter-grader>=3.1.4 # for notebook exporting - - nbconvert[webpdf]==7.16.4 + - nbconvert>=6.4.0 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 From 1fd2c66a30d7d29d1b7efcd3da048cd2937750ba Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 13:59:59 -0700 Subject: [PATCH 2/8] remove infrastructure install; add a few packages --- deployments/astro/image/environment.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 353b406a4..afcd7c9ea 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -18,7 +18,10 @@ dependencies: - torchvision - pytorch-lightning - cartopy +- cpuonly +- pandas>=2.2.2 - protobuf +- jupysql - setuptools>=65.3.0 - mpich - mpi4py @@ -35,7 +38,6 @@ dependencies: - pip: - syncthing - - -r infra-requirements.txt - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 @@ -98,6 +100,7 @@ dependencies: - twine - otter-grader>=3.1.4 # for notebook exporting - - nbconvert>=6.4.0 + - nbconvert[webpdf] - nb2pdf==0.6.2 - nbpdfexport==0.2.1 + - pytest-notebook==0.8.1 \ No newline at end of file From 556dce1545ec3a3e5d8a59f2b8fcd5d71c21d793 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:07:32 -0700 Subject: [PATCH 3/8] syncthing->jupyter-syncthing-proxy --- deployments/astro/image/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index afcd7c9ea..2ea2f5304 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -37,7 +37,7 @@ dependencies: - pip - pip: - - syncthing + - jupyter-syncthing-proxy==1.0.3 - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 From 7c57a698ef8d1688e295789c1aed669923c56b68 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:20:55 -0700 Subject: [PATCH 4/8] remove some deps --- deployments/astro/image/environment.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 2ea2f5304..e5cb36d61 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -22,7 +22,6 @@ dependencies: - pandas>=2.2.2 - protobuf - jupysql -- setuptools>=65.3.0 - mpich - mpi4py - mkl @@ -37,8 +36,6 @@ dependencies: - pip - pip: - - jupyter-syncthing-proxy==1.0.3 - - jupyter-desktop-server==0.1.3 - opencv-python>=4.5.5.62 - nose2>=0.10.0 - sympy From e6df54e8c4b9312f09b8c4ce24702aef0522d400 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:46:27 -0700 Subject: [PATCH 5/8] add tensorboard and jupyter goodies --- deployments/astro/image/environment.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index e5cb36d61..e6235550d 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -7,6 +7,15 @@ channels: dependencies: - python=3.11.* - jupyter-server-proxy>=4.3 +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- nbgitpuller==1.2.1 +- notebook==7.0.7 # A linux desktop environment - websockify - numpy>=2.1.0 @@ -30,6 +39,7 @@ dependencies: - scikit-learn==1.5.* - scikit-image==0.24.* - scipy==1.14.* +- tensorflow-cpu==2.12.1 - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 @@ -92,6 +102,7 @@ dependencies: - fire - pycodestyle - flake8 + - tdtax>=0.1.6 # - nb_black - pycodestyle_magic - twine @@ -100,4 +111,5 @@ dependencies: - nbconvert[webpdf] - nb2pdf==0.6.2 - nbpdfexport==0.2.1 - - pytest-notebook==0.8.1 \ No newline at end of file + - pytest-notebook==0.8.1 + - jupyter-tensorboard>=0.2.0 \ No newline at end of file From e421e19577acf77648d2ef88b35258f468232b9b Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 14:53:08 -0700 Subject: [PATCH 6/8] proper version tensorflow --- deployments/astro/image/environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index e6235550d..ef3ea1fad 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -39,7 +39,8 @@ dependencies: - scikit-learn==1.5.* - scikit-image==0.24.* - scipy==1.14.* -- tensorflow-cpu==2.12.1 +- tensorflow-cpu==2.17.0 +- tensorflow==2.17.0 - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 From e24d632402155c89f9fae71188aff21cda1f8644 Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 15:11:09 -0700 Subject: [PATCH 7/8] add cudatools; remove some jupyter --- deployments/astro/image/environment.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index ef3ea1fad..3dc843cfc 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -7,15 +7,6 @@ channels: dependencies: - python=3.11.* - jupyter-server-proxy>=4.3 -- git==2.39.1 -- jupyter-resource-usage==1.0.0 -- jupyterlab==4.0.11 -- jupyterlab-favorites==3.0.0 -- jupyterlab_server==2.23.0 -- jupyterlab_widgets==3.0.8 -- jupyter_server==2.7.0 -- nbgitpuller==1.2.1 -- notebook==7.0.7 # A linux desktop environment - websockify - numpy>=2.1.0 @@ -27,7 +18,6 @@ dependencies: - torchvision - pytorch-lightning - cartopy -- cpuonly - pandas>=2.2.2 - protobuf - jupysql @@ -41,6 +31,7 @@ dependencies: - scipy==1.14.* - tensorflow-cpu==2.17.0 - tensorflow==2.17.0 +- cudatoolkit - jax>=0.4.31 - jaxlib>=0.4.31 - cython==3.0.11 From 4169d3ae793d69345615dd54381a0d5adb57480c Mon Sep 17 00:00:00 2001 From: jbloom Date: Mon, 19 Aug 2024 15:20:34 -0700 Subject: [PATCH 8/8] downgrade numpy to < 2.0 to make compat with tf --- deployments/astro/image/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/astro/image/environment.yml b/deployments/astro/image/environment.yml index 3dc843cfc..40287df28 100644 --- a/deployments/astro/image/environment.yml +++ b/deployments/astro/image/environment.yml @@ -9,8 +9,8 @@ dependencies: - jupyter-server-proxy>=4.3 # A linux desktop environment - websockify -- numpy>=2.1.0 -- numexpr>=2.10.0 +- numpy==1.26.4 +- numexpr>=2.8.7 - matplotlib==3.9.* - graphviz>=2.50.0 - psycopg2==2.9.*