From 6cc7851e51913444a5053e649c81c73891f76d72 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 7 Jun 2024 15:36:48 -0700 Subject: [PATCH 1/4] Set up dev environment for discovery interns --- deployments/dev/config/common.yaml | 16 +++- .../dev/images/secondary/environment.yml | 82 +++++++++++++++---- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml index 90cb67801..7c406f742 100644 --- a/deployments/dev/config/common.yaml +++ b/deployments/dev/config/common.yaml @@ -43,6 +43,7 @@ jupyterhub: # this role will be assigned to... groups: - course::1524699::group::all-admins + ## Course NUM, Spring 2024, #xyz #course-staff-N: # description: Enable course staff to view and access servers. @@ -66,8 +67,8 @@ jupyterhub: - display_name: "Dockerfile image" description: "This is the original dev image." default: true - - display_name: "repo2docker image" - description: "A newer repo2docker-based image with similar components as the primary." + - display_name: "Dashboard image" + description: "A newer repo2docker-based image with a primary focus of building dashboards." kubespawner_override: image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary:1a64e0a - display_name: "1524699: DataHub Infrastructure" @@ -113,7 +114,16 @@ jupyterhub: # admin: true # mem_limit: 2096M # mem_guarantee: 2048M - # + + ## Datahub Discovery Program, Summer 2024, #DH-303 + course::1524699::group::datahub-discovery-su24-interns: + mem_limit: 12288M #12 GB RAM should be enough for parsing data and generating dashboards + mem_guarantee: 12288M + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/discovery-su24-dataset + subPath: _shared/course/discovery-su24-dataset + readOnly: true # # # Example: a fully specified CanvasOAuthenticator group name. # # This could be useful for temporary resource bumps where the diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml index 0133f62fe..774816f2a 100644 --- a/deployments/dev/images/secondary/environment.yml +++ b/deployments/dev/images/secondary/environment.yml @@ -1,20 +1,68 @@ -dependencies: -- python=3.11.* -- pip=23.2.* -- jupyter-server-proxy==4.1.2 -- jupyter-rsession-proxy==2.2.0 -- jupyterlab-myst==2.0.2 -- syncthing==1.25.0 -- pyppeteer==1.0.2 - -# for nbconvert -- pandoc==3.1.3 +name: dashboard-image -# for jupyter-tree-download -- zip==3.0 +channels: +- conda-forge +- pytorch -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* +dependencies: +- python==3.11.* +- git==2.39.1 +- jupyter-resource-usage==1.0.0 +- jupyterlab==4.0.11 +- jupyterlab-favorites==3.0.0 +- jupyterlab_server==2.23.0 +- jupyterlab_widgets==3.0.8 +- jupyter_server==2.7.0 +- nbgitpuller==1.2.1 +- notebook==7.0.7 +- folium==0.14.0 +- h5netcdf==1.0.2 +- ipywidgets==8.0.7 +- jupysql==0.8.0 +- jupyter-archive==3.4.0 +- matplotlib==3.7.1 +- mdit-py-plugins==0.4.0 +- numpy==1.24.2 +- pandas==2.0.2 +- plotly==5.13.1 +- requests==2.28.2 +- scikit-image==0.19.3 +- scikit-learn==1.2.2 +- scipy==1.10.1 +- seaborn==0.12.2 +- statsmodels==0.14.0 +- tensorflow-cpu==2.12.1 +- sqlalchemy==2.0.16 +- mlxtend==0.23.0 +# Spring 2024 data 100 +- pytorch==2.1.2 +- cpuonly==2.0 +- transformers==4.37.1 +# Spring 2024 table demos +- lxml==5.1.0 +# Spring 2024 Econ 148 Packages +- geopandas==0.14.2 +- geopy==2.4.1 +- lifelines==0.27.8 +- pycountry==22.3.5 +# Install voila for generating dashboards +- voila==0.5.7 +# Install myst for generating dashboards +- mystmd==1.2.5 +- jupyterlab-myst==2.4.2 +- pip - pip: - - -r infra-requirements.txt - - jupyter-shiny-proxy==1.1 + # - -r infra-requirements.txt + - ipywidgets==8.0.7 + # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) + # - jupyter_collaboration==1.0.1 + - jupyterhub==4.1.5 + - nbconvert[webpdf] + # - pyppeteer==2.0.0 + - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 + - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 + - otter-grader==5.4.0 + - duckdb==0.10.1 + - duckdb_engine==0.11.2 \ No newline at end of file From e68c18e1536ca96dd3c848b3c8a2a43f41c3afdc Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 7 Jun 2024 16:03:36 -0700 Subject: [PATCH 2/4] Mimic the _shared directory created in filestore --- deployments/dev/config/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml index 7c406f742..21a59d623 100644 --- a/deployments/dev/config/common.yaml +++ b/deployments/dev/config/common.yaml @@ -122,7 +122,7 @@ jupyterhub: extraVolumeMounts: - name: home mountPath: /home/jovyan/discovery-su24-dataset - subPath: _shared/course/discovery-su24-dataset + subPath: _shared/discovery-su24-dataset readOnly: true # # # Example: a fully specified CanvasOAuthenticator group name. From 50f578166f800a606a6d291e3b2386314c5f44d4 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 7 Jun 2024 19:10:38 -0700 Subject: [PATCH 3/4] moving few packages to conda block --- .../dev/images/secondary/environment.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml index 774816f2a..9ada2bbbd 100644 --- a/deployments/dev/images/secondary/environment.yml +++ b/deployments/dev/images/secondary/environment.yml @@ -49,20 +49,21 @@ dependencies: - voila==0.5.7 # Install myst for generating dashboards - mystmd==1.2.5 -- jupyterlab-myst==2.4.2 +- jupyterlab-myst==2.4.2 +- ipywidgets==8.1.3 +- pytest-notebook==0.10.0 +- gh-scoped-creds==4.1 +- ydata-profiling==4.8.3 +- otter-grader==5.5.0 +- python-duckdb==1.0.0 +- jupyterhub==5.0.0 + - pip - pip: # - -r infra-requirements.txt - - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 - - jupyterhub==4.1.5 - nbconvert[webpdf] # - pyppeteer==2.0.0 - - pytest-notebook==0.8.1 - - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling - - ydata-profiling==4.6.4 - - otter-grader==5.4.0 - - duckdb==0.10.1 - duckdb_engine==0.11.2 \ No newline at end of file From 36280727a9f2dc1e9e240464b56c2a1d6fc71a7e Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 7 Jun 2024 19:30:24 -0700 Subject: [PATCH 4/4] Reverting changes --- .../dev/images/secondary/environment.yml | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml index 9ada2bbbd..c1483db3e 100644 --- a/deployments/dev/images/secondary/environment.yml +++ b/deployments/dev/images/secondary/environment.yml @@ -50,20 +50,27 @@ dependencies: # Install myst for generating dashboards - mystmd==1.2.5 - jupyterlab-myst==2.4.2 -- ipywidgets==8.1.3 -- pytest-notebook==0.10.0 -- gh-scoped-creds==4.1 -- ydata-profiling==4.8.3 -- otter-grader==5.5.0 -- python-duckdb==1.0.0 -- jupyterhub==5.0.0 +#- ipywidgets==8.1.3 +#- pytest-notebook==0.10.0 +#- gh-scoped-creds==4.1 +#- ydata-profiling==4.8.3 +#- otter-grader==5.5.0 +#- python-duckdb==1.0.0 +#- jupyterhub==5.0.0 - pip - pip: # - -r infra-requirements.txt + - ipywidgets==8.0.7 # disable until fixed (probably this: https://github.com/jupyterlab/jupyter-collaboration/issues/162) # - jupyter_collaboration==1.0.1 + - jupyterhub==4.1.5 - nbconvert[webpdf] # - pyppeteer==2.0.0 + - pytest-notebook==0.8.1 + - gh-scoped-creds==4.1 - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling + - ydata-profiling==4.6.4 + - otter-grader==5.4.0 + - duckdb==0.10.1 - duckdb_engine==0.11.2 \ No newline at end of file