Merge pull request #5788 from berkeley-dsep-infra/staging

Merge 5782, 5783, 5785 and 5786 to prod!
berkeley-dsep-infra · Jun 11, 2024 · a3afd21 · a3afd21
2 parents d441468 + c3d17f2
commit a3afd21
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 22 deletions.
diff --git a/deployments/a11y/config/common.yaml b/deployments/a11y/config/common.yaml
@@ -33,6 +33,18 @@ jupyterhub:
         # this role will be assigned to...
         groups:
           - course::1524699::group::all-admins
+      # For folks doing work related to a11y extension
+      course-staff-1535449:
+        description: Enable olks doing work related to a11y checker extension to view and access servers.
+        # this role provides permissions to...
+        scopes:
+          - admin-ui
+          - list:users!group=course::1535449
+          - admin:servers!group=course::1535449
+          - access:servers!group=course::1535449
+        # this role will be assigned to...
+        groups:
+         - course::1535449::group::admin
   singleuser:
     extraFiles:
       # DH-216

diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml
@@ -43,6 +43,7 @@ jupyterhub:
         # this role will be assigned to...
         groups:
           - course::1524699::group::all-admins
+
       ## Course NUM, Spring 2024, #xyz
       #course-staff-N:
       #  description: Enable course staff to view and access servers.
@@ -66,8 +67,8 @@ jupyterhub:
       - display_name: "Dockerfile image"
         description: "This is the original dev image."
         default: true
-      - display_name: "repo2docker image"
-        description: "A newer repo2docker-based image with similar components as the primary."
+      - display_name: "Dashboard image"
+        description: "A newer repo2docker-based image with a primary focus of building dashboards."
         kubespawner_override:
           image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary:1a64e0a
       - display_name: "1524699: DataHub Infrastructure"
@@ -79,7 +80,7 @@ jupyterhub:
               mountPath: /home/jovyan
               subPath: "{username}/1524699"
     nodeSelector:
-      hub.jupyter.org/pool-name: small-courses-pool
+      hub.jupyter.org/pool-name: dev-pool
     storage:
       type: static
       static:
@@ -89,8 +90,8 @@ jupyterhub:
       guarantee: 512M
       limit: 1G
 
-  #custom:
-  #  group_profiles:
+  custom:
+    group_profiles:
   #
   #    # Example: increase memory for everyone affiliated with a course.
   #
@@ -113,7 +114,16 @@ jupyterhub:
   #      admin: true
   #      mem_limit: 2096M
   #      mem_guarantee: 2048M
-  #
+
+      ## Datahub Discovery Program, Summer 2024, #DH-303
+      course::1524699::group::datahub-discovery-su24-interns:
+        mem_limit: 12288M #12 GB RAM should be enough for parsing data and generating dashboards
+        mem_guarantee: 12288M
+        extraVolumeMounts:
+          - name: home
+            mountPath: /home/jovyan/discovery-su24-dataset
+            subPath: _shared/discovery-su24-dataset
+            readOnly: true
   #
   #    # Example: a fully specified CanvasOAuthenticator group name.
   #    # This could be useful for temporary resource bumps where the

diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml
@@ -1,20 +1,76 @@
-dependencies:
-- python=3.11.*
-- pip=23.2.*
-- jupyter-server-proxy==4.1.2
-- jupyter-rsession-proxy==2.2.0
-- jupyterlab-myst==2.0.2
-- syncthing==1.25.0
-- pyppeteer==1.0.2
+name: dashboard-image
 
-# for nbconvert
-- pandoc==3.1.3
+channels:
+- conda-forge
+- pytorch
 
-# for jupyter-tree-download
-- zip==3.0
+dependencies:
+- python==3.11.*
+- git==2.39.1
+- jupyter-resource-usage==1.0.0
+- jupyterlab==4.0.11
+- jupyterlab-favorites==3.0.0
+- jupyterlab_server==2.23.0
+- jupyterlab_widgets==3.0.8
+- jupyter_server==2.7.0
+- nbgitpuller==1.2.1
+- notebook==7.0.7
+- folium==0.14.0
+- h5netcdf==1.0.2
+- ipywidgets==8.0.7
+- jupysql==0.8.0
+- jupyter-archive==3.4.0
+- matplotlib==3.7.1
+- mdit-py-plugins==0.4.0
+- numpy==1.24.2
+- pandas==2.0.2
+- plotly==5.13.1
+- requests==2.28.2
+- scikit-image==0.19.3
+- scikit-learn==1.2.2
+- scipy==1.10.1
+- seaborn==0.12.2
+- statsmodels==0.14.0
+- tensorflow-cpu==2.12.1
+- sqlalchemy==2.0.16
+- mlxtend==0.23.0
+# Spring 2024 data 100
+- pytorch==2.1.2
+- cpuonly==2.0
+- transformers==4.37.1
+# Spring 2024 table demos
+- lxml==5.1.0
+# Spring 2024 Econ 148 Packages
+- geopandas==0.14.2
+- geopy==2.4.1
+- lifelines==0.27.8
+- pycountry==22.3.5
+# Install voila for generating dashboards
+- voila==0.5.7
+# Install myst for generating dashboards
+- mystmd==1.2.5
+- jupyterlab-myst==2.4.2
+#- ipywidgets==8.1.3
+#- pytest-notebook==0.10.0
+#- gh-scoped-creds==4.1
+#- ydata-profiling==4.8.3
+#- otter-grader==5.5.0
+#- python-duckdb==1.0.0
+#- jupyterhub==5.0.0
 
-# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048
-- traitlets=5.9.*
+- pip
 - pip:
-  - -r infra-requirements.txt
-  - jupyter-shiny-proxy==1.1
+  # - -r infra-requirements.txt
+  - ipywidgets==8.0.7
+  # disable until fixed (probably this:  https://github.com/jupyterlab/jupyter-collaboration/issues/162)
+  # - jupyter_collaboration==1.0.1
+  - jupyterhub==4.1.5
+  - nbconvert[webpdf]
+  #  - pyppeteer==2.0.0
+  - pytest-notebook==0.8.1
+  - gh-scoped-creds==4.1
+  - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
+  - ydata-profiling==4.6.4
+  - otter-grader==5.4.0
+  - duckdb==0.10.1
+  - duckdb_engine==0.11.2
diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml
@@ -135,6 +135,14 @@ nodePools:
         # Some value slightly lower than allocatable RAM on the nodepool
         memory: 60929654784
     replicas: 1
+  dev:
+    nodeSelector:
+      hub.jupyter.org/pool-name: dev-pool
+    resources:
+      requests:
+        # Some value slightly lower than allocatable RAM on the nodepool
+        memory: 29247442944
+    replicas: 0
   dlab:
     nodeSelector:
       hub.jupyter.org/pool-name: dlab-pool