Merge pull request #5919 from berkeley-dsep-infra/staging

Merge #5844, #5899, #5901, #5904, #5905, #5909, #5910, #5912, #5916, #5917, #5918
berkeley-dsep-infra · Aug 8, 2024 · e0cc085 · e0cc085
2 parents bac9422 + 12cec7a
commit e0cc085
Show file tree

Hide file tree

Showing 30 changed files with 839 additions and 22 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -325,6 +325,12 @@ jobs:
             hubploy deploy --timeout 30m stat20 hub ${CIRCLE_BRANCH}
           no_output_timeout: 30m
 
+      - run:
+          name: Deploy ugr01
+          command: |
+            hubploy deploy --timeout 30m ugr01 hub ${CIRCLE_BRANCH}
+          no_output_timeout: 30m
+
       - run:
           name: Deploy workshop
           command: |
@@ -626,6 +632,15 @@ workflows:
                 ignore:
                 - staging
                 - prod
+      - hubploy/build-image:
+          deployment: ugr01
+          name: ugr01 image build
+          # Filters can only be per-job? wtf
+          filters:
+              branches:
+                ignore:
+                - staging
+                - prod
 
   deploy:
     jobs:
@@ -799,6 +814,14 @@ workflows:
               branches:
                 only:
                 - staging
+      - hubploy/build-image:
+          deployment: ugr01
+          name: ugr01 image build
+          push: true
+          filters:
+              branches:
+                only:
+                - staging
       # Build images only during the staging deploy. All merges
       # to prod need to go via staging, so prod should *never*
       # use images not built for staging. By enforcing this at the

diff --git a/deployments/data100/config/common.yaml b/deployments/data100/config/common.yaml
@@ -103,11 +103,6 @@ jupyterhub:
         admin: true
         mem_limit: 4G
 
-      # Data 100, Summer 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5862, to be removed by 7/29
-      course::1535115:
-        mem_limit: 4G
-        mem_guarantee: 4G
-
       # Data 100, Spring 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5376
       #course::1531798::group::Admins: # Spring 2024, Data 100 Admins, ensured 4G RAM
       #  mem_limit: 4G

diff --git a/deployments/datahub/images/default/Dockerfile b/deployments/datahub/images/default/Dockerfile
@@ -21,6 +21,9 @@ RUN apt-get -qq update --yes && \
 
 RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER}
 
+# Do not exclude manpages from being installed.
+RUN sed -i -e '/usr.share.man/s/^/#/' /etc/dpkg/dpkg.cfg.d/excludes
+
 # Install all apt packages
 COPY apt.txt /tmp/apt.txt
 RUN apt-get -qq update --yes && \
@@ -30,6 +33,15 @@ RUN apt-get -qq update --yes && \
     apt-get -qq clean && \
     rm -rf /var/lib/apt/lists/*
 
+# From docker-ce-packaging
+# Remove diverted man binary to prevent man-pages being replaced with "minimized" message. See docker/for-linux#639
+RUN if  [ "$(dpkg-divert --truename /usr/bin/man)" = "/usr/bin/man.REAL" ]; then \
+        rm -f /usr/bin/man; \
+        dpkg-divert --quiet --remove --rename /usr/bin/man; \
+    fi
+
+RUN mandb -c
+
 # Create user owned R libs dir
 # This lets users temporarily install packages
 RUN install -d -o ${NB_USER} -g ${NB_USER} ${R_LIBS_USER}

diff --git a/deployments/datahub/images/default/apt.txt b/deployments/datahub/images/default/apt.txt
@@ -78,3 +78,7 @@ libgdal-dev
 libgeos-dev
 libproj-dev
 libmysqlclient-dev
+
+# 2024-08 workshop
+# https://github.com/berkeley-dsep-infra/datahub/issues/5906
+man
diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml
@@ -2,7 +2,7 @@
 # For conda, == is exact constraint, while = is fuzzy constraint.
 # pip uses https://peps.python.org/pep-0440/ which does not have =.
 dependencies:
-- nodejs=16.*
+- nodejs=18.*
 - traitlets=5.9.*
 - pip=22.2.*
 - python=3.11.*
@@ -18,7 +18,7 @@ dependencies:
 # Base scientific packages that other conda packages we install depend on
 # We don't want to have conda packages depend on pip packages if possible
 - numpy=1.26.*
-- matplotlib=3.7.*
+- matplotlib=3.9.*
 - scipy=1.10.0
 - ipympl=0.9.*
 - pandas==2.2.2
@@ -99,9 +99,21 @@ dependencies:
 - obspy==1.4.1
 
 # Install VSCode for ESPM courses - https://github.com/berkeley-dsep-infra/datahub/issues/5716
-- nodejs=16 # code-server requires node < 17
-- jupyter-vscode-proxy==0.5
-- code-server==4.10.1
+- jupyter-vscode-proxy==0.6
+- code-server==4.23.1
+
+# 2024-08 workshop, #5908
+- tree==2.1.3
+- ruff==0.5.6
+
+# Stat 165/265 requires prophet, Spring, 2024
+- prophet==1.1.5
+
+# [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157
+- altair==5.3.0
+- leafmap==0.36.4
+- mystmd==1.3.1
+- jupyterlab-git==0.50.1
 
 - pip:
   # - -r /tmp/infra-requirements.txt
@@ -205,9 +217,6 @@ dependencies:
   # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323
   - jupytext==1.16.2
 
-  # Stat 165/265 requires prophet, Spring, 2024
-  - prophet==1.1.5
-
   # https://github.com/berkeley-dsep-infra/datahub/issues/5497
   - ottr-force-save-labextension==0.1.1
 
@@ -219,8 +228,6 @@ dependencies:
   - rtree==1.2.0
 
   # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157
-  - altair==5.3.0
   - ibis-framework[pandas]==9.2.0
-  - leafmap==0.36.1
-  - mystmd==1.3.0
-  - jupyterlab-git==0.50.1
+
+  # ATTEMPT TO PUT NEW PACKAGES IN THE CONDA LIST ABOVE FIRST, RATHER THAN PIP
diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml
@@ -62,6 +62,10 @@ jupyterhub:
     extraEnv:
       # Unset NotebookApp from hub/values. Necessary for recent lab versions.
       JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp"
+
+      # Specify our app for gh-scoped-creds that has read/write scopes
+      GH_SCOPED_CREDS_CLIENT_ID: Iv23ct7Qx1mAotaIIYx9
+      GH_SCOPED_CREDS_APP_URL: https://github.com/apps/berkeley-datahub-git-access
     defaultUrl: /lab
     profileList:
       - display_name: "Dockerfile image"
@@ -97,9 +101,7 @@ jupyterhub:
       course::1524699::group::all-admins:
         mem_limit: 12288M
         mem_guarantee: 12288M
-        env:
-          GH_SCOPED_CREDS_CLIENT_ID: "Iv23liFjgKFgkSBWph4C"
-          GH_SCOPED_CREDS_APP_URL: "https://github.com/apps/test-gh-scoped-creds"
+
   #
   #    # Example: increase memory for everyone affiliated with a course.
   #

diff --git a/deployments/dev/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml
@@ -8,6 +8,8 @@ dependencies:
 - python==3.11.*
 - git==2.39.1
 - jupyterhub==4.1.5
+- jupyter_app_launcher==0.2.1
+- jupyter-cache==1.0.0
 - jupyter-resource-usage==1.0.0
 - jupyterlab==4.0.11
 - jupyterlab-favorites==3.0.0

diff --git a/deployments/ugr01/config/common.yaml b/deployments/ugr01/config/common.yaml
@@ -0,0 +1,96 @@
+nfsPVC:
+  enabled: true
+  nfs:
+    serverIP: 10.49.181.18
+
+jupyterhub:
+  scheduling:
+    userScheduler:
+      nodeSelector:
+        hub.jupyter.org/pool-name: core-pool-2024-07-07
+  proxy:
+    chp:
+      nodeSelector:
+        hub.jupyter.org/pool-name: core-pool-2024-07-07
+    service:
+      type: LoadBalancer
+
+  hub:
+    nodeSelector:
+      hub.jupyter.org/pool-name: core-pool-2024-07-07
+    config:
+    loadRoles:
+      # datahub staff
+      datahub-staff:
+        description: Enable admin for datahub staff
+        # this role provides permissions to...
+        scopes:
+          - admin-ui
+          - admin:groups
+          - admin:users
+          - admin:servers
+          - read:roles
+          - read:hub
+          - access:servers
+        # this role will be assigned to...
+        groups:
+          - course::1524699::group::all-admins
+  singleuser:
+    extraFiles:
+      # DH-216
+      remove-exporters:
+        mountPath: /etc/jupyter/jupyter_notebook_config.py
+        stringData: |
+          c.QtPDFExporter.enabled = False
+          c.QtPNGExporter.enabled = False
+          c.WebPDFExporter.enabled = False
+    extraEnv:
+      # Unset NotebookApp from hub/values. Necessary for recent lab versions.
+      JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp"
+    nodeSelector:
+      hub.jupyter.org/pool-name: user-ugr01
+    storage:
+      type: static
+      static:
+        pvcName: home-nfs-v3
+        subPath: "{username}"
+    memory:
+      guarantee: 512M
+      limit: 1G
+
+  #custom:
+  #  group_profiles:
+  #
+  #    # Example: increase memory for everyone affiliated with a course.
+  #
+  #    # Name of Class 100, Fall '22; requested in #98765
+  #    course::123456:
+  #      mem_limit: 4096M
+  #      mem_guarantee: 2048M
+  #
+  #    # Example: grant admin rights to course staff.
+  #    # Enrollment types returned by the Canvas API are `teacher`,
+  #    # `student`, `ta`, `observer`, and `designer`.
+  #    # https://canvas.instructure.com/doc/api/enrollments.html
+  #
+  #    # Some other class 200, Spring '23; requested in #98776
+  #    course::234567::enrollment_type::teacher:
+  #      mem_limit: 2096M
+  #      mem_guarantee: 2048M
+  #    course::234567::enrollment_type::ta:
+  #      mem_limit: 2096M
+  #      mem_guarantee: 2048M
+  #
+  #
+  #    # Example: a fully specified CanvasOAuthenticator group name.
+  #    # This could be useful for temporary resource bumps where the
+  #    # instructor could add people to groups in the bCourses UI. This
+  #    # would benefit from the ability to read resource bumps from
+  #    # jupyterhub's properties. (attributes in the ORM)
+  #
+  #    # Name of Class 100, Fall '22; requested in #98770
+  #    course::123456::group::lab4-bigdata:
+  #      - mountPath: /home/rstudio/.ssh
+  #        name: home
+  #        subPath: _some_directory/_ssh
+  #        readOnly: true
diff --git a/deployments/ugr01/config/filestore/squash-flags.json b/deployments/ugr01/config/filestore/squash-flags.json
@@ -0,0 +1,16 @@
+{
+"--file-share":
+  {
+    "name": "shares",
+    "capacity": "desired-capacity",
+    "nfs-export-options": [
+      {
+        "access-mode": "READ_WRITE",
+        "ip-ranges": ["10.0.0.0/8"],
+        "squash-mode": "ROOT_SQUASH",
+        "anon_uid": 1000,
+        "anon_gid": 1000
+      }
+    ],
+  }
+}
diff --git a/deployments/ugr01/config/prod.yaml b/deployments/ugr01/config/prod.yaml
@@ -0,0 +1,18 @@
+nfsPVC:
+  nfs:
+    shareName: shares/ugr01/prod
+
+jupyterhub:
+  ingress:
+    enabled: true
+    hosts:
+      - ugr01.datahub.berkeley.edu
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - ugr01.datahub.berkeley.edu
+  hub:
+    db:
+      pvc:
+        # This also holds logs
+        storage: 4Gi
diff --git a/deployments/ugr01/config/staging.yaml b/deployments/ugr01/config/staging.yaml
@@ -0,0 +1,22 @@
+nfsPVC:
+  nfs:
+    shareName: shares/ugr01/staging
+
+jupyterhub:
+  scheduling:
+    userScheduler:
+      replicas: 1
+  prePuller:
+    continuous:
+      enabled: false
+  proxy:
+    service:
+      loadBalancerIP: 34.172.42.174
+  ingress:
+    enabled: true
+    hosts:
+      - ugr01-staging.datahub.berkeley.edu
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - ugr01-staging.datahub.berkeley.edu
diff --git a/deployments/ugr01/hubploy.yaml b/deployments/ugr01/hubploy.yaml
@@ -0,0 +1,19 @@
+images:
+  images:
+    - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ugr01-user-image
+      path: image/
+      repo2docker:
+        base_image: docker.io/library/buildpack-deps:jammy
+  registry:
+    provider: gcloud
+    gcloud:
+      project: ucb-datahub-2018
+      service_key: gcr-key.json
+
+cluster:
+  provider: gcloud
+  gcloud:
+    project: ucb-datahub-2018
+    service_key: gke-key.json
+    cluster: ugresearch-cluster
+    zone: us-central1