Merge pull request #5844 from gmerritt/ugrgke

UGR GKE as GCP Deployments; minor docs edit

.circleci/config.yml

@@ -325,6 +325,12 @@ jobs:
             hubploy deploy --timeout 30m stat20 hub ${CIRCLE_BRANCH}
           no_output_timeout: 30m
 
+      - run:
+          name: Deploy ugr01
+          command: |
+            hubploy deploy --timeout 30m ugr01 hub ${CIRCLE_BRANCH}
+          no_output_timeout: 30m
+
       - run:
           name: Deploy workshop
           command: |
@@ -626,6 +632,15 @@ workflows:
                 ignore:
                 - staging
                 - prod
+      - hubploy/build-image:
+          deployment: ugr01
+          name: ugr01 image build
+          # Filters can only be per-job? wtf
+          filters:
+              branches:
+                ignore:
+                - staging
+                - prod
 
   deploy:
     jobs:
@@ -799,6 +814,14 @@ workflows:
               branches:
                 only:
                 - staging
+      - hubploy/build-image:
+          deployment: ugr01
+          name: ugr01 image build
+          push: true
+          filters:
+              branches:
+                only:
+                - staging
       # Build images only during the staging deploy. All merges
       # to prod need to go via staging, so prod should *never*
       # use images not built for staging. By enforcing this at the

deployments/ugr01/config/common.yaml

@@ -0,0 +1,94 @@
+nfsPVC:
+  enabled: true
+  nfs:
+    serverIP: 10.49.181.18
+
+jupyterhub:
+  scheduling:
+    userScheduler:
+      nodeSelector:
+        hub.jupyter.org/pool-name: core-pool-2024-07-07
+  proxy:
+    chp:
+      nodeSelector:
+        hub.jupyter.org/pool-name: core-pool-2024-07-07
+
+  hub:
+    nodeSelector:
+      hub.jupyter.org/pool-name: core-pool-2024-07-07
+    config:
+    loadRoles:
+      # datahub staff
+      datahub-staff:
+        description: Enable admin for datahub staff
+        # this role provides permissions to...
+        scopes:
+          - admin-ui
+          - admin:groups
+          - admin:users
+          - admin:servers
+          - read:roles
+          - read:hub
+          - access:servers
+        # this role will be assigned to...
+        groups:
+          - course::1524699::group::all-admins
+  singleuser:
+    extraFiles:
+      # DH-216
+      remove-exporters:
+        mountPath: /etc/jupyter/jupyter_notebook_config.py
+        stringData: |
+          c.QtPDFExporter.enabled = False
+          c.QtPNGExporter.enabled = False
+          c.WebPDFExporter.enabled = False
+    extraEnv:
+      # Unset NotebookApp from hub/values. Necessary for recent lab versions.
+      JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp"
+    nodeSelector:
+      hub.jupyter.org/pool-name: user-ugr01
+    storage:
+      type: static
+      static:
+        pvcName: home-nfs-v3
+        subPath: "{username}"
+    memory:
+      guarantee: 512M
+      limit: 1G
+
+  #custom:
+  #  group_profiles:
+  #
+  #    # Example: increase memory for everyone affiliated with a course.
+  #
+  #    # Name of Class 100, Fall '22; requested in #98765
+  #    course::123456:
+  #      mem_limit: 4096M
+  #      mem_guarantee: 2048M
+  #
+  #    # Example: grant admin rights to course staff.
+  #    # Enrollment types returned by the Canvas API are `teacher`,
+  #    # `student`, `ta`, `observer`, and `designer`.
+  #    # https://canvas.instructure.com/doc/api/enrollments.html
+  #
+  #    # Some other class 200, Spring '23; requested in #98776
+  #    course::234567::enrollment_type::teacher:
+  #      mem_limit: 2096M
+  #      mem_guarantee: 2048M
+  #    course::234567::enrollment_type::ta:
+  #      mem_limit: 2096M
+  #      mem_guarantee: 2048M
+  #
+  #
+  #    # Example: a fully specified CanvasOAuthenticator group name.
+  #    # This could be useful for temporary resource bumps where the
+  #    # instructor could add people to groups in the bCourses UI. This
+  #    # would benefit from the ability to read resource bumps from
+  #    # jupyterhub's properties. (attributes in the ORM)
+  #
+  #    # Name of Class 100, Fall '22; requested in #98770
+  #    course::123456::group::lab4-bigdata:
+  #      - mountPath: /home/rstudio/.ssh
+  #        name: home
+  #        subPath: _some_directory/_ssh
+  #        readOnly: true

deployments/ugr01/config/filestore/squash-flags.json

@@ -0,0 +1,16 @@
+{
+"--file-share":
+  {
+    "name": "shares",
+    "capacity": "desired-capacity",
+    "nfs-export-options": [
+      {
+        "access-mode": "READ_WRITE",
+        "ip-ranges": ["10.0.0.0/8"],
+        "squash-mode": "ROOT_SQUASH",
+        "anon_uid": 1000,
+        "anon_gid": 1000
+      }
+    ],
+  }
+}

deployments/ugr01/config/prod.yaml

@@ -0,0 +1,18 @@
+nfsPVC:
+  nfs:
+    shareName: shares/ugr01/prod
+
+jupyterhub:
+  ingress:
+    enabled: true
+    hosts:
+      - ugr01.datahub.berkeley.edu
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - ugr01.datahub.berkeley.edu
+  hub:
+    db:
+      pvc:
+        # This also holds logs
+        storage: 4Gi

deployments/ugr01/config/staging.yaml

@@ -0,0 +1,19 @@
+nfsPVC:
+  nfs:
+    shareName: shares/ugr01/staging
+
+jupyterhub:
+  scheduling:
+    userScheduler:
+      replicas: 1
+  prePuller:
+    continuous:
+      enabled: false
+  ingress:
+    enabled: true
+    hosts:
+      - ugr01-staging.datahub.berkeley.edu
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - ugr01-staging.datahub.berkeley.edu

deployments/ugr01/hubploy.yaml

@@ -0,0 +1,19 @@
+images:
+  images:
+    - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/ugr01-user-image
+      path: image/
+      repo2docker:
+        base_image: docker.io/library/buildpack-deps:jammy
+  registry:
+    provider: gcloud
+    gcloud:
+      project: ucb-datahub-2018
+      service_key: gcr-key.json
+
+cluster:
+  provider: gcloud
+  gcloud:
+    project: ucb-datahub-2018
+    service_key: gke-key.json
+    cluster: ugresearch-cluster
+    zone: us-central1

deployments/ugr01/image/apt.txt

@@ -0,0 +1,97 @@
+# Some linux packages for basic terminal work, particularly
+# oriented at users new to Unix/cmd line environments.
+
+# installing less as more just isn't enough
+less
+
+# Basic unix tools
+man
+man-db
+manpages-posix
+manpages-dev
+manpages-posix-dev
+
+# Download tools
+curl
+wget
+
+# Core text editors on a *nix box: vim
+vim
+
+# A couple of CLI editors that are easier than vim
+# micro  # currently not working on 18.04
+nano
+jed
+jed-extra
+
+# powerful terminal-based file manager, better than the one in JLab
+mc
+
+# for easily managing multiple repositories with one command (perl-doc
+# is needed for its help pages to work)
+mr
+perl-doc
+
+# Regular build tools for compiling common stuff
+build-essential
+gfortran
+
+# Dependencies for nbconvert
+texlive-xetex
+texlive-fonts-recommended
+texlive-plain-generic
+# https://github.com/berkeley-dsep-infra/datahub/issues/3719
+texlive-lang-chinese
+lmodern
+
+# Other useful document-related tools
+pandoc
+latexdiff
+
+# Some useful git utilities use basic Ruby
+ruby
+
+# Other niceties for command-line work and life
+ack   # powerful grep-like tool
+pydf  # colorized disk usage
+tmux
+screen
+htop
+nnn   # cmd line file manager
+zsh
+rsync
+tig  # console UI for git
+multitail
+
+# For later, these are not available in 18.04
+#browsh # text-based web browser, occasionally handy
+#dasel  # json/yml/csv/etc data wrangling at the terminal
+#fzf   # fuzzy file finder
+
+## This section adds tools for desktop environment usage
+dbus-x11
+xorg
+xubuntu-icon-theme
+xfce4
+xfce4-goodies
+xclip
+xsel
+firefox
+chromium-browser
+
+# GUI text editors
+vim-gtk3
+gedit
+
+# Git clients and tools
+git-gui
+gitg
+qgit
+meld
+
+# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979			
+zip
+
+# playwright deps https://jira-secure.berkeley.edu/browse/DH-305
+libnss3
+libnspr4

deployments/ugr01/image/environment.yml

@@ -0,0 +1,63 @@
+name: ugr01-FA24
+
+channels:
+- conda-forge
+- pytorch
+
+dependencies:
+- python==3.11.*
+- git==2.39.1
+- jupyter-resource-usage==1.0.0
+- jupyterlab==4.0.11
+- jupyterlab-favorites==3.0.0
+- jupyterlab_server==2.23.0
+- jupyterlab_widgets==3.0.8
+- jupyter_server==2.7.0
+- nbgitpuller==1.2.1
+- notebook==7.0.7
+- folium==0.14.0
+- h5netcdf==1.0.2
+- ipywidgets==8.0.7
+- jupysql==0.8.0
+- jupyter-archive==3.4.0
+- matplotlib==3.7.1
+- mdit-py-plugins==0.4.0
+- numpy==1.24.2
+- pandas==2.0.2
+- plotly==5.13.1
+- requests==2.28.2
+- scikit-image==0.19.3
+- scikit-learn==1.2.2
+- scipy==1.10.1
+- seaborn==0.12.2
+- statsmodels==0.14.0
+- tensorflow-cpu==2.12.1
+- sqlalchemy==2.0.16
+- mlxtend==0.23.0
+# Spring 2024 data 100
+- pytorch==2.1.2
+- cpuonly==2.0
+- transformers==4.37.1
+# Spring 2024 table demos
+- lxml==5.1.0
+# Spring 2024 Econ 148 Packages
+- geopandas==0.14.2
+- geopy==2.4.1
+- lifelines==0.27.8
+- pycountry==22.3.5
+- pip
+- pip:
+  # - -r infra-requirements.txt
+  - ipywidgets==8.0.7
+  # disable until fixed (probably this:  https://github.com/jupyterlab/jupyter-collaboration/issues/162)
+  # - jupyter_collaboration==1.0.1
+  - jupyterhub==4.1.5
+  - nbconvert[webpdf]
+  #  - pyppeteer==2.0.0
+  - pytest-notebook==0.8.1
+  - gh-scoped-creds==4.1
+  - git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling
+  - ydata-profiling==4.6.4
+  - otter-grader==5.4.0
+  - duckdb==0.10.1
+  - duckdb_engine==0.11.2