merging 5887/5889/5890/5891/5892/5894 to prod

.circleci/config.yml

@@ -283,11 +283,11 @@ jobs:
             hubploy deploy --timeout 30m julia hub ${CIRCLE_BRANCH}
           no_output_timeout: 30m
 
-      - run:
-          name: Deploy logodev
-          command: |
-            hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH}
-          no_output_timeout: 30m
+#      - run:
+#          name: Deploy logodev
+#          command: |
+#            hubploy deploy --timeout 30m logodev hub ${CIRCLE_BRANCH}
+#          no_output_timeout: 30m
 
       - run:
           name: Deploy prob140
@@ -581,15 +581,15 @@ workflows:
                 ignore:
                 - staging
                 - prod
-      - hubploy/build-image:
-          deployment: logodev
-          name: logodev hub image build
-          # Filters can only be per-job? wtf
-          filters:
-              branches:
-                ignore:
-                - staging
-                - prod
+#      - hubploy/build-image:
+#          deployment: logodev
+#          name: logodev hub image build
+#          # Filters can only be per-job? wtf
+#          filters:
+#              branches:
+#                ignore:
+#                - staging
+#                - prod
       - hubploy/build-image:
           deployment: publichealth
           name: publichealth image build
@@ -754,15 +754,15 @@ workflows:
               branches:
                 only:
                 - staging
-      - hubploy/build-image:
-          deployment: logodev
-          name: logodev image build
-          push: true
-          # Filters can only be per-job? wtf
-          filters:
-              branches:
-                only:
-                - staging
+#      - hubploy/build-image:
+#          deployment: logodev
+#          name: logodev image build
+#          push: true
+#          # Filters can only be per-job? wtf
+#          filters:
+#              branches:
+#                only:
+#                - staging
       - hubploy/build-image:
           deployment: publichealth
           name: publichealth image build
@@ -819,7 +819,7 @@ workflows:
               - eecs image build
               - ischool image build
               - julia hub image build
-              - logodev image build
+#              - logodev image build
               - publichealth image build
               - shiny image build
               - stat159 image build

deployments/datahub/config/common.yaml

@@ -106,6 +106,8 @@ jupyterhub:
         hub.jupyter.org/pool-name: core-pool-2024-05-08
   singleuser:
     extraEnv:
+      GH_SCOPED_CREDS_CLIENT_ID: Iv23ct7Qx1mAotaIIYx9
+      GH_SCOPED_CREDS_APP_URL: https://github.com/apps/berkeley-datahub-git-access
       # Unset NotebookApp from hub/values. Necessary for recent lab versions.
       JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp"
     nodeSelector:

deployments/datahub/images/default/environment.yml

@@ -217,3 +217,10 @@ dependencies:
   # [DH-243] https://github.com/berkeley-dsep-infra/datahub/issues/5571, Legal Studies 123, Spring 24
   - geopandas==0.14.3
   - rtree==1.2.0
+
+  # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157
+  - altair==5.3.0
+  - ibis-framework[pandas]==9.2.0
+  - leafmap==0.36.1
+  - mystmd==1.3.0
+  - jupyterlab-git==0.50.1

deployments/logodev/hubploy.yaml

@@ -1,7 +1,7 @@
 images:
   images:
     # temporary update
-    - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image
+    - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a02e39fbaedb
       path: image/
       repo2docker:
         base_image: docker.io/library/buildpack-deps:jammy

docs/admins/howto/index.rst

@@ -11,9 +11,11 @@ Common Administrator Tasks
    core-pool
    new-hub
    rebuild-hub-image
+   new-image
    new-packages
    course-config
    calendar-scaler
    prometheus-grafana
    remove-users-orm
    delete-hub
+   clusterswitch.md

docs/admins/howto/new-image.rst

@@ -0,0 +1,97 @@
+.. _howto/new-packages:
+
+================================
+Creating a new single user image
+================================
+
+When deploying a new hub, or moving from a shared single user server image,
+you might need to create a new image for users. We use 
+`repo2docker <https://github.com/jupyterhub/repo2docker>`_ to do this.
+
+There are two approaches to creating a repo2docker image:
+1. Use a repo2docker-style image `template <https://github.com/berkeley-dsep-infra/datahub/tree/staging/deployments/data100/image>`_ (environment.yaml, etc)
+2. Use a `Dockerfile <https://github.com/berkeley-dsep-infra/datahub/tree/staging/deployments/datahub/images/default>`_ (useful for larger/more complex images)
+
+Generally, we prefer to use the former approach, unless we need to install
+specific packages or utilities outside of python/apt as ``root``.  If that is 
+the case, only a :file:`Dockerfile` format will work.
+
+Of course, as always create a feature branch for your changes, and submit a
+PR when done.
+
+Find a hub to use as a template
+===============================
+
+Browse through our :file:`deployments/` directory to find a hub that is similar to
+the one you are trying to create. This will give you a good starting point.
+
+Create the :file:`image/` directory for your new hub
+====================================================
+
+Create a new directory under :file:`deployments/` with the name of your hub. This
+directory will contain the files that will be used to create the image.
+
+Then, copy the contents (and any subdirectories) of the source :file:`image/`
+directory in to the new directory.
+
+Modify :file:`hubploy.yaml` for the hub
+=======================================
+
+In the deployment's :file:`hubploy.yaml` file, add or modify the ``name``, ``path`` and
+``base_image`` fields to configure the image build and where it's stored in the
+Google Artifcat Registry.
+
+``name`` should contain the path to the image in the Google Artifact Registry and the name of the image.
+``path`` points to the directory containing the image configuration (typically :file::`image/`).
+``base_image`` is the base Docker image to use for the image build.
+
+For example, :file:`hubploy.yaml` for the data100 image looks like this:
+
+.. code:: yaml
+
+      images:
+         images:
+            - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data100-user-image
+               path: image/
+               repo2docker:
+                  base_image: docker.io/library/buildpack-deps:jammy
+         registry:
+            provider: gcloud
+            gcloud:
+               project: ucb-datahub-2018
+               service_key: gcr-key.json
+
+      cluster:
+      provider: gcloud
+      gcloud:
+         project: ucb-datahub-2018
+         service_key: gke-key.json
+         cluster: spring-2024
+         zone: us-central1
+
+Modify the image configuration as necessary
+===========================================
+
+This step is straightforward: edit/modify/delete/add any files in the :file:`image/`
+directory to configure the image as needed.
+
+Update CI/CD configuration
+==========================
+
+Next, ensure that this image will be built and deployed by updating the
+:file:`.circleci/config.yml` file in the root of the repository. Add new steps
+under the ``jobs/deploy:``, ``workflows/test-build-images:`` and ``workflows/deploy:``
+stanzas.
+
+Submitting a pull request
+=========================
+
+Familiarize yourself with `pull requests <https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests>`_ and `repo2docker <https://github.com/jupyter/repo2docker>`_ , and create a fork of the `datahub staging branch <https://github.com/berkeley-dsep-infra/datahub>`_.
+
+#. Set up your git/dev environment by `following the instructions here <https://github.com/berkeley-dsep-infra/datahub/#setting-up-your-fork-and-clones>`_.
+#. Create a new branch for this PR.
+#. Test the changes locally using ``repo2docker``, then submit a PR to ``staging``.
+	* To use ``repo2docker``, you have to point it at the correct image directory. For example, to build the data100 image, you would run ``repo2docker deployments/data100/image`` from the base datahub directory.
+#. Commit and push your changes to your fork of the datahub repo, and create a new pull request at `<https://github.com/berkeley-dsep-infra/datahub/>`__.
+#. Once the PR is merged to staging and the new image is built and pushed to Artifact Registry, you can test it out on :code:`<hub>-staging.datahub.berkeley.edu`.
+#. Changes are only deployed to prod once the relevant CI job is completed. See `<https://circleci.com/gh/berkeley-dsep-infra/datahub>`__ to view CircleCI job statuses. 

docs/conf.py

@@ -68,7 +68,7 @@
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = None
+pygments_style = 'sphinx'
 
 
 # -- Options for HTML output -------------------------------------------------

docs/users/index.rst

@@ -8,4 +8,3 @@ Using DataHub
    private-repo
    hubs
    authentication
-   storage-retention

requirements.txt

@@ -7,6 +7,6 @@ chartpress
 jupyter-repo2docker==2024.3.0
 myst-parser
 chardet
-# requests==2.31.0
-requests==2.32.3
-urllib3==2.2.2
+requests==2.31.0
+# requests==2.32.3
+urllib3<2.0.0