From 2d423890ce600eb691ce79e8f5d615414de7e597 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 21 May 2024 09:46:10 -0700 Subject: [PATCH 1/6] pin requests to work around https://github.com/psf/requests/issues/6707 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7546f2c7a..024eec7cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ chartpress jupyter-repo2docker==2024.3.0 myst-parser chardet +requests==2.31.0 From cb82873a064c35fb4ebda6178b228781866135a1 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 21 May 2024 09:50:26 -0700 Subject: [PATCH 2/6] adding vscode to datahub --- deployments/datahub/images/default/environment.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml index 32488822d..6bddc73d6 100644 --- a/deployments/datahub/images/default/environment.yml +++ b/deployments/datahub/images/default/environment.yml @@ -95,6 +95,11 @@ dependencies: # EPS 256, https://github.com/berkeley-dsep-infra/datahub/issues/1775 - obspy==1.4.1 +# Install VSCode for ESPM courses - https://github.com/berkeley-dsep-infra/datahub/issues/5716 +- nodejs=16 # code-server requires node < 17 +- jupyter-vscode-proxy==0.5 +- code-server==4.10.1 + - pip: # - -r /tmp/infra-requirements.txt # Econ 148, Spring 2023 https://github.com/berkeley-dsep-infra/datahub/issues/4093 From d00569871ad1fc06a99ecda161d6e4d6285e6233 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 21 May 2024 10:27:47 -0700 Subject: [PATCH 3/6] bump python in ci to 3.11 --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9867afa6..4df87009f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ orbs: default: false docker: - - image: python:3.9-slim-bullseye + - image: python:3.11-slim-bullseye working_directory: ~/repo resource_class: large steps: @@ -23,9 +23,9 @@ orbs: apt-get update && apt-get install --yes --no-install-recommends git curl - restore_cache: keys: - - v3.9-dependencies-{{ checksum "requirements.txt" }} + - v3.11-dependencies-{{ checksum "requirements.txt" }} # fallback to using the latest cache if no exact match is found - - v3.9-dependencies- + - v3.11-dependencies- - run: name: install dependencies @@ -91,7 +91,7 @@ orbs: - save_cache: paths: - ./venv - key: v3.9-dependencies-{{ checksum "requirements.txt" }} + key: v3.11-dependencies-{{ checksum "requirements.txt" }} - run: name: Build image if needed @@ -107,7 +107,7 @@ orbs: jobs: deploy: docker: - - image: python:3.9-slim-bullseye + - image: python:3.11-slim-bullseye working_directory: ~/repo steps: - run: @@ -119,9 +119,9 @@ jobs: # Download and cache dependencies - restore_cache: keys: - - v3.9-dependencies-gcloud-457-{{ checksum "requirements.txt" }} + - v3.11-dependencies-gcloud-457-{{ checksum "requirements.txt" }} # fallback to using the latest cache if no exact match is found - - v3.9-dependencies-gcloud-457- + - v3.11-dependencies-gcloud-457- - run: name: install dependencies @@ -146,7 +146,7 @@ jobs: - save_cache: paths: - ./venv - key: v3.9-dependencies-gcloud-457-{{ checksum "requirements.txt" }} + key: v3.11-dependencies-gcloud-457-{{ checksum "requirements.txt" }} - run: name: Authenticating with google service account for kms/sops From 411aec64d52173b74e3cb6d22645638ce6604cae Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 21 May 2024 10:37:09 -0700 Subject: [PATCH 4/6] trying out niquests --- scripts/delete-unused-users.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/delete-unused-users.py b/scripts/delete-unused-users.py index aef81f523..0da572147 100755 --- a/scripts/delete-unused-users.py +++ b/scripts/delete-unused-users.py @@ -19,7 +19,7 @@ import json import logging import os -import requests +import niquests import sys from dateutil.parser import parse @@ -55,7 +55,7 @@ def retrieve_users(hub_url, headers, inactive_since): params = {} while next_page: - r = requests.get(url, headers=headers, params=params) + r = niquests.get(url, headers=headers, params=params) r.raise_for_status() resp = r.json() user_list = resp["items"] @@ -104,7 +104,7 @@ def should_delete(user, inactive_since): def delete_user(hub_url, headers, name): """Delete a given user by name via JupyterHub API""" - r = requests.delete( + r = niquests.delete( hub_url.rstrip("/") + f"/hub/api/users/{name}", headers=headers, ) From 08ef990c02d53eb36c0afdfd6720444a34908ed6 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 21 May 2024 15:43:50 -0700 Subject: [PATCH 5/6] Rename dev-r hubs to dev. --- .circleci/config.yml | 14 +++++++------- .github/labeler.yml | 4 ++-- deployments/dev-r/config/common.yaml | 4 ++-- deployments/dev-r/config/prod.yaml | 6 +++--- deployments/dev-r/config/staging.yaml | 6 +++--- deployments/dev-r/hubploy.yaml | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4df87009f..00032e729 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -236,9 +236,9 @@ jobs: no_output_timeout: 30m - run: - name: Deploy dev-r + name: Deploy dev command: | - hubploy deploy --timeout 30m dev-r hub ${CIRCLE_BRANCH} + hubploy deploy --timeout 30m dev hub ${CIRCLE_BRANCH} no_output_timeout: 30m - run: @@ -538,8 +538,8 @@ workflows: # This has no effect yet because docker-py doesn't support buildkit. #environment: # DOCKER_BUILDKIT: 1 - deployment: dev-r - name: dev-r image build + deployment: dev + name: dev image build filters: branches: ignore: @@ -701,8 +701,8 @@ workflows: only: - staging - hubploy/build-image: - deployment: dev-r - name: dev-r image build + deployment: dev + name: dev image build push: true # Filters can only be per-job? wtf filters: @@ -805,7 +805,7 @@ workflows: - data101 image build - data102 image build - datahub image build - - dev-r image build + - dev image build - edx image build - eecs image build - ischool image build diff --git a/.github/labeler.yml b/.github/labeler.yml index 54ea56eff..7d76422a4 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -101,8 +101,8 @@ node-placeholder-scaler: - 'deployments/datahub/**' 'hub: dlab': - 'deployments/dlab/**' -'hub: dev-r': - - 'deployments/dev-r/**' +'hub: dev': + - 'deployments/dev/**' 'hub: edx': - 'deployments/edx/**' 'hub: eecs': diff --git a/deployments/dev-r/config/common.yaml b/deployments/dev-r/config/common.yaml index 0d180181d..90cb67801 100644 --- a/deployments/dev-r/config/common.yaml +++ b/deployments/dev-r/config/common.yaml @@ -64,12 +64,12 @@ jupyterhub: defaultUrl: /rstudio profileList: - display_name: "Dockerfile image" - description: "This is the original dev-r image." + description: "This is the original dev image." default: true - display_name: "repo2docker image" description: "A newer repo2docker-based image with similar components as the primary." kubespawner_override: - image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-r-secondary:1a64e0a + image: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary:1a64e0a - display_name: "1524699: DataHub Infrastructure" slug: "1524699" description: "Regular image with per-course subpath." diff --git a/deployments/dev-r/config/prod.yaml b/deployments/dev-r/config/prod.yaml index cd639e48a..3f65b43cf 100644 --- a/deployments/dev-r/config/prod.yaml +++ b/deployments/dev-r/config/prod.yaml @@ -1,16 +1,16 @@ nfsPVC: nfs: - shareName: shares/dev-r/prod + shareName: shares/dev/prod jupyterhub: ingress: enabled: true hosts: - - dev-r.datahub.berkeley.edu + - dev.datahub.berkeley.edu tls: - secretName: tls-cert hosts: - - dev-r.datahub.berkeley.edu + - dev.datahub.berkeley.edu hub: db: pvc: diff --git a/deployments/dev-r/config/staging.yaml b/deployments/dev-r/config/staging.yaml index 2999ee041..08f3c3ca2 100644 --- a/deployments/dev-r/config/staging.yaml +++ b/deployments/dev-r/config/staging.yaml @@ -1,6 +1,6 @@ nfsPVC: nfs: - shareName: shares/dev-r/staging + shareName: shares/dev/staging jupyterhub: scheduling: @@ -12,8 +12,8 @@ jupyterhub: ingress: enabled: true hosts: - - dev-r-staging.datahub.berkeley.edu + - dev-staging.datahub.berkeley.edu tls: - secretName: tls-cert hosts: - - dev-r-staging.datahub.berkeley.edu + - dev-staging.datahub.berkeley.edu diff --git a/deployments/dev-r/hubploy.yaml b/deployments/dev-r/hubploy.yaml index ba6a74b59..73a41f621 100644 --- a/deployments/dev-r/hubploy.yaml +++ b/deployments/dev-r/hubploy.yaml @@ -1,8 +1,8 @@ images: images: - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-r-user-image + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-user-image path: images/default - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-r-secondary + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/dev-secondary path: images/secondary repo2docker: base_image: docker.io/library/buildpack-deps:jammy From 8d3dffd8eddf326a0a3fd5a61b708f7bb30f0a63 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 21 May 2024 15:48:37 -0700 Subject: [PATCH 6/6] Rename deployment directory. --- deployments/{dev-r => dev}/config/common.yaml | 0 deployments/{dev-r => dev}/config/prod.yaml | 0 deployments/{dev-r => dev}/config/staging.yaml | 0 deployments/{dev-r => dev}/hubploy.yaml | 0 deployments/{dev-r => dev}/images/default/Dockerfile | 0 deployments/{dev-r => dev}/images/default/Rprofile.site | 0 deployments/{dev-r => dev}/images/default/apt.txt | 0 deployments/{dev-r => dev}/images/default/class-libs.R | 0 deployments/{dev-r => dev}/images/default/environment.yml | 0 deployments/{dev-r => dev}/images/default/infra-requirements.txt | 0 deployments/{dev-r => dev}/images/default/install-mambaforge.bash | 0 deployments/{dev-r => dev}/images/default/install.R | 0 deployments/{dev-r => dev}/images/default/rsession.conf | 0 deployments/{dev-r => dev}/images/secondary/Rprofile.site | 0 deployments/{dev-r => dev}/images/secondary/apt.txt | 0 deployments/{dev-r => dev}/images/secondary/environment.yml | 0 .../{dev-r => dev}/images/secondary/infra-requirements.txt | 0 deployments/{dev-r => dev}/images/secondary/install.R | 0 deployments/{dev-r => dev}/images/secondary/rsession.conf | 0 deployments/{dev-r => dev}/images/secondary/runtime.txt | 0 deployments/{dev-r => dev}/secrets/gcr-key.json | 0 deployments/{dev-r => dev}/secrets/gke-key.json | 0 deployments/{dev-r => dev}/secrets/prod.yaml | 0 deployments/{dev-r => dev}/secrets/staging.yaml | 0 24 files changed, 0 insertions(+), 0 deletions(-) rename deployments/{dev-r => dev}/config/common.yaml (100%) rename deployments/{dev-r => dev}/config/prod.yaml (100%) rename deployments/{dev-r => dev}/config/staging.yaml (100%) rename deployments/{dev-r => dev}/hubploy.yaml (100%) rename deployments/{dev-r => dev}/images/default/Dockerfile (100%) rename deployments/{dev-r => dev}/images/default/Rprofile.site (100%) rename deployments/{dev-r => dev}/images/default/apt.txt (100%) rename deployments/{dev-r => dev}/images/default/class-libs.R (100%) rename deployments/{dev-r => dev}/images/default/environment.yml (100%) rename deployments/{dev-r => dev}/images/default/infra-requirements.txt (100%) rename deployments/{dev-r => dev}/images/default/install-mambaforge.bash (100%) rename deployments/{dev-r => dev}/images/default/install.R (100%) rename deployments/{dev-r => dev}/images/default/rsession.conf (100%) rename deployments/{dev-r => dev}/images/secondary/Rprofile.site (100%) rename deployments/{dev-r => dev}/images/secondary/apt.txt (100%) rename deployments/{dev-r => dev}/images/secondary/environment.yml (100%) rename deployments/{dev-r => dev}/images/secondary/infra-requirements.txt (100%) rename deployments/{dev-r => dev}/images/secondary/install.R (100%) rename deployments/{dev-r => dev}/images/secondary/rsession.conf (100%) rename deployments/{dev-r => dev}/images/secondary/runtime.txt (100%) rename deployments/{dev-r => dev}/secrets/gcr-key.json (100%) rename deployments/{dev-r => dev}/secrets/gke-key.json (100%) rename deployments/{dev-r => dev}/secrets/prod.yaml (100%) rename deployments/{dev-r => dev}/secrets/staging.yaml (100%) diff --git a/deployments/dev-r/config/common.yaml b/deployments/dev/config/common.yaml similarity index 100% rename from deployments/dev-r/config/common.yaml rename to deployments/dev/config/common.yaml diff --git a/deployments/dev-r/config/prod.yaml b/deployments/dev/config/prod.yaml similarity index 100% rename from deployments/dev-r/config/prod.yaml rename to deployments/dev/config/prod.yaml diff --git a/deployments/dev-r/config/staging.yaml b/deployments/dev/config/staging.yaml similarity index 100% rename from deployments/dev-r/config/staging.yaml rename to deployments/dev/config/staging.yaml diff --git a/deployments/dev-r/hubploy.yaml b/deployments/dev/hubploy.yaml similarity index 100% rename from deployments/dev-r/hubploy.yaml rename to deployments/dev/hubploy.yaml diff --git a/deployments/dev-r/images/default/Dockerfile b/deployments/dev/images/default/Dockerfile similarity index 100% rename from deployments/dev-r/images/default/Dockerfile rename to deployments/dev/images/default/Dockerfile diff --git a/deployments/dev-r/images/default/Rprofile.site b/deployments/dev/images/default/Rprofile.site similarity index 100% rename from deployments/dev-r/images/default/Rprofile.site rename to deployments/dev/images/default/Rprofile.site diff --git a/deployments/dev-r/images/default/apt.txt b/deployments/dev/images/default/apt.txt similarity index 100% rename from deployments/dev-r/images/default/apt.txt rename to deployments/dev/images/default/apt.txt diff --git a/deployments/dev-r/images/default/class-libs.R b/deployments/dev/images/default/class-libs.R similarity index 100% rename from deployments/dev-r/images/default/class-libs.R rename to deployments/dev/images/default/class-libs.R diff --git a/deployments/dev-r/images/default/environment.yml b/deployments/dev/images/default/environment.yml similarity index 100% rename from deployments/dev-r/images/default/environment.yml rename to deployments/dev/images/default/environment.yml diff --git a/deployments/dev-r/images/default/infra-requirements.txt b/deployments/dev/images/default/infra-requirements.txt similarity index 100% rename from deployments/dev-r/images/default/infra-requirements.txt rename to deployments/dev/images/default/infra-requirements.txt diff --git a/deployments/dev-r/images/default/install-mambaforge.bash b/deployments/dev/images/default/install-mambaforge.bash similarity index 100% rename from deployments/dev-r/images/default/install-mambaforge.bash rename to deployments/dev/images/default/install-mambaforge.bash diff --git a/deployments/dev-r/images/default/install.R b/deployments/dev/images/default/install.R similarity index 100% rename from deployments/dev-r/images/default/install.R rename to deployments/dev/images/default/install.R diff --git a/deployments/dev-r/images/default/rsession.conf b/deployments/dev/images/default/rsession.conf similarity index 100% rename from deployments/dev-r/images/default/rsession.conf rename to deployments/dev/images/default/rsession.conf diff --git a/deployments/dev-r/images/secondary/Rprofile.site b/deployments/dev/images/secondary/Rprofile.site similarity index 100% rename from deployments/dev-r/images/secondary/Rprofile.site rename to deployments/dev/images/secondary/Rprofile.site diff --git a/deployments/dev-r/images/secondary/apt.txt b/deployments/dev/images/secondary/apt.txt similarity index 100% rename from deployments/dev-r/images/secondary/apt.txt rename to deployments/dev/images/secondary/apt.txt diff --git a/deployments/dev-r/images/secondary/environment.yml b/deployments/dev/images/secondary/environment.yml similarity index 100% rename from deployments/dev-r/images/secondary/environment.yml rename to deployments/dev/images/secondary/environment.yml diff --git a/deployments/dev-r/images/secondary/infra-requirements.txt b/deployments/dev/images/secondary/infra-requirements.txt similarity index 100% rename from deployments/dev-r/images/secondary/infra-requirements.txt rename to deployments/dev/images/secondary/infra-requirements.txt diff --git a/deployments/dev-r/images/secondary/install.R b/deployments/dev/images/secondary/install.R similarity index 100% rename from deployments/dev-r/images/secondary/install.R rename to deployments/dev/images/secondary/install.R diff --git a/deployments/dev-r/images/secondary/rsession.conf b/deployments/dev/images/secondary/rsession.conf similarity index 100% rename from deployments/dev-r/images/secondary/rsession.conf rename to deployments/dev/images/secondary/rsession.conf diff --git a/deployments/dev-r/images/secondary/runtime.txt b/deployments/dev/images/secondary/runtime.txt similarity index 100% rename from deployments/dev-r/images/secondary/runtime.txt rename to deployments/dev/images/secondary/runtime.txt diff --git a/deployments/dev-r/secrets/gcr-key.json b/deployments/dev/secrets/gcr-key.json similarity index 100% rename from deployments/dev-r/secrets/gcr-key.json rename to deployments/dev/secrets/gcr-key.json diff --git a/deployments/dev-r/secrets/gke-key.json b/deployments/dev/secrets/gke-key.json similarity index 100% rename from deployments/dev-r/secrets/gke-key.json rename to deployments/dev/secrets/gke-key.json diff --git a/deployments/dev-r/secrets/prod.yaml b/deployments/dev/secrets/prod.yaml similarity index 100% rename from deployments/dev-r/secrets/prod.yaml rename to deployments/dev/secrets/prod.yaml diff --git a/deployments/dev-r/secrets/staging.yaml b/deployments/dev/secrets/staging.yaml similarity index 100% rename from deployments/dev-r/secrets/staging.yaml rename to deployments/dev/secrets/staging.yaml