diff --git a/.github/workflows/deploy-jupyterhub-base-images.yaml b/.github/workflows/deploy-jupyterhub-base-images.yaml new file mode 100644 index 000000000..ff84d0845 --- /dev/null +++ b/.github/workflows/deploy-jupyterhub-base-images.yaml @@ -0,0 +1,194 @@ +name: Deploy base hub images to staging +on: + workflow_dispatch: + push: + branches: + - staging + - prod + +jobs: + deploy-hub-images-staging: + if: github.event_name == 'push' && github.ref == 'refs/heads/staging' + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + if [[ -n "${DEPLOY}" ]]; then + echo "Deploying base hub images to all deployments" + else + echo "No hub images to deploy" + fi + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy base hub images to staging + if: ${{ env.DEPLOY }} + run: | + ignored_directories=("template") # these are directories that we never want to deploy to + while read deployment; do + for ignored in "${ignored_directories[@]}"; do + if [[ "${deployment}" == "${ignored}" ]]; then + continue 2 # skip to the next iteration of "while read deployment" + fi + done + echo "Pretending to deploy base hub image to ${deployment} :P" + echo "hubploy deploy --debug ${deployment} hub staging" + done < <(ls deployments/ | sed -e 's,/,,g') + + deploy-hub-images-prod: + if: github.event_name == 'push' && github.ref == 'refs/heads/prod' + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Pull out any hubs that need deploying from the labels on the merge commit to prod + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + if [[ -n "${DEPLOY}" ]]; then + echo "Deploying base hub images to all deployments" + else + echo "No hub images to deploy" + fi + + - name: Setup python + if: ${{ env.DEPLOY }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + if: ${{ env.DEPLOY }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm repo update + + - name: Deploy base hub images to prod + if: ${{ env.DEPLOY }} + run: | + ignored_directories=("template") # these are directories that we never want to deploy to + while read deployment; do + for ignored in "${ignored_directories[@]}"; do + if [[ "${deployment}" == "${ignored}" ]]; then + continue 2 # skip to the next iteration of "while read deployment" + fi + done + echo "Pretending to deploy base hub image to ${deployment} :P" + echo "hubploy deploy --debug ${deployment} hub prod" + done < <(ls deployments/ | sed -e 's,/,,g') diff --git a/.github/workflows/deploy-node-placeholder.yaml.disabled b/.github/workflows/deploy-node-placeholder.yaml.disabled new file mode 100644 index 000000000..1099d5ca7 --- /dev/null +++ b/.github/workflows/deploy-node-placeholder.yaml.disabled @@ -0,0 +1,89 @@ +name: Deploy node placeholder helm chart +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it +on: + workflow_dispatch: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Check if the nbde placeholder helm chart needs to be deployed + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == node-placeholder-* ]]; then + echo "Deploying node placeholder charts!" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + + - name: Activate credentials for cluster + if: ${{ env.DEPLOY }} + run: | + sops -d -i deployments/datahub/secrets/gke-key.json + gcloud auth \ + activate-service-account \ + --key-file deployments/datahub/secrets/gke-key.json \ + gcloud container clusters \ + --region=us-central1 --project=ucb-datahub-2018 \ + get-credentials spring-2024 + + - name: Deploy node placeholder helm chart + if: ${{ env.DEPLOY }} + run: | + sops -d -i node-placeholder/secrets.yaml + helm upgrade \ + --install --wait \ + --namespace=node-placeholder node-placeholder node-placeholder \ + -f node-placeholder/secrets.yaml --debug \ No newline at end of file diff --git a/.github/workflows/deploy-support.yaml.disabled b/.github/workflows/deploy-support.yaml.disabled new file mode 100644 index 000000000..88442fc9d --- /dev/null +++ b/.github/workflows/deploy-support.yaml.disabled @@ -0,0 +1,92 @@ +name: Deploy support helm chart +# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it +on: + workflow_dispatch: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Get PR labels + id: pr-labels + uses: irby/get-labels-on-push@v1.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out the image repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Check if the support helm chart needs to be deployed + run: | + echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" + for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do + if [[ "$label" == support-deployment ]]; then + echo "Deploying support charts!" + echo "DEPLOY=1" >> $GITHUB_ENV + fi + done + + - name: Auth to gcloud + if: ${{ env.DEPLOY }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GKE_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Install Google Cloud SDK + if: ${{ env.DEPLOY }} + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Install SOPS + if: ${{ env.DEPLOY }} + run: | + mkdir -p ${HOME}/bin + curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops + chmod 755 ${HOME}/bin/sops + echo "${HOME}/bin" >> $GITHUB_PATH + + - name: Store SOPS secret in a file + if: ${{ env.DEPLOY }} + run: | + cat << EOF > ${HOME}/sops.key + ${{ secrets.SOPS_KEY }} + EOF + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV + + - name: Install Helm + if: ${{ env.DEPLOY }} + run: | + curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - + mv linux-amd64/helm /usr/local/bin + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + + - name: Activate credentials for cluster + if: ${{ env.DEPLOY }} + run: | + sops -d -i deployments/datahub/secrets/gke-key.json + gcloud auth \ + activate-service-account \ + --key-file deployments/datahub/secrets/gke-key.json + gcloud container clusters \ + --region=us-central1 --project=ucb-datahub-2018 \ + get-credentials spring-2024 + + - name: Deploy support helm chart + if: ${{ env.DEPLOY }} + run: | + sops -d -i support/secrets.yaml + helm dep up support + helm upgrade \ + --install --wait \ + --namespace=support \ + support support/ \ + -f support/secrets.yaml \ + --set installCRDs=true --debug diff --git a/deployments/biology/image/environment.yml b/deployments/biology/image/environment.yml index 43f807b0c..cc4bee30c 100644 --- a/deployments/biology/image/environment.yml +++ b/deployments/biology/image/environment.yml @@ -59,6 +59,9 @@ dependencies: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets=5.9.* +# for MCELLBI201B (FA24) https://github.com/berkeley-dsep-infra/datahub/issues/5988 +- macs2==2.2.9.1 + # For https://github.com/berkeley-dsep-infra/datahub/issues/1846 # Conda does not have these - pip: diff --git a/deployments/datahub/config/common.yaml b/deployments/datahub/config/common.yaml index 91206c55f..0eba373f4 100644 --- a/deployments/datahub/config/common.yaml +++ b/deployments/datahub/config/common.yaml @@ -207,6 +207,24 @@ jupyterhub: subPath: _shared/course/mba-247 readOnly: true + # COMPSS 214A, Fall 2024, https://jira-secure.berkeley.edu/browse/DH-311 + course::1538139::enrollment_type::teacher: + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/compss-214a-readwrite + subPath: _shared/course/compss-214a + course::1538139::enrollment_type::ta: + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/compss-214a-readwrite + subPath: _shared/course/compss-214a + course::1538139::enrollment_type::student: + extraVolumeMounts: + - name: home + mountPath: /home/jovyan/compss-214a + subPath: _shared/course/compss-214a + readOnly: true + course::1534506: # Demog Data Event, April 1 - Sep 30, https://github.com/berkeley-dsep-infra/datahub/issues/5643 mem_limit: 8192M mem_guarantee: 8192M diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index f6a83f0fd..d532f0a5b 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:a9443779a3a9 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:c44b36bfede9 cluster: provider: gcloud diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 679e2901c..2c3932b5b 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -79,7 +79,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 biology: nodeSelector: hub.jupyter.org/pool-name: biology-pool @@ -87,7 +87,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 cee: nodeSelector: hub.jupyter.org/pool-name: cee-pool @@ -95,7 +95,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 data100: nodeSelector: hub.jupyter.org/pool-name: data100-pool @@ -103,7 +103,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 207314055168 - replicas: 0 + replicas: 1 data101: nodeSelector: hub.jupyter.org/pool-name: data101-pool @@ -111,7 +111,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 data102: nodeSelector: hub.jupyter.org/pool-name: data102-pool @@ -119,7 +119,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 data8: nodeSelector: hub.jupyter.org/pool-name: data8-pool @@ -127,7 +127,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 datahub: nodeSelector: hub.jupyter.org/pool-name: datahub-pool @@ -143,7 +143,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 29247442944 - replicas: 1 + replicas: 0 dlab: nodeSelector: hub.jupyter.org/pool-name: dlab-pool @@ -151,7 +151,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 eecs: nodeSelector: hub.jupyter.org/pool-name: eecs-pool @@ -159,7 +159,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 ischool: nodeSelector: hub.jupyter.org/pool-name: ischool-pool @@ -175,7 +175,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 stat159: nodeSelector: hub.jupyter.org/pool-name: stat159-pool @@ -191,7 +191,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 r: nodeSelector: hub.jupyter.org/pool-name: r-pool @@ -199,7 +199,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1 small-courses: nodeSelector: hub.jupyter.org/pool-name: small-courses-pool @@ -207,4 +207,4 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 0 + replicas: 1