Merge pull request #5998 from berkeley-dsep-infra/staging

merging 5977/5986/5989/5990/5991/5992/5993/5994/5995/5996/5997/6000/6001/6003 to prod

.github/workflows/deploy-jupyterhub-base-images.yaml

@@ -0,0 +1,194 @@
+name: Deploy base hub images to staging
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - staging
+      - prod
+
+jobs:
+  deploy-hub-images-staging:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/staging'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR labels
+        id: pr-labels
+        uses: irby/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check out the image repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.
+
+      - name: Pull out any hubs that need deploying from the labels on the merge commit to staging
+        run: |
+          echo "PR labels: ${{ steps.pr-labels.outputs.labels }}"
+          for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do
+            if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then
+              echo "DEPLOY=1" >> $GITHUB_ENV
+            fi
+          done
+          if [[ -n "${DEPLOY}" ]]; then
+            echo "Deploying base hub images to all deployments"
+          else
+            echo "No hub images to deploy"
+          fi
+
+      - name: Setup python
+        if: ${{ env.DEPLOY }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        if: ${{ env.DEPLOY }}
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor
+
+      - name: Auth to gcloud
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GKE_KEY }}
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+      - name: Install Google Cloud SDK
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: 'gke-gcloud-auth-plugin'
+
+      - name: Install SOPS
+        if: ${{ env.DEPLOY }}
+        run: |
+          mkdir -p ${HOME}/bin
+          curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops
+          chmod 755 ${HOME}/bin/sops
+          echo "${HOME}/bin" >> $GITHUB_PATH
+
+      - name: Store SOPS secret in a file
+        if: ${{ env.DEPLOY }}
+        run: |
+          cat << EOF > ${HOME}/sops.key
+          ${{ secrets.SOPS_KEY }}
+          EOF
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV
+
+      - name: Install Helm
+        if: ${{ env.DEPLOY }}
+        run: |
+            curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf -
+            mv linux-amd64/helm /usr/local/bin
+            helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
+            helm repo update
+
+      - name: Deploy base hub images to staging
+        if: ${{ env.DEPLOY }}
+        run: |
+          ignored_directories=("template") # these are directories that we never want to deploy to
+          while read deployment; do
+            for ignored in "${ignored_directories[@]}"; do
+              if [[ "${deployment}" == "${ignored}" ]]; then
+                continue 2 # skip to the next iteration of "while read deployment"
+              fi
+            done
+            echo "Pretending to deploy base hub image to ${deployment}  :P"
+            echo "hubploy deploy --debug ${deployment} hub staging"
+          done < <(ls deployments/ | sed -e 's,/,,g')
+
+  deploy-hub-images-prod:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/prod'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR labels
+        id: pr-labels
+        uses: irby/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check out the image repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.
+
+      - name: Pull out any hubs that need deploying from the labels on the merge commit to prod
+        run: |
+          echo "PR labels: ${{ steps.pr-labels.outputs.labels }}"
+          for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do
+            if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then
+              echo "DEPLOY=1" >> $GITHUB_ENV
+            fi
+          done
+          if [[ -n "${DEPLOY}" ]]; then
+            echo "Deploying base hub images to all deployments"
+          else
+            echo "No hub images to deploy"
+          fi
+
+      - name: Setup python
+        if: ${{ env.DEPLOY }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        if: ${{ env.DEPLOY }}
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor
+
+      - name: Auth to gcloud
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GKE_KEY }}
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+      - name: Install Google Cloud SDK
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: 'gke-gcloud-auth-plugin'
+
+      - name: Install SOPS
+        if: ${{ env.DEPLOY }}
+        run: |
+          mkdir -p ${HOME}/bin
+          curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops
+          chmod 755 ${HOME}/bin/sops
+          echo "${HOME}/bin" >> $GITHUB_PATH
+
+      - name: Store SOPS secret in a file
+        if: ${{ env.DEPLOY }}
+        run: |
+          cat << EOF > ${HOME}/sops.key
+          ${{ secrets.SOPS_KEY }}
+          EOF
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV
+
+      - name: Install Helm
+        if: ${{ env.DEPLOY }}
+        run: |
+            curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf -
+            mv linux-amd64/helm /usr/local/bin
+            helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
+            helm repo update
+
+      - name: Deploy base hub images to prod
+        if: ${{ env.DEPLOY }}
+        run: |
+          ignored_directories=("template") # these are directories that we never want to deploy to
+          while read deployment; do
+            for ignored in "${ignored_directories[@]}"; do
+              if [[ "${deployment}" == "${ignored}" ]]; then
+                continue 2 # skip to the next iteration of "while read deployment"
+              fi
+            done
+            echo "Pretending to deploy base hub image to ${deployment}  :P"
+            echo "hubploy deploy --debug ${deployment} hub prod"
+          done < <(ls deployments/ | sed -e 's,/,,g')

.github/workflows/deploy-node-placeholder.yaml.disabled

@@ -0,0 +1,89 @@
+name: Deploy node placeholder helm chart
+# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - staging
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR labels
+        id: pr-labels
+        uses: irby/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check out the image repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.
+
+      - name: Check if the nbde placeholder helm chart needs to be deployed
+        run: |
+          echo "PR labels: ${{ steps.pr-labels.outputs.labels }}"
+          for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do
+            if [[ "$label" == node-placeholder-* ]]; then
+              echo "Deploying node placeholder charts!"
+              echo "DEPLOY=1" >> $GITHUB_ENV
+            fi
+          done
+
+      - name: Auth to gcloud
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GKE_KEY }}
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+      - name: Install Google Cloud SDK
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: 'gke-gcloud-auth-plugin'
+
+      - name: Install SOPS
+        if: ${{ env.DEPLOY }}
+        run: |
+          mkdir -p ${HOME}/bin
+          curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops
+          chmod 755 ${HOME}/bin/sops
+          echo "${HOME}/bin" >> $GITHUB_PATH
+
+      - name: Store SOPS secret in a file
+        if: ${{ env.DEPLOY }}
+        run: |
+          cat << EOF > ${HOME}/sops.key
+          ${{ secrets.SOPS_KEY }}
+          EOF
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV
+
+      - name: Install Helm
+        if: ${{ env.DEPLOY }}
+        run: |
+          curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf -
+          mv linux-amd64/helm /usr/local/bin
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo update
+
+      - name: Activate credentials for cluster
+        if: ${{ env.DEPLOY }}
+        run: |
+          sops -d -i deployments/datahub/secrets/gke-key.json
+          gcloud auth \
+            activate-service-account \
+            --key-file deployments/datahub/secrets/gke-key.json \
+          gcloud container clusters \
+            --region=us-central1 --project=ucb-datahub-2018 \
+            get-credentials spring-2024
+
+      - name: Deploy node placeholder helm chart
+        if: ${{ env.DEPLOY }}
+        run: |
+          sops -d -i node-placeholder/secrets.yaml
+          helm upgrade \
+            --install --wait \
+            --namespace=node-placeholder node-placeholder node-placeholder \
+            -f node-placeholder/secrets.yaml --debug

.github/workflows/deploy-support.yaml.disabled

@@ -0,0 +1,92 @@
+name: Deploy support helm chart
+# use echo ${VAR##*: } to get the value of a variable that is a string with a colon in it
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - staging
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR labels
+        id: pr-labels
+        uses: irby/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check out the image repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.
+
+      - name: Check if the support helm chart needs to be deployed
+        run: |
+          echo "PR labels: ${{ steps.pr-labels.outputs.labels }}"
+          for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do
+            if [[ "$label" == support-deployment ]]; then
+              echo "Deploying support charts!"
+              echo "DEPLOY=1" >> $GITHUB_ENV
+            fi
+          done
+
+      - name: Auth to gcloud
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GKE_KEY }}
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+      - name: Install Google Cloud SDK
+        if: ${{ env.DEPLOY }}
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: 'gke-gcloud-auth-plugin'
+
+      - name: Install SOPS
+        if: ${{ env.DEPLOY }}
+        run: |
+          mkdir -p ${HOME}/bin
+          curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops
+          chmod 755 ${HOME}/bin/sops
+          echo "${HOME}/bin" >> $GITHUB_PATH
+
+      - name: Store SOPS secret in a file
+        if: ${{ env.DEPLOY }}
+        run: |
+          cat << EOF > ${HOME}/sops.key
+          ${{ secrets.SOPS_KEY }}
+          EOF
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV
+
+      - name: Install Helm
+        if: ${{ env.DEPLOY }}
+        run: |
+          curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf -
+          mv linux-amd64/helm /usr/local/bin
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo update
+
+      - name: Activate credentials for cluster
+        if: ${{ env.DEPLOY }}
+        run: |
+          sops -d -i deployments/datahub/secrets/gke-key.json
+          gcloud auth \
+            activate-service-account \
+            --key-file deployments/datahub/secrets/gke-key.json
+          gcloud container clusters \
+            --region=us-central1 --project=ucb-datahub-2018 \
+            get-credentials spring-2024
+
+      - name: Deploy support helm chart
+        if: ${{ env.DEPLOY }}
+        run: |
+          sops -d -i support/secrets.yaml
+          helm dep up support
+          helm upgrade \
+            --install --wait \
+            --namespace=support \
+            support support/ \
+            -f support/secrets.yaml \
+            --set installCRDs=true --debug

deployments/biology/image/environment.yml

@@ -59,6 +59,9 @@ dependencies:
 # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048
 - traitlets=5.9.*
 
+# for MCELLBI201B (FA24) https://github.com/berkeley-dsep-infra/datahub/issues/5988
+- macs2==2.2.9.1
+
 # For https://github.com/berkeley-dsep-infra/datahub/issues/1846
 # Conda does not have these
 - pip: