diff --git a/.github/scripts/determine-hub-deployments.py b/.github/scripts/determine-hub-deployments.py new file mode 100755 index 000000000..cc4010bb7 --- /dev/null +++ b/.github/scripts/determine-hub-deployments.py @@ -0,0 +1,69 @@ +#! /usr/bin/env python +""" +Check the Github environment variables for hub deployments and determine if we +will deploy all hubs or just a subset. + +All hubs will be deployed if the environment variable +GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT or GITHUB_PR_LABEL_HUB_IMAGES is set. + +Otherwise, the environment variables GITHUB_PR_LABEL_HUB_ will be +checked to determine which hubs to deploy. + +If no hubs need deploying, nothing will be emitted. +""" +import argparse +import os + +def main(args): + hubs = [] + + # Deploy all hubs by getting deployment names from the dirs in deployments/ + if ( + "GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT" or + "GITHUB_PR_LABEL_HUB_IMAGES" + ) in os.environ.keys(): + for deployment in next(os.walk(args.deployments))[1]: + if deployment not in args.ignore: + hubs.append(deployment) + + # Deploy only the modified/flagged hubs by PR labels + else: + hub_labels = [ + k.lower() for k in os.environ.keys() + if k.startswith("GITHUB_PR_LABEL_HUB_") + ] + hubs = [x.split("_")[-1] for x in hub_labels] + hubs = [x for x in hubs if x not in args.ignore] + + hubs.sort() + for h in hubs: + if args.only_deploy and h not in args.only_deploy: + continue + print(h) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Get hubs that need to be deployed from environment variables." + ) + parser.add_argument( + "--deployments", + "-d", + default="deployments", + help="The directory to search for deployments." + ) + parser.add_argument( + "--ignore", + "-i", + nargs="*", + default=["template"], + help="Ignore one or more deployment targets." + ) + parser.add_argument( + "--only-deploy", + "-o", + nargs="*", + help="Only deploy the specified hubs." + ) + args = parser.parse_args() + + main(args) diff --git a/.github/workflows/deploy-all-hubs.yaml b/.github/workflows/deploy-all-hubs.yaml deleted file mode 100644 index cd032018c..000000000 --- a/.github/workflows/deploy-all-hubs.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# if the PR labels "hub-images" or "jupyterhub-deployment" are present, this -# means the base hub image has changed, and all hubs (staging or prod) need to -# be redeployed. -# -name: Deploy base hub images to all hubs in staging and prod -on: - workflow_dispatch: - push: - branches: - - staging - - prod - -jobs: - deploy-hub-images-staging: - if: github.event_name == 'push' && github.ref == 'refs/heads/staging' - runs-on: ubuntu-latest - steps: - - name: Get PR labels - id: pr-labels - uses: irby/get-labels-on-push@v1.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check to see if the base jupyterhub image has changed, and deploy all hubs to prod if it has - run: | - echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done - - - name: Check out the image repo - if: ${{ env.DEPLOY }} - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Setup python - if: ${{ env.DEPLOY }} - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - if: ${{ env.DEPLOY }} - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - - name: Auth to gcloud - if: ${{ env.DEPLOY }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GKE_KEY }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Install Google Cloud SDK - if: ${{ env.DEPLOY }} - uses: google-github-actions/setup-gcloud@v2 - with: - install_components: 'gke-gcloud-auth-plugin' - - - name: Install SOPS - if: ${{ env.DEPLOY }} - run: | - mkdir -p ${HOME}/bin - curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops - chmod 755 ${HOME}/bin/sops - echo "${HOME}/bin" >> $GITHUB_PATH - - - name: Store SOPS secret in a file - if: ${{ env.DEPLOY }} - run: | - cat << EOF > ${HOME}/sops.key - ${{ secrets.SOPS_KEY }} - EOF - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV - - - name: Install Helm - if: ${{ env.DEPLOY }} - run: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ - helm repo update - - - name: Deploy base hub images to staging - if: ${{ env.DEPLOY }} - run: | - ignored_directories=("template") # these are directories that we never want to deploy - while read deployment; do - for ignored in "${ignored_directories[@]}"; do - if [[ "${deployment}" == "${ignored}" ]]; then - continue 2 # skip to the next iteration of "while read deployment" - fi - done - echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --verbose ${deployment} hub staging" - done < <(ls deployments/ | sed -e 's,/,,g') - - deploy-hub-images-prod: - if: github.event_name == 'push' && github.ref == 'refs/heads/prod' - runs-on: ubuntu-latest - steps: - - name: Get PR labels - id: pr-labels - uses: irby/get-labels-on-push@v1.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check to see if the base jupyterhub image has changed, and deploy all hubs to prod if it has - run: | - echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do - if [[ "$label" == jupyterhub-deployment || "$label" == hub-images ]]; then - echo "DEPLOY=1" >> $GITHUB_ENV - fi - done - - - name: Check out the image repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Setup python - if: ${{ env.DEPLOY }} - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - if: ${{ env.DEPLOY }} - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install --force-reinstall git+https://github.com/shaneknapp/hubploy.git@major-refactor - - - name: Auth to gcloud - if: ${{ env.DEPLOY }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GKE_KEY }} - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Install Google Cloud SDK - if: ${{ env.DEPLOY }} - uses: google-github-actions/setup-gcloud@v2 - with: - install_components: 'gke-gcloud-auth-plugin' - - - name: Install SOPS - if: ${{ env.DEPLOY }} - run: | - mkdir -p ${HOME}/bin - curl -sSL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o ${HOME}/bin/sops - chmod 755 ${HOME}/bin/sops - echo "${HOME}/bin" >> $GITHUB_PATH - - - name: Store SOPS secret in a file - if: ${{ env.DEPLOY }} - run: | - cat << EOF > ${HOME}/sops.key - ${{ secrets.SOPS_KEY }} - EOF - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/sops.key" >> $GITHUB_ENV - - - name: Install Helm - if: ${{ env.DEPLOY }} - run: | - curl -L https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar -xzf - - mv linux-amd64/helm /usr/local/bin - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ - helm repo update - - - name: Deploy base hub images to prod - if: ${{ env.DEPLOY }} - run: | - ignored_directories=("template") # these are directories that we never want to deploy - while read deployment; do - for ignored in "${ignored_directories[@]}"; do - if [[ "${deployment}" == "${ignored}" ]]; then - continue 2 # skip to the next iteration of "while read deployment" - fi - done - echo "Pretending to deploy base hub image to ${deployment} :P" - echo "hubploy deploy --verbose ${deployment} hub prod" - done < <(ls deployments/ | sed -e 's,/,,g') diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 807240013..a072ba509 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -1,9 +1,5 @@ -# this workflow re-deploys SPECIFIC hubs to staging or prod if the single-user -# server image or config has changed based on the PR labels "hub: ". -# -# however, this workflow will be not run if the PR labels of "hub-images" or -# "jupyterhub-deployment" are present, as these labels will trigger the -# "deploy-jupyterhub-base-images.yaml" workflow which re-deploys every hub. +# This workflow will determine if the base hub image and/or single-user server +# image for any or all hubs has has changed, and if so, deploy accordingly. # name: Deploy staging and prod hubs on: @@ -24,25 +20,24 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Pull out any hubs that need deploying from the labels on the merge commit to staging + - name: Check for hubs that need deploying from the labels on the merge commit to staging run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - HUBS=() - # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this - # means the base hub image has changed, and all hubs (staging or prod) need to - # be redeployed. The rest of this job will not run in that case. - if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then - echo "Base hub image has changed, not deploying individual hubs to staging" + # If the PR labels "hub-images" or "jupyterhub-deployment" are + # present, this means the base hub image has changed, and all hubs + # (staging or prod) need to be redeployed. + # + if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then + echo "DEPLOY=1" >> $GITHUB_ENV + # Otherwise, check to see if the PR labels contain any hubs, and + # deploy just those hubs to staging. + # else - # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then - hub_name=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$hub_name " echo "DEPLOY=1" >> $GITHUB_ENV fi done - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi - name: Check out the image repo @@ -104,10 +99,10 @@ jobs: - name: Deploy hubs to staging if: ${{ env.DEPLOY }} run: | - for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do - echo "Deploying $hub to staging" - echo "hubploy --verbose deploy $hub hub staging" - done + while read deployment; do + echo "Pretending to deploy base hub image to ${deployment} :P" + hubploy --verbose deploy --timeout 30m ${deployment} hub staging + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) deploy-hubs-to-prod: if: github.event_name == 'push' && github.ref == 'refs/heads/prod' @@ -119,25 +114,24 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Pull out any hubs that need deploying from the labels on the merge commit to prod + - name: Check for hubs that need deploying from the labels on the merge commit to prod run: | echo "PR labels: ${{ steps.pr-labels.outputs.labels }}" - HUBS=() - # If the PR labels "hub-images" or "jupyterhub-deployment" are present, this - # means the base hub image has changed, and all hubs (staging or prod) need to - # be redeployed. The rest of this job will not run in that case. - if [ -c "${GITHUB_PR_LABEL_HUB_IMAGES}" ] || [ -c "${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT}" ]; then - echo "Base hub image has changed, not deploying individual hubs to prod" + # If the PR labels "hub-images" or "jupyterhub-deployment" are + # present, this means the base hub image has changed, and all hubs + # (staging or prod) need to be redeployed. + # + if [[ -n ${GITHUB_PR_LABEL_HUB_IMAGES} || -n ${GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT} ]]; then + echo "DEPLOY=1" >> $GITHUB_ENV + # Otherwise, check to see if the PR labels contain any hubs, and + # deploy just those hubs to prod. + # else - # deploy any hubs that have been labeled for deployment for label in $(echo -e "${{ steps.pr-labels.outputs.labels }}"); do if [[ "$label" == hub-* ]]; then - hub_name=$(echo $label | awk -F'-' '{print $2}') - HUBS+="$hub_name " echo "DEPLOY=1" >> $GITHUB_ENV fi done - echo "DEPLOY_HUBS=${HUBS[@]}" >> $GITHUB_ENV fi - name: Check out the image repo @@ -199,7 +193,7 @@ jobs: - name: Deploy hubs to prod if: ${{ env.DEPLOY }} run: | - for hub in $(echo -e "${{ env.DEPLOY_HUBS }}"); do - echo "Deploying $hub to prod" - echo "hubploy --verbose deploy $hub hub prod" - done + while read deployment; do + echo "Pretending to deploy base hub image to ${deployment} :P" + hubploy --verbose deploy --timeout 30m ${deployment} hub prod + done < <(python .github/scripts/determine-hub-deployments.py --only-deploy logodev) diff --git a/deployments/logodev/hubploy.yaml b/deployments/logodev/hubploy.yaml index d532f0a5b..893cc72ba 100644 --- a/deployments/logodev/hubploy.yaml +++ b/deployments/logodev/hubploy.yaml @@ -1,7 +1,7 @@ images: images: # temporary update - - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:c44b36bfede9 + - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/logodev-user-image:cb7832b0a131 cluster: provider: gcloud diff --git a/deployments/workshop/config/common.yaml b/deployments/workshop/config/common.yaml index 1efc418de..02710f096 100644 --- a/deployments/workshop/config/common.yaml +++ b/deployments/workshop/config/common.yaml @@ -27,6 +27,14 @@ jupyterhub: JupyterHub: admin_access: false authenticator_class: dummy + Authenticator: + admin_users: + # infrastructure + - rylo + - sknapp + - felder + - balajialwar + - gmerritt proxy: chp: nodeSelector: diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 2c3932b5b..fb8d0ed88 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -135,7 +135,7 @@ nodePools: requests: # Some value slightly lower than allocatable RAM on the nodepool memory: 60929654784 - replicas: 1 + replicas: 2 dev: nodeSelector: hub.jupyter.org/pool-name: dev-pool