Bump actions/checkout from 2 to 3

.github/actions/deploy/action.yml

@@ -19,7 +19,6 @@ runs:
     - name: Setup dependencies
       run: |
         python3 -m pip install -r requirements.txt
-        python3 -m pip install -r dev-requirements.txt
       shell: bash
     - name: Deploy support components
       run: |

.github/workflows/approved_prs_reminder.yaml

@@ -15,7 +15,7 @@ jobs:
   create-sync-issue:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
       - name: Install python
         run: pip install ghapi pandas

.github/workflows/deploy-grafana-dashboards.yaml

@@ -17,12 +17,11 @@ jobs:
           - cluster_name: utoronto
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup dependencies
         run: |
           python3 -m pip install -r requirements.txt
-          python3 -m pip install -r dev-requirements.txt
           sudo apt install jsonnet
 
       - name: Setup gcloud

.github/workflows/deploy-hubs.yaml

@@ -52,7 +52,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Check if any of our base files have changed
         uses: dorny/paths-filter@v2
@@ -63,7 +63,6 @@ jobs:
               - "deployer/**"
               - "helm-charts/**"
               - "requirements.txt"
-              - "dev-requirements.txt"
               - "config/secrets.yaml"
               - ".github/workflows/deploy-hubs.yaml"
               - ".github/actions/deploy/*"

.github/workflows/doc-links.yml

@@ -12,12 +12,7 @@ jobs:
   linkcheck:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-        with:
-          # chartpress is used by doc/conf.py,
-          # and requires information about the latest tagged commit, which
-          # requires the git history.
-          fetch-depth: 0
+      - uses: actions/checkout@v3
 
       - name: Install environment
         uses: conda-incubator/setup-miniconda@v2

.github/workflows/python-lint.yaml

@@ -9,7 +9,7 @@ jobs:
   flake8_py3:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v3
       - name: Setup Python
         uses: actions/setup-python@v3
         with:

.github/workflows/terraform-lint.yaml

@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: hashicorp/setup-terraform@v1
       - uses: terraform-linters/setup-tflint@v1
       - name: Run terraform linter

.github/workflows/validate-clusters.yaml

@@ -0,0 +1,96 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps. ref:
+# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+#
+# Runs the deployer script to validate clusters. This will both validate
+# cluster.yaml files as well as each hubs passed non-encrypted values files
+# against the Helm charts' values schema.
+#
+name: Validate clusters
+
+on:
+  pull_request:
+    paths:
+      - config/clusters/**
+      - deployer/**
+      - helm-charts/basehub/**
+      - helm-charts/daskhub/**
+      - requirements.txt
+      - .github/workflows/validate-hubs.yaml
+  push:
+    paths:
+      - config/clusters/**
+      - deployer/**
+      - helm-charts/basehub/**
+      - helm-charts/daskhub/**
+      - requirements.txt
+      - .github/workflows/validate-hubs.yaml
+    branches-ignore:
+      - "dependabot/**"
+      - "pre-commit-ci-update-config"
+    tags:
+      - "**"
+  workflow_dispatch:
+
+jobs:
+  validate-hubs-values-files:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - cluster_name: 2i2c
+          - cluster_name: azure.carbonplan
+          - cluster_name: carbonplan
+          - cluster_name: cloudbank
+          - cluster_name: farallon
+          - cluster_name: meom-ige
+          - cluster_name: openscapes
+          - cluster_name: pangeo-hubs
+          - cluster_name: utoronto
+          - cluster_name: uwhackweeks
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check if any cluster common files has changed
+        uses: dorny/paths-filter@v2
+        id: cluster_common_files
+        with:
+          filters: |
+            files:
+              - deployer/**
+              - helm-charts/basehub/**
+              - helm-charts/daskhub/**
+              - requirements.txt
+              - .github/workflows/validate-hubs.yaml
+
+      - name: Check if cluster specific files has changes
+        uses: dorny/paths-filter@v2
+        id: cluster_specific_files
+        with:
+          filters: |
+            changes:
+              - config/clusters/${{ matrix.cluster_name }}/**
+
+      # To continue this cluster specific job we must either have manually
+      # invoked this workflow to run for all clusters, or there should have been
+      # changes to the cluster common files or cluster specific files.
+      - name: Decide if the job should continue
+        id: decision
+        run: |
+          echo ::set-output name=continue-job::${{ github.event_name == 'workflow_dispatch' || (steps.cluster_common_files.outputs.files == 'true' || steps.cluster_specific_files.outputs.changes == 'true') }}
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
+      - name: Install deployer script dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: "Validate cluster: ${{ matrix.cluster_name }}"
+        if: steps.decision.outputs.continue-job == 'true'
+        env:
+          TERM: xterm
+        run: |
+          python deployer validate ${{ matrix.cluster_name }}

deployer/hub.py

@@ -126,7 +126,7 @@ def deploy_support(self):
         print_colour("Provisioning support charts...")
 
         support_dir = (Path(__file__).parent.parent).joinpath("helm-charts", "support")
-        subprocess.check_call(["helm", "dep", "up", support_dir])
+        subprocess.check_call(["helm", "dep", "up", "--skip-refresh", support_dir])
 
         support_secrets_file = support_dir.joinpath("enc-support.secret.yaml")
         with tempfile.NamedTemporaryFile(mode="w") as f, get_decrypted_file(

deployer/utils.py

@@ -181,11 +181,11 @@ def prepare_helm_charts_dependencies_and_schemas():
     """
     basehub_dir = helm_charts_dir.joinpath("basehub")
     _generate_values_schema_json(basehub_dir)
-    subprocess.check_call(["helm", "dep", "up", basehub_dir])
+    subprocess.check_call(["helm", "dep", "up", "--skip-refresh", basehub_dir])
 
     daskhub_dir = helm_charts_dir.joinpath("daskhub")
     _generate_values_schema_json(daskhub_dir)
-    subprocess.check_call(["helm", "dep", "up", daskhub_dir])
+    subprocess.check_call(["helm", "dep", "up", "--skip-refresh", daskhub_dir])
 
 
 def print_colour(msg: str):

dev-requirements.txt

@@ -1,5 +1,13 @@
-pytest
-pytest-asyncio
+# These requirements represents the needs for doing various tasks in this git
+# repo besides using the deployer script.
+#
+
+# chartpress is relevant to build and push helm-charts/images/hub/Dockerfile and
+# update basehub's default values to reference the new image.
+chartpress
+
+# requests is used by extra_scripts/rsync-active-users.py
 requests
-beautifulsoup4
+
+# rich is used by extra_scripts/count-auth0-apps.py
 rich

docs/reference/ci-cd.md

@@ -11,7 +11,6 @@ following paths are modified:
 - deployer/**
 - helm-charts/**
 - requirements.txt
-- dev-requirements.txt
 - config/secrets.yaml
 - config/clusters/**
 - .github/workflows/deploy-hubs.yaml

requirements.txt

@@ -1,6 +1,18 @@
-backoff
-chartpress
+# This file represents the needs for the deployer script to function, while the
+# dev-requirements.txt file represents the needs in this repo in general.
+#
+
+# ruamel.yaml is used to read and write .yaml files.
 ruamel.yaml
+
+# auth0 is used to communicate with Auth0's REST API that we integrate with in
+# various ways.
 auth0-python
-jhub-client==0.1.4
+
+# jsonschema is used for validating cluster.yaml configurations
 jsonschema
+
+# jhub_client, pytest, and pytest_asyncio are used for our health checks
+jhub-client==0.1.4
+pytest
+pytest-asyncio