diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index e57c8f5023..56f4f92991 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -19,7 +19,6 @@ runs: - name: Setup dependencies run: | python3 -m pip install -r requirements.txt - python3 -m pip install -r dev-requirements.txt shell: bash - name: Deploy support components run: | diff --git a/.github/workflows/approved_prs_reminder.yaml b/.github/workflows/approved_prs_reminder.yaml index 8f077b7744..9709f7fb7a 100644 --- a/.github/workflows/approved_prs_reminder.yaml +++ b/.github/workflows/approved_prs_reminder.yaml @@ -15,7 +15,7 @@ jobs: create-sync-issue: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - name: Install python run: pip install ghapi pandas diff --git a/.github/workflows/deploy-grafana-dashboards.yaml b/.github/workflows/deploy-grafana-dashboards.yaml index 2c9832b5a7..7296a2fc99 100644 --- a/.github/workflows/deploy-grafana-dashboards.yaml +++ b/.github/workflows/deploy-grafana-dashboards.yaml @@ -17,12 +17,11 @@ jobs: - cluster_name: utoronto steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup dependencies run: | python3 -m pip install -r requirements.txt - python3 -m pip install -r dev-requirements.txt sudo apt install jsonnet - name: Setup gcloud diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index feeb54c64c..9e1154476d 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -52,7 +52,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Check if any of our base files have changed uses: dorny/paths-filter@v2 @@ -63,7 +63,6 @@ jobs: - "deployer/**" - "helm-charts/**" - "requirements.txt" - - "dev-requirements.txt" - "config/secrets.yaml" - ".github/workflows/deploy-hubs.yaml" - ".github/actions/deploy/*" diff --git a/.github/workflows/doc-links.yml b/.github/workflows/doc-links.yml index 9f782bc0b1..727a9f50f0 100644 --- a/.github/workflows/doc-links.yml +++ b/.github/workflows/doc-links.yml @@ -12,12 +12,7 @@ jobs: linkcheck: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 - with: - # chartpress is used by doc/conf.py, - # and requires information about the latest tagged commit, which - # requires the git history. - fetch-depth: 0 + - uses: actions/checkout@v3 - name: Install environment uses: conda-incubator/setup-miniconda@v2 diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml index 808522b6ba..afd853bbe6 100644 --- a/.github/workflows/python-lint.yaml +++ b/.github/workflows/python-lint.yaml @@ -9,7 +9,7 @@ jobs: flake8_py3: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: diff --git a/.github/workflows/terraform-lint.yaml b/.github/workflows/terraform-lint.yaml index 149c1e8b57..86c0b6ffe9 100644 --- a/.github/workflows/terraform-lint.yaml +++ b/.github/workflows/terraform-lint.yaml @@ -9,7 +9,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: hashicorp/setup-terraform@v1 - uses: terraform-linters/setup-tflint@v1 - name: Run terraform linter diff --git a/.github/workflows/validate-clusters.yaml b/.github/workflows/validate-clusters.yaml new file mode 100644 index 0000000000..8424984830 --- /dev/null +++ b/.github/workflows/validate-clusters.yaml @@ -0,0 +1,96 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. ref: +# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +# +# Runs the deployer script to validate clusters. This will both validate +# cluster.yaml files as well as each hubs passed non-encrypted values files +# against the Helm charts' values schema. +# +name: Validate clusters + +on: + pull_request: + paths: + - config/clusters/** + - deployer/** + - helm-charts/basehub/** + - helm-charts/daskhub/** + - requirements.txt + - .github/workflows/validate-hubs.yaml + push: + paths: + - config/clusters/** + - deployer/** + - helm-charts/basehub/** + - helm-charts/daskhub/** + - requirements.txt + - .github/workflows/validate-hubs.yaml + branches-ignore: + - "dependabot/**" + - "pre-commit-ci-update-config" + tags: + - "**" + workflow_dispatch: + +jobs: + validate-hubs-values-files: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - cluster_name: 2i2c + - cluster_name: azure.carbonplan + - cluster_name: carbonplan + - cluster_name: cloudbank + - cluster_name: farallon + - cluster_name: meom-ige + - cluster_name: openscapes + - cluster_name: pangeo-hubs + - cluster_name: utoronto + - cluster_name: uwhackweeks + + steps: + - uses: actions/checkout@v3 + + - name: Check if any cluster common files has changed + uses: dorny/paths-filter@v2 + id: cluster_common_files + with: + filters: | + files: + - deployer/** + - helm-charts/basehub/** + - helm-charts/daskhub/** + - requirements.txt + - .github/workflows/validate-hubs.yaml + + - name: Check if cluster specific files has changes + uses: dorny/paths-filter@v2 + id: cluster_specific_files + with: + filters: | + changes: + - config/clusters/${{ matrix.cluster_name }}/** + + # To continue this cluster specific job we must either have manually + # invoked this workflow to run for all clusters, or there should have been + # changes to the cluster common files or cluster specific files. + - name: Decide if the job should continue + id: decision + run: | + echo ::set-output name=continue-job::${{ github.event_name == 'workflow_dispatch' || (steps.cluster_common_files.outputs.files == 'true' || steps.cluster_specific_files.outputs.changes == 'true') }} + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + + - name: Install deployer script dependencies + run: | + pip install -r requirements.txt + + - name: "Validate cluster: ${{ matrix.cluster_name }}" + if: steps.decision.outputs.continue-job == 'true' + env: + TERM: xterm + run: | + python deployer validate ${{ matrix.cluster_name }} diff --git a/deployer/hub.py b/deployer/hub.py index 505e8b33d0..74fd262383 100644 --- a/deployer/hub.py +++ b/deployer/hub.py @@ -126,7 +126,7 @@ def deploy_support(self): print_colour("Provisioning support charts...") support_dir = (Path(__file__).parent.parent).joinpath("helm-charts", "support") - subprocess.check_call(["helm", "dep", "up", support_dir]) + subprocess.check_call(["helm", "dep", "up", "--skip-refresh", support_dir]) support_secrets_file = support_dir.joinpath("enc-support.secret.yaml") with tempfile.NamedTemporaryFile(mode="w") as f, get_decrypted_file( diff --git a/deployer/utils.py b/deployer/utils.py index 90164eed8c..f656e61697 100644 --- a/deployer/utils.py +++ b/deployer/utils.py @@ -181,11 +181,11 @@ def prepare_helm_charts_dependencies_and_schemas(): """ basehub_dir = helm_charts_dir.joinpath("basehub") _generate_values_schema_json(basehub_dir) - subprocess.check_call(["helm", "dep", "up", basehub_dir]) + subprocess.check_call(["helm", "dep", "up", "--skip-refresh", basehub_dir]) daskhub_dir = helm_charts_dir.joinpath("daskhub") _generate_values_schema_json(daskhub_dir) - subprocess.check_call(["helm", "dep", "up", daskhub_dir]) + subprocess.check_call(["helm", "dep", "up", "--skip-refresh", daskhub_dir]) def print_colour(msg: str): diff --git a/dev-requirements.txt b/dev-requirements.txt index 4f81875cca..5d92dc7bd2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,13 @@ -pytest -pytest-asyncio +# These requirements represents the needs for doing various tasks in this git +# repo besides using the deployer script. +# + +# chartpress is relevant to build and push helm-charts/images/hub/Dockerfile and +# update basehub's default values to reference the new image. +chartpress + +# requests is used by extra_scripts/rsync-active-users.py requests -beautifulsoup4 + +# rich is used by extra_scripts/count-auth0-apps.py rich diff --git a/docs/reference/ci-cd.md b/docs/reference/ci-cd.md index 7646574c16..d3c2deb681 100644 --- a/docs/reference/ci-cd.md +++ b/docs/reference/ci-cd.md @@ -11,7 +11,6 @@ following paths are modified: - deployer/** - helm-charts/** - requirements.txt -- dev-requirements.txt - config/secrets.yaml - config/clusters/** - .github/workflows/deploy-hubs.yaml diff --git a/requirements.txt b/requirements.txt index c72318ec25..9e172679bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,18 @@ -backoff -chartpress +# This file represents the needs for the deployer script to function, while the +# dev-requirements.txt file represents the needs in this repo in general. +# + +# ruamel.yaml is used to read and write .yaml files. ruamel.yaml + +# auth0 is used to communicate with Auth0's REST API that we integrate with in +# various ways. auth0-python -jhub-client==0.1.4 + +# jsonschema is used for validating cluster.yaml configurations jsonschema + +# jhub_client, pytest, and pytest_asyncio are used for our health checks +jhub-client==0.1.4 +pytest +pytest-asyncio