diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..e2e3ef6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,43 @@ +--- +name: Bug Report +about: Report a bug to help us improve +title: '[Bug]: ' +labels: 'bug' +assignees: '' + +--- + +## Checked for duplicates + +> Have you checked for duplicate issue tickets? + +- Ex. Yes - I've already checked +- Ex. No - I haven't checked + +## Describe the bug + +> A clear and concise description of what the bug is. Plain-text snippets preferred but screenshots welcome. + +Ex. When I did [...] action, I noticed [...] + +## What did you expect? + +> A clear and concise description of what you expect to happen + +Ex. I expected [...] + +## Reproducible steps + +> How would we reproduce this bug? Please walk us through it step by step. Plain-text snippets preferred but screenshots welcome. + +1. +2. +3. + +## What is your environment? + +> Include any computer hardware, operating system, framework, browser, time-of-day or other contextual information related to your issue + +- Ex. Version of this software [e.g. vX.Y.Z] +- Ex. Operating System: [e.g. MacOSX with Docker Desktop vX.Y] +- ... diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..1ac76ec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,34 @@ +--- +name: New Feature +about: Suggest a new feature for us to implement +title: '[New Feature]: ' +labels: 'enhancement' +assignees: '' + +--- + +## Checked for duplicates + +> Have you checked for duplicate issue tickets? + +- Ex. Yes - I've already checked +- Ex. No - I haven't checked + +## Alternatives considered + +> Have you considered alternative solutions to your feature request? + +- Ex. Yes - and alternatives don't suffice +- Ex. No - I haven't considered + +## Related problems + +> Is your feature request related to any problems? Please help us understand if so, including linking to any other issue tickets. + +Ex. I'm frustrated when [...] happens as documented in issue-XYZ + +## Describe the feature request + +> A clear and concise description of your request. + +Ex. I need or want [...] diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml new file mode 100644 index 0000000..80090b4 --- /dev/null +++ b/.github/workflows/build_docker_images.yml @@ -0,0 +1,39 @@ +name: Build Docker Images + +on: + workflow_dispatch: + inputs: + tag: + description: "Docker Image Tag" + required: false + default: "development-tag" + +env: + REGISTRY: ghcr.io + TAG: ${{ github.event.inputs.tag }} + SPS_AIRFLOW: ${{ github.repository }}/sps-airflow + +jobs: + build-sps-airflow: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for SPS Airflow Docker image + id: metascheduler + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.SPS_AIRFLOW }} + - name: Build and push SPS Airflow Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + file: airflow/docker/custom_airflow/Dockerfile + push: true + tags: ${{ env.REGISTRY }}/${{ env.SPS_AIRFLOW }}:${{ env.TAG }} + labels: ${{ steps.metascheduler.outputs.labels }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..b53c7c2 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,34 @@ +name: pre-commit + +on: + pull_request: + workflow_dispatch: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: '3.11' + - name: Read .terraform-version + id: tf_version + run: echo "TF_VERSION=$(cat .terraform-version)" >> $GITHUB_ENV + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + - name: Install Hadolint for pre-commit hook + run: | + wget -O /usr/local/bin/hadolint https://github.com/hadolint/hadolint/releases/latest/download/hadolint-Linux-x86_64 + chmod +x /usr/local/bin/hadolint + - name: Install dependencies for Python tests + run: | + pip install -e ".[test]" + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files + - name: Test with pytest + run: | + pytest -vv --gherkin-terminal-reporter unity-test/unit diff --git a/.github/workflows/regression_test_mcp_venue_dev.yml b/.github/workflows/regression_test_mcp_venue_dev.yml new file mode 100644 index 0000000..d3a4eb2 --- /dev/null +++ b/.github/workflows/regression_test_mcp_venue_dev.yml @@ -0,0 +1,38 @@ +name: U-SPS Regression Test for MCP Venue Dev + +on: + schedule: + # Runs around midnight Pacific Time (UTC-7) + # Doesn't account for daylight saving transitions + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + AIRFLOW_ENDPOINT: + description: "Base URL for the Airflow endpoint (i.e. http://abc.def.ghi:port-number)" + type: string + +jobs: + print_inputs: + runs-on: ubuntu-latest + steps: + - run: | + echo "Base URL for the Airflow endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.AIRFLOW_ENDPOINT || vars.MCP_DEV_AIRFLOW_ENDPOINT }}" + regression_test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + working-directory: ${{ github.workspace }}/unity-test + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt --use-pep517 + - name: Run the regression test + working-directory: ${{ github.workspace }}/unity-test + run: > + pytest -s -vv --gherkin-terminal-reporter + unity-test/system/smoke + --airflow-endpoint=${{ github.event.inputs.AIRFLOW_ENDPOINT || vars.MCP_DEV_AIRFLOW_ENDPOINT }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..571aeb0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,245 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +.idea +.idea/ + +.vscode +.vscode/ + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/.markdownlintrc b/.markdownlintrc new file mode 100644 index 0000000..7a2f7ac --- /dev/null +++ b/.markdownlintrc @@ -0,0 +1,16 @@ +{ + "default": true, + "MD029": { + "style": "ordered" + }, + "MD033": false, + "MD013": false, + "MD002": false, + "MD026": false, + "MD041": false, + "MD005": false, + "MD007": false, + "MD034": false, + "MD024": false, + "MD045": false +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ed1fddb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,84 @@ +fail_fast: true +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-xml + - id: check-added-large-files + args: + - --maxkb=50000 + - id: check-json # Checks json files for parsable syntax. + - id: pretty-format-json # Sets a standard for formatting json files. + args: + - --autofix + - id: requirements-txt-fixer # Sorts entries in requirements.txt. + - id: check-ast # Simply checks whether the files parse as valid python. + - id: detect-private-key # Detects the presence of private keys. + - id: detect-aws-credentials # Detects *your* aws credentials from the aws cli credentials file. + args: + - --allow-missing-credentials + - id: check-toml # Checks toml files for parsable syntax. + + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: "v0.39.0" + hooks: + - id: markdownlint + args: ["--config", ".markdownlintrc", "--ignore", "CHANGELOG.md"] + + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.1.1 + hooks: + - id: black + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.2.1 + hooks: + - id: ruff + + - repo: https://github.com/PyCQA/bandit + rev: "1.7.7" # you must change this to newest version + hooks: + - id: bandit + args: + [ + "--configfile=pyproject.toml", + "--severity-level=high", + "--confidence-level=high", + ] + additional_dependencies: [".[toml]"] + + - repo: https://github.com/hadolint/hadolint + rev: v2.12.1-beta + hooks: + - id: hadolint # requires hadolint is installed (brew install hadolint) + args: + - --no-color + - --failure-threshold=error + - --verbose + + # - repo: https://github.com/antonbabenko/pre-commit-terraform + # rev: v1.86.0 + # hooks: + # - id: terraform_validate # Validates all Terraform configuration files. + # args: + # - --tf-init-args=-upgrade + # - id: terraform_fmt # Rewrites all Terraform configuration files to a canonical format. + # - id: terraform_tflint # Validates all Terraform configuration files with TFLint. + # - id: terraform_trivy # Static analysis of Terraform templates to spot potential security issues. + # args: + # - > + # --args=--severity=CRITICAL + # --skip-dirs="**/.terraform" + # --tf-exclude-downloaded-modules + # - id: terraform_docs + # args: + # - --hook-config=--add-to-existing-file=true + # - --hook-config=--create-file-if-not-exist=true diff --git a/.terraform-version b/.terraform-version new file mode 100644 index 0000000..10c0880 --- /dev/null +++ b/.terraform-version @@ -0,0 +1 @@ +1.7.4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 835c17d..2f04957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,7 +91,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - https://unity-sds.gitbook.io/docs/developer-docs/science-processing/docs/developers-guide/the-jobs-database#how-jobs-are-updated-in-the-database ## Deployments - + -------- # [Unity Release 23.2] - 2023-07-14 @@ -120,7 +120,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [unity-sps-prototype #193) [New Feature]: Create SNS topic, SQS queue and Lambda function for Jobs Database as part of U-SPS deployment - [https://github.com/unity-sds/unity-sps-prototype/issues/193](https://github.com/unity-sds/unity-sps-prototype/issues/193) - [unity-sps-prototype $194] [Bug]: Prevent duplicate documents in jobs database - - [https://github.com/unity-sds/unity-sps-prototype/issues/194](https://github.com/unity-sds/unity-sps-prototype/issues/194) + - [https://github.com/unity-sds/unity-sps-prototype/issues/194](https://github.com/unity-sds/unity-sps-prototype/issues/194) - EPIC #10: `chirp-workflow-execution` - [unity-sps-workflows #8] [Dependency]: Stub implementation for CHIRP workflow - [https://github.com/unity-sds/unity-sps-workflows/issues/8](https://github.com/unity-sds/unity-sps-workflows/issues/8) @@ -136,7 +136,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - EPIC #4: `no magic` - [unity-sps-prototype #132] As a user, i want to be explicit about inputs into my process execution (no magic)! - [https://github.com/unity-sds/unity-sps-prototype/issues/132](https://github.com/unity-sds/unity-sps-prototype/issues/132) - + ## Docker Containers - ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0 - ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0 @@ -168,7 +168,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MCP Dev: - Processing Endpoint (WPS-T API): **http://aa17aedf4454a4cc596a67a1efb73411-1350404365.us-west-2.elb.amazonaws.com:5001** - Scaling Endpoint (SPS API): **http://a440158f49fab4278bdcf2bcb145082b-625745.us-west-2.elb.amazonaws.com:5002** - + -------- # [Unity Release 23.1] - 2023-04-11 @@ -176,7 +176,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - unity-sps : https://github.com/unity-sds/unity-sps/releases/tag/1.0.0 - unity-sps-prototype: https://github.com/unity-sds/unity-sps-prototype/releases/tag/1.0.0 - unity-sps-api: https://github.com/unity-sds/unity-sps-api/releases/tag/1.0.0 -- unity-sps-register_job: +- unity-sps-register_job: - MCP-Dev: https://github.com/unity-sds/unity-sps-register_job/releases/tag/1.0.0-MCP_Dev - MCP-Test: https://github.com/unity-sds/unity-sps-register_job/releases/tag/1.0.0-MCP_Test - unity-sps-workflows: https://github.com/unity-sds/unity-sps-workflows/releases/tag/1.0.0 @@ -194,7 +194,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - https://github.com/unity-sds/unity-sps-prototype/issues/154 - [unity-sps-prototype #159]: Return the input parameters as part of the WPS-T DescribeProcess method - https://github.com/unity-sds/unity-sps-prototype/issues/159 - - [unity-sps-prototype #167]: The WPS-T method to register a process may time out while building the PGE Docker image + - [unity-sps-prototype #167]: The WPS-T method to register a process may time out while building the PGE Docker image - https://github.com/unity-sds/unity-sps-prototype/issues/167 - EPIC #142: `processing-instance-types` - [unity-sps-prototype #142]: As a project manager, i want to set default compute types for processing nodes at deploy time, so that i can predict costs for the system (Static) @@ -206,7 +206,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - https://github.com/unity-sds/unity-sps-prototype/issues/170 - [unity-sps-prototype #167]: Scale the number of worker nodes in the Kubernetes cluster - https://github.com/unity-sds/unity-sps-prototype/issues/171 - + ## Docker Containers - ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.0.0 - ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.0.0 @@ -238,5 +238,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MCP Dev: - Processing Endpoint (WPS-T API): http://aa17aedf4454a4cc596a67a1efb73411-1350404365.us-west-2.elb.amazonaws.com:5001 - Scaling Endpoint (SPS API): http://a440158f49fab4278bdcf2bcb145082b-625745.us-west-2.elb.amazonaws.com:5002 - + ------------ diff --git a/README.md b/README.md index 3882d50..c28d20f 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ [Website](https://unity-sds.gitbook.io) | [Docs](https://unity-sds.gitbook.io/docs/developer-docs/science-processing) This repository contains high level information (such as documentation, change log, etc.) about the U-SPS software stack. The actual U-SPS code is contained within the following set of GitHub repositories: + * [U-SPS Prototype](https://github.com/unity-sds/unity-sps-prototype): Terraform scripts to deploy the U-SPS cluster (either the HySDS or Airflow implementations) * [U-SPS Workflows](https://github.com/unity-sds/unity-sps-workflows): Examples of CWL workflows that can be executed on a U-SPS cluster * [U-SPS API](https://github.com/unity-sds/unity-sps-api): The API used to manage a U-SPS cluster @@ -49,7 +50,7 @@ This guide provides a quick way to get started with our project. Please see our * Docker 20.10 or higher * Kubernetes 1.20 or higher * Terraform 0.14 or higher - + ### Setup Instructions 1. Follow the [U-SPS Setup Guide](https://unity-sds.gitbook.io/docs/developer-docs/science-processing/docs/admin-guide/cluster-provisioning-with-terraform). @@ -95,7 +96,8 @@ See our: [LICENSE](LICENSE) ## Support -Key points of contact are: +Key points of contact are: + * [@Luca Cinquini](https://github.com/LucaCinquini) * [@Namrata Malarout](https://github.com/NamrataM) * [@Drew Meyers](https://github.com/drewm-jpl) diff --git a/airflow/dags/cwltool_hello_world.py b/airflow/dags/cwltool_hello_world.py new file mode 100644 index 0000000..a05775e --- /dev/null +++ b/airflow/dags/cwltool_hello_world.py @@ -0,0 +1,27 @@ +from datetime import datetime + +from airflow.operators.bash import BashOperator + +from airflow import DAG + +default_args = { + "owner": "unity-sps", + "start_date": datetime.utcfromtimestamp(0), +} + +dag = DAG( + "cwltool_help_dag", + default_args=default_args, + description="A simple DAG to run cwltool --help", + schedule=None, + is_paused_upon_creation=False, + tags=["test"], +) + +run_cwl_help = BashOperator( + task_id="run_cwltool_help", + bash_command="cwltool --help", + dag=dag, +) + +run_cwl_help diff --git a/airflow/dags/hello_world.py b/airflow/dags/hello_world.py new file mode 100644 index 0000000..50d0a08 --- /dev/null +++ b/airflow/dags/hello_world.py @@ -0,0 +1,36 @@ +""" +# DAG Name: Hello World + +# Purpose + +# Usage +""" # noqa: E501 + +import time +from datetime import datetime + +from airflow.operators.python import PythonOperator + +from airflow import DAG + +default_args = { + "owner": "unity-sps", + "start_date": datetime.utcfromtimestamp(0), +} + + +def hello_world(): + print("Hello World") + time.sleep(30) + + +with DAG( + dag_id="hello_world", + doc_md=__doc__, + default_args=default_args, + schedule=None, + is_paused_upon_creation=False, + tags=["test"], +) as dag: + hello_world_task = PythonOperator(task_id="hello_world", python_callable=hello_world) + hello_world_task diff --git a/airflow/docker/custom_airflow/Dockerfile b/airflow/docker/custom_airflow/Dockerfile new file mode 100644 index 0000000..8620dbb --- /dev/null +++ b/airflow/docker/custom_airflow/Dockerfile @@ -0,0 +1,5 @@ +FROM apache/airflow:2.8.1-python3.11 + +COPY ./airflow/dags/ ${AIRFLOW_HOME}/dags/ + +RUN pip install cwltool==3.1.20240112164112 diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml new file mode 100644 index 0000000..4443fb6 --- /dev/null +++ b/airflow/helm/values.tmpl.yaml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Source of default values: https://github.com/apache/airflow/blob/main/chart/values.yaml + +# Airflow create user job settings +createUserJob: + # In case you need to disable the helm hooks that create the jobs after install. + # Disable this if you are using ArgoCD for example + useHelmHooks: false + applyCustomEnv: false + +# Airflow database migration job settings +migrateDatabaseJob: + # In case you need to disable the helm hooks that create the jobs after install. + # Disable this if you are using ArgoCD for example + useHelmHooks: false + applyCustomEnv: false + # To run database migrations with Argo CD automatically, you will need to add the + # following. This will run database migrations every time there is a Sync event + # in Argo CD. While it is not ideal to run the migrations on every sync, it is a + # trade-off that allows them to be run automatically. + jobAnnotations: + "argocd.argoproj.io/hook": Sync + +images: + airflow: + repository: ${airflow_image_repo} + tag: ${airflow_image_tag} + +scheduler: + replicas: 1 + +triggerer: + replicas: 1 + +dagProcessor: + enabled: true + replicas: 1 + +postgresql: + enabled: false + +pgbouncer: + enabled: true + replicas: 1 + +webserverSecretKeySecretName: ${webserver_secret_name} + +webserver: + replicas: 1 + +workers: + keda: + enabled: true + pollingInterval: 1 + maxReplicaCount: 128 + # Specify HPA related options + # https://github.com/kubernetes/enhancements/blob/master/keps/sig-autoscaling/853-configurable-hpa-scale-velocity/README.md + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleUp: + policies: + - type: Percent + value: 900 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 5 + +data: + metadataSecretName: ${metadata_secret_name} + resultBackendSecretName: ~ + +config: + logging: + remote_logging: true + logging_level: "INFO" + remote_base_log_folder: ${airflow_logs_s3_location} + remote_log_conn_id: "aws_default" + encrypt_s3_logs: false + celery: + worker_concurrency: 1 + +env: + - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE" + value: "${kubernetes_namespace}" + +# https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/security/api.html +extraEnv: | + - name: AIRFLOW__API__AUTH_BACKENDS + value: "airflow.api.auth.backend.basic_auth" + - name: AIRFLOW__CORE__PARALLELISM + value: "128" + - name: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG + value: "64" + - name: AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG + value: "64" + - name: AIRFLOW__SCHEDULER__MAX_DAGRUNS_TO_CREATE_PER_LOOP + value: "64" + - name: AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC + value: "1" + - name: AIRFLOW__KUBERNETES__WORKER_PODS_CREATION_BATCH_SIZE + value: "8" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1f07df4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,62 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "unity-sps" +version = "0.0.1-beta.1" +authors = [ + { name = "Drew Meyers", email = "drew.meyers@jpl.nasa.gov" }, +] +description = "The science processing service area of Unity." +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", +] + +[project.license] +text = "Apache-2.0" + +[project.readme] +file = "README.md" +content-type = "text/markdown" + +[project.urls] +Homepage = "https://github.com/unity-sds/unity-sps" + +[project.optional-dependencies] +develop = [] +test = [ + "python-dotenv==1.0.1", + "pytest-bdd==7.0.1", + "pytest-mock==3.12.0", + "requests==2.31.0", + "apache-airflow==2.8.1", + "kubernetes==29.0.0", + "boto3==1.34.46" +] +experiment = [] + +[tool.setuptools.packages.find] +exclude = ["tests*"] +namespaces = false + +[tool.isort] +line_length = 110 + +[tool.black] +line-length = 110 + +[tool.ruff] +line-length = 110 + +[tool.ruff.lint] +ignore = ["E501", "E402", "E731"] + +[tool.bandit.assert_used] +skips = ['*_test.py', '*/test_*.py'] diff --git a/terraform-unity/.terraform.lock.hcl b/terraform-unity/.terraform.lock.hcl new file mode 100644 index 0000000..8e8f845 --- /dev/null +++ b/terraform-unity/.terraform.lock.hcl @@ -0,0 +1,101 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.37.0" + constraints = "~> 5.0" + hashes = [ + "h1:jy1tY8vUGirfcC9GwSS2Uf01GXcxnNnotvIK/WjI2WI=", + "zh:00f40a3d9593476693a7a72d993fd289f7be374fe3f2799776c6296eb6ff890a", + "zh:1010a9fbf55852a8da3473de4ec0f1fcf29efa85d66f61cbe2b086dbbd7747ae", + "zh:103a5674d1eb1cff05fe35e9baa9875afd18d740868b63f9c0c25eadb5eb4eb7", + "zh:270ac1b7a1327c1456a43df44c0b5cc3e26ed6d8861a709adeea1da684a563f5", + "zh:424362c02c8917c0586f3dd49aca27b7e0c21f5a23374b7045e9be3b5646c028", + "zh:549fa2ea187964ab9a0c354310947ead30e09b3199db1ff377c21d7547d78299", + "zh:6492d2ccc7f7d60e83cd8b7244adc53f30efc17d84b1ffc1b8fd6c385f8255fd", + "zh:66fb7b3b8a357071d26c5996c16d426edf07502a05ac86f4a6f73646ee7d1bbb", + "zh:6ecc05fb466d06ea8945564d2cdb8c2a8827d8cfca1550e9fb7eac0e95920196", + "zh:7932360b627b211dad937d278a8692a6c52bd6c0a71e4ec9e94ccbe825053822", + "zh:97ed1b4a18842c4d56a735329e87b4ef91a47e820e5a5c3c2dd64e293408bfc8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:d5e022052011e1984b9c2f8bc5a6b05c909e3b5bf40c3baddf191bf90e3169c2", + "zh:d7e9488b2ce5904efb91c8577b3fe9b0cd599c4cd508f1f163f292930f54fdf0", + "zh:e57cd93d5cd81dd0f446076af6e47a53ce83df2947ec64ed39a1090d4bdf8f0b", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.12.1" + hashes = [ + "h1:aBfcqM4cbywa7TAxfT1YoFS+Cst9waerlm4XErFmJlk=", + "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", + "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", + "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", + "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", + "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", + "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", + "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", + "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", + "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", + "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", + "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.26.0" + hashes = [ + "h1:LxZk5Vc0TnfeLYnp7HXZui53PEV+gFd+mznBzdNm+po=", + "zh:3f8ee1bffab1ba4f6ae549daae1648974214880d3606b6821cb0aceb365284a4", + "zh:5596b1248231cc3b8f6a98f5b78df7120cd3153fd2b34b369dc20356a75bf35b", + "zh:64420c9e4aa49c5e443afcd60f3e8d293ea6bd78797d402e21e23605f7757954", + "zh:8327a488854e15f8d7eaf8272c3b9d6d1d9a6e68212a8dcb111d7b4023aac6b5", + "zh:94c1c9b65280847d28a3e90e5046650858ac0bf87feefd2349336444e21e68e8", + "zh:a3fb0b0b4bfd1844bb94011ae80111cedc188085235cf466313ca2151e75c8ca", + "zh:ab5e381928144e0c2a9d9768a48e38797642e5c5fb2184370c7c08df500e5db3", + "zh:da78995e8d6daf3acfd4c455ebbd12f6bf154cadf455f14ef35c0862e58dd2ec", + "zh:e24cdd5b90196df93215f40d821af3a7b4473c53992be4c3038940d117a50eb4", + "zh:e632efb3bce6d089b7c08507660af8b2c5e3f94c34fe401bfa228f154405e26e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f5aea9da0eba25d35fee49db193c4b44cd3746a5578065092c62a53077e50b84", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.0" + hashes = [ + "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", + "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", + "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", + "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", + "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", + "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", + "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", + "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", + "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", + "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", + "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + ] +} diff --git a/terraform-unity/README.md b/terraform-unity/README.md new file mode 100644 index 0000000..98ea24e --- /dev/null +++ b/terraform-unity/README.md @@ -0,0 +1,194 @@ +# Unity SPS Cluster Provisioning with Terraform + +## Development Workflow + +### Dev Requirements + +- [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) +- [tfenv](https://github.com/tfutils/tfenv) - Terraform version manager. +- [Pre-commit](https://pre-commit.com/) - Framework for managing and maintaining multi-language pre-commit hooks. +- [act](https://github.com/nektos/act) - Run Github Actions locally. +- [tflint](https://github.com/terraform-linters/tflint) - Terraform Linter. +- [terrascan](https://github.com/accurics/terrascan) - Static code analyzer for Infrastructure as Code. +- [tfsec](https://github.com/aquasecurity/tfsec) - Security scanner for Terraform code. +- [terraform-docs](https://github.com/terraform-docs/terraform-docs) - Generate documentation from Terraform modules. +- [Terratest](https://terratest.gruntwork.io) - Go library that provides patterns and helper functions for testing infrastructure, with 1st-class support for Terraform. + +### Auto-generate a terraform.tfvars template file + +```shell +cd terraform-unity +terraform-docs tfvars hcl . --output-file "terraform.tfvars" +``` + +```json + +celeryconfig_filename = "celeryconfig_remote.py" +counter = "" +datasets_filename = "datasets.remote.template.json" +deployment_environment = "mcp" +docker_images = { + "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v0.0.1", + "busybox": "k8s.gcr.io/busybox", + "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v0.0.1", + "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v0.0.1", + "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v0.0.1", + "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v0.0.1", + "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v0.0.1", + "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v0.0.1", + "logstash": "docker.elastic.co/logstash/logstash:7.10.2", + "mc": "minio/mc:RELEASE.2022-03-13T22-34-00Z", + "minio": "minio/minio:RELEASE.2022-03-17T06-34-49Z", + "rabbitmq": "rabbitmq:3-management", + "redis": "redis:latest" +} +kubeconfig_filepath = "" +mozart_es = { + "volume_claim_template": { + "storage_class_name": "gp2-sps" + } +} +namespace = "" +node_port_map = { + "ades_wpst_api_service": 30011, + "grq2_es": 30012, + "grq2_service": 30002, + "hysds_ui_service": 30009, + "minio_service_api": 30007, + "minio_service_interface": 30008, + "mozart_es": 30013, + "mozart_service": 30001 +} +service_type = "LoadBalancer" +venue = "" +% +``` + +## Deploy the Cluster + +### Deploying in into Different MCP Venues + +### Deploying into Different EKS Clusters + +This method will use Terraform to deploy the Kubernetes cluster represented by the `~/.kube/config` file which is referenced in `terraform-unity/main.tf`. Terraform will deploy the resources in the Kubernetes namespace named in `terrafrom/variables.tf` (defaults to `unity-sps`). Additional variables (including secrets) can be set in `terraform.tfvars`, a template is shown below. + +From within the Terraform root module directory (`terraform-unity/`), run the following commands to initialize, and apply the Terraform module: + +```bash +cd terraform-unity/ +terraform init +terraform apply +``` + +## Teardown the Cluster + +From within the Terraform root module directory (terraform-unity/), run the following command to destroy the SPS cluster: + +```shell +terraform destroy +``` + +## Prior to pushing new changes to the repo, please ensure that you done have the following and the checks have passed + +1. Run the pre-commit hooks. These hooks will perform static analysis, linting, security checks. The hooks will also reformat the code to conform to the style guide, and produce the auto-generated documentation of the Terraform module. + + ```shell + # Run all hooks: + pre-commit run --files terraform-modules/* + pre-commit run --files terraform-unity/* + # Run specific hook: + pre-commit run --files terraform-modules/terraform-unity-sps-hysds-cluster/*.tf + pre-commit run --files terraform-unity/*.tf + ``` + +2. Run the Github Actions locally. These actions include similar checks to the pre-commit hooks, however, the actions not have the ability to perform reformatting or auto-generation of documentation. This step is meant to mimic the Github Actions which run on the remote CI/CD pipeline. + + ```shell + # Run all actions: + act + # Run specific action: + act -j "" + act -j terraform_validate + act -j terraform_fmt + act -j terraform_tflint + act -j terraform_tfsec + act -j checkov + # You may need to authenticate with Docker hub in order to successfully pull some of the associated images. + act -j terraform_validate -s DOCKER_USERNAME= -s DOCKER_PASSWORD= + ``` + +3. Run the Terratest smoke test. At the moment, this represents a **_very_** basic smoke test for our deployment which simply checks the endpoints of the various services. + + ```shell + cd terraform-test + go test -v -run TestTerraformUnity -timeout 30m | tee terratest_output.txt + ``` + +## Debugging a Terraform Deployment + +It is often useful to modify the level of TF_LOG environment variable when debugging +a Terraform deployment. The levels include: `TRACE`, `DEBUG`, `INFO`, `WARN`, and `ERROR`. + +An example of setting the `TF_LOG` environment variable to `INFO`: + +```bash +export TF_LOG=INFO +``` + +Additionally, it is also often useful to pipe the output of a Terraform deployment into a log file. + +An example of piping the `terraform apply` output into a file named apply_output.txt: + +```bash +terraform apply -no-color 2>&1 | tee apply_output.txt +``` + +## Auto-generated Documentation of the Unity SPS Terraform Root Module + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.7.2 | +| [aws](#requirement\_aws) | 5.35.0 | +| [helm](#requirement\_helm) | 2.12.1 | +| [kubernetes](#requirement\_kubernetes) | 2.25.2 | +| [null](#requirement\_null) | 3.2.2 | +| [random](#requirement\_random) | 3.6.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [unity-sps-airflow](#module\_unity-sps-airflow) | ./modules/terraform-unity-sps-airflow | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | value | `string` | n/a | yes | +| [counter](#input\_counter) | value | `string` | `""` | no | +| [custom\_airflow\_docker\_image](#input\_custom\_airflow\_docker\_image) | Docker image for the customized Airflow image. | 
object({
    name = string
    tag  = string
  })
| 
{
  "name": "ghcr.io/unity-sds/unity-sps-prototype/sps-airflow",
  "tag": "develop"
}
| no | +| [eks\_cluster\_name](#input\_eks\_cluster\_name) | The name of the EKS cluster. | `string` | n/a | yes | +| [helm\_charts](#input\_helm\_charts) | Settings for the required Helm charts. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| 
{
  "airflow": {
    "chart": "airflow",
    "repository": "https://airflow.apache.org",
    "version": "1.11.0"
  },
  "keda": {
    "chart": "keda",
    "repository": "https://kedacore.github.io/charts",
    "version": "v2.13.1"
  }
}
| no | +| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | `"../k8s/kubernetes.yml"` | no | +| [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | +| [release](#input\_release) | The SPS release version | `string` | n/a | yes | +| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | +| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [load\_balancer\_hostnames](#output\_load\_balancer\_hostnames) | Load Balancer Ingress Hostnames | + diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf new file mode 100644 index 0000000..c512283 --- /dev/null +++ b/terraform-unity/main.tf @@ -0,0 +1,13 @@ +module "unity-sps-airflow" { + source = "./modules/terraform-unity-sps-airflow" + project = var.project + venue = var.venue + service_area = var.service_area + counter = var.counter + release = var.release + eks_cluster_name = var.eks_cluster_name + kubeconfig_filepath = var.kubeconfig_filepath + airflow_webserver_password = var.airflow_webserver_password + custom_airflow_docker_image = var.custom_airflow_docker_image + helm_charts = var.helm_charts +} diff --git a/terraform-unity/modules/README.md b/terraform-unity/modules/README.md new file mode 100644 index 0000000..cfd4be9 --- /dev/null +++ b/terraform-unity/modules/README.md @@ -0,0 +1,5 @@ +# Auto-generated Documentation of Unity SPS Terraform Sub-modules + +Each sub-module contained in this directory contains its own auto-generated documentation: + +- [terraform-unity-sps-hysds-cluster](terraform-unity-sps-hysds-cluster/README.md) diff --git a/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl b/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl new file mode 100644 index 0000000..da3772c --- /dev/null +++ b/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl @@ -0,0 +1,124 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.35.0" + constraints = ">= 3.72.0, >= 4.47.0, >= 4.57.0, 5.35.0" + hashes = [ + "h1:MKNFmhsOIirK7Qzr6TWkVaBcVGN81lCU0BPiaPOeQ8s=", + "zh:3a2a6f40db82d30ea8c5e3e251ca5e16b08e520570336e7e342be823df67e945", + "zh:420a23b69b412438a15b8b2e2c9aac2cf2e4976f990f117e4bf8f630692d3949", + "zh:4d8b887f6a71b38cff77ad14af9279528433e279eed702d96b81ea48e16e779c", + "zh:4edd41f8e1c7d29931608a7b01a7ae3d89d6f95ef5502cf8200f228a27917c40", + "zh:6337544e2ded5cf37b55a70aa6ce81c07fd444a2644ff3c5aad1d34680051bdc", + "zh:668faa3faaf2e0758bf319ea40d2304340f4a2dc2cd24460ddfa6ab66f71b802", + "zh:79ddc6d7c90e59fdf4a51e6ea822ba9495b1873d6a9d70daf2eeaf6fc4eb6ff3", + "zh:885822027faf1aa57787f980ead7c26e7d0e55b4040d926b65709b764f804513", + "zh:8c50a8f397b871388ff2e048f5eb280af107faa2e8926694f1ffd9f32a7a7cdf", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a2f5d2553df5573a060641f18ee7585587047c25ba73fd80617f59b5893d22b4", + "zh:c43833ae2a152213ee92eb5be7653f9493779eddbe0ce403ea49b5f1d87fd766", + "zh:dab01527a3a55b4f0f958af6f46313d775e27f9ad9d10bedbbfea4a35a06dc5f", + "zh:ed49c65620ec42718d681a7fc00c166c295ff2795db6cede2c690b83f9fb3e65", + "zh:f0a358c0ae1087c466d0fbcc3b4da886f33f881a145c3836ec43149878b86a1a", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.3" + constraints = ">= 2.0.0" + hashes = [ + "h1:GmJ8PxLjjPr+lh02Bw3u7RYqA3UtpE2hQ1T43Vt7PTQ=", + "zh:0bd6ee14ca5cf0f0c83d3bb965346b1225ccd06a6247e80774aaaf54c729daa7", + "zh:3055ad0dcc98de1d4e45b72c5889ae91b62f4ae4e54dbc56c4821be0fdfbed91", + "zh:32764cfcff0d7379ca8b7dde376ac5551854d454c5881945f1952b785a312fa2", + "zh:55c2a4dc3ebdeaa1dec3a36db96dab253c7fa10b9fe1209862e1ee77a01e0aa1", + "zh:5c71f260ba5674d656d12f67cde3bb494498e6b6b6e66945ef85688f185dcf63", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9617280a853ec7caedb8beb7864e4b29faf9c850a453283980c28fccef2c493d", + "zh:ac8bda21950f8dddade3e9bc15f7bcfdee743738483be5724169943cafa611f5", + "zh:ba9ab567bbe63dee9197a763b3104ea9217ba27449ed54d3afa6657f412e3496", + "zh:effd1a7e34bae3879c02f03ed3afa979433a518e11de1f8afd35a8710231ac14", + "zh:f021538c86d0ac250d75e59efde6d869bbfff711eb744c8bddce79d2475bf46d", + "zh:f1e3984597948a2103391a26600e177b19f16a5a4c66acee27a4343fb141571f", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.12.1" + hashes = [ + "h1:aBfcqM4cbywa7TAxfT1YoFS+Cst9waerlm4XErFmJlk=", + "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", + "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", + "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", + "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", + "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", + "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", + "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", + "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", + "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", + "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", + "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.25.2" + constraints = ">= 2.10.0" + hashes = [ + "h1:T1WAQt40cAk721H0AM/eZ5YuodJaIfS8r3Tu7rKCJJE=", + "zh:044788ac936e0e8ece8f78a2e4e366ecd435ea8235388eaf2cbc8e7975d9d970", + "zh:24f5ff01df91f51f00ee7ff39430adeb63bb2ca4ea0042e68f06d6b65808c02f", + "zh:49984aa0aa1faa8c4f01e8faa039322f1e6fdaeab0b7e32f5c6e96edfde36a38", + "zh:4eeceaff56bac9fc782e7e33f157fa2c7e9a47b2c3c3d12da2642c312ace73f6", + "zh:4f49b6419345960d5af475e0200c243af4c9c140b0ee64799fe1fc9b023c49ea", + "zh:7958414d516867a2263a978792a24843f80023fb233cf051ff4095adc9803d85", + "zh:c633a755fc95e9ff0cd73656f052947afd85883a0987dde5198113aa48474156", + "zh:cbfe958d119795004ce1e8001449d01c056fa2a062b51d07843d98be216337d7", + "zh:cfb85392e18768578d4c943438897083895719be678227fd90efbe3500702a56", + "zh:d705a661ed5da425dd236a48645bec39fe78a67d2e70e8460b720417cbf260ac", + "zh:ddd7a01263da3793df4f3b5af65f166307eed5acf525e51e058cda59009cc856", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.10.0" + constraints = ">= 0.9.0" + hashes = [ + "h1:NAl8eupFAZXCAbE5uiHZTz+Yqler55B3fMG+jNPrjjM=", + "zh:0ab31efe760cc86c9eef9e8eb070ae9e15c52c617243bbd9041632d44ea70781", + "zh:0ee4e906e28f23c598632eeac297ab098d6d6a90629d15516814ab90ad42aec8", + "zh:3bbb3e9da728b82428c6f18533b5b7c014e8ff1b8d9b2587107c966b985e5bcc", + "zh:6771c72db4e4486f2c2603c81dfddd9e28b6554d1ded2996b4cb37f887b467de", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:833c636d86c2c8f23296a7da5d492bdfd7260e22899fc8af8cc3937eb41a7391", + "zh:c545f1497ae0978ffc979645e594b57ff06c30b4144486f4f362d686366e2e42", + "zh:def83c6a85db611b8f1d996d32869f59397c23b8b78e39a978c8a2296b0588b2", + "zh:df9579b72cc8e5fac6efee20c7d0a8b72d3d859b50828b1c473d620ab939e2c7", + "zh:e281a8ecbb33c185e2d0976dc526c93b7359e3ffdc8130df7422863f4952c00e", + "zh:ecb1af3ae67ac7933b5630606672c94ec1f54b119bf77d3091f16d55ab634461", + "zh:f8109f13e07a741e1e8a52134f84583f97a819e33600be44623a21f6424d6593", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.5" + constraints = ">= 3.0.0" + hashes = [ + "h1:zeG5RmggBZW/8JWIVrdaeSJa0OG62uFX5HY1eE8SjzY=", + "zh:01cfb11cb74654c003f6d4e32bbef8f5969ee2856394a96d127da4949c65153e", + "zh:0472ea1574026aa1e8ca82bb6df2c40cd0478e9336b7a8a64e652119a2fa4f32", + "zh:1a8ddba2b1550c5d02003ea5d6cdda2eef6870ece86c5619f33edd699c9dc14b", + "zh:1e3bb505c000adb12cdf60af5b08f0ed68bc3955b0d4d4a126db5ca4d429eb4a", + "zh:6636401b2463c25e03e68a6b786acf91a311c78444b1dc4f97c539f9f78de22a", + "zh:76858f9d8b460e7b2a338c477671d07286b0d287fd2d2e3214030ae8f61dd56e", + "zh:a13b69fb43cb8746793b3069c4d897bb18f454290b496f19d03c3387d1c9a2dc", + "zh:a90ca81bb9bb509063b736842250ecff0f886a91baae8de65c8430168001dad9", + "zh:c4de401395936e41234f1956ebadbd2ed9f414e6908f27d578614aaa529870d4", + "zh:c657e121af8fde19964482997f0de2d5173217274f6997e16389e7707ed8ece8", + "zh:d68b07a67fbd604c38ec9733069fbf23441436fecf554de6c75c032f82e1ef19", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform-unity/modules/terraform-eks-cluster/README.md b/terraform-unity/modules/terraform-eks-cluster/README.md new file mode 100644 index 0000000..421c19f --- /dev/null +++ b/terraform-unity/modules/terraform-eks-cluster/README.md @@ -0,0 +1,42 @@ +# terraform-eks-cluster + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.7.2 | +| [aws](#requirement\_aws) | 5.35.0 | +| [kubernetes](#requirement\_kubernetes) | 2.25.2 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | 5.35.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [unity-eks](#module\_unity-eks) | git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module | 0.1.3 | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role_policy.sps_airflow_eks_inline_policy](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/iam_role_policy) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.auth](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster_auth) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/terraform-unity/modules/terraform-eks-cluster/data.tf b/terraform-unity/modules/terraform-eks-cluster/data.tf new file mode 100644 index 0000000..ddfcf84 --- /dev/null +++ b/terraform-unity/modules/terraform-eks-cluster/data.tf @@ -0,0 +1,17 @@ +data "aws_caller_identity" "current" {} + +data "aws_eks_cluster" "cluster" { + name = var.cluster_name + depends_on = [module.unity-eks] +} + +data "aws_eks_cluster_auth" "auth" { + name = var.cluster_name + depends_on = [module.unity-eks] +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.auth.token +} diff --git a/terraform-unity/modules/terraform-eks-cluster/main.tf b/terraform-unity/modules/terraform-eks-cluster/main.tf new file mode 100644 index 0000000..3b96d9d --- /dev/null +++ b/terraform-unity/modules/terraform-eks-cluster/main.tf @@ -0,0 +1,55 @@ +module "unity-eks" { + source = "git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=0.1.3" + deployment_name = var.cluster_name + + nodegroups = { + defaultGroup = { + instance_types = ["m5.xlarge"] + min_size = 1 + max_size = 1 + desired_size = 1 + } + } + aws_auth_roles = [{ + rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/mcp-tenantOperator" + username = "admin" + groups = ["system:masters"] + }] + + cluster_version = "1.27" +} + +# add extra policies as inline policy +resource "aws_iam_role_policy" "sps_airflow_eks_inline_policy" { + name = "sps_airflow_eks_inline_policy" + role = module.unity-eks.cluster_iam_role + policy = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.7.2 | +| [aws](#requirement\_aws) | 5.35.0 | +| [helm](#requirement\_helm) | 2.12.1 | +| [kubernetes](#requirement\_kubernetes) | 2.25.2 | +| [null](#requirement\_null) | 3.2.2 | +| [random](#requirement\_random) | 3.6.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | 5.36.0 | +| [helm](#provider\_helm) | 2.12.1 | +| [kubernetes](#provider\_kubernetes) | 2.25.2 | +| [null](#provider\_null) | 3.2.2 | +| [random](#provider\_random) | 3.6.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_db_instance.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/db_instance) | resource | +| [aws_db_subnet_group.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/db_subnet_group) | resource | +| [aws_s3_bucket.airflow_logs](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/s3_bucket) | resource | +| [aws_secretsmanager_secret.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/secretsmanager_secret) | resource | +| [aws_secretsmanager_secret_version.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/secretsmanager_secret_version) | resource | +| [aws_security_group.rds_sg](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/security_group) | resource | +| [aws_security_group_rule.eks_egress_to_rds](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.rds_ingress_from_eks](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/security_group_rule) | resource | +| [helm_release.airflow](https://registry.terraform.io/providers/hashicorp/helm/2.12.1/docs/resources/release) | resource | +| [helm_release.keda](https://registry.terraform.io/providers/hashicorp/helm/2.12.1/docs/resources/release) | resource | +| [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/ingress_v1) | resource | +| [kubernetes_namespace.airflow](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/namespace) | resource | +| [kubernetes_namespace.keda](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/namespace) | resource | +| [kubernetes_role.airflow_pod_creator](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/role) | resource | +| [kubernetes_role_binding.airflow_pod_creator_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/role_binding) | resource | +| [kubernetes_secret.airflow_metadata](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/secret) | resource | +| [kubernetes_secret.airflow_webserver](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/secret) | resource | +| [null_resource.remove_finalizers](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | +| [random_id.airflow_webserver_secret](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/id) | resource | +| [random_id.counter](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/id) | resource | +| [random_password.airflow_db](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/password) | resource | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster_auth) | data source | +| [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/security_group) | data source | +| [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/ssm_parameter) | data source | +| [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/data-sources/ingress_v1) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | value | `string` | n/a | yes | +| [counter](#input\_counter) | value | `string` | n/a | yes | +| [custom\_airflow\_docker\_image](#input\_custom\_airflow\_docker\_image) | Docker image for the customized Airflow image. | 
object({
    name = string
    tag  = string
  })
| n/a | yes | +| [eks\_cluster\_name](#input\_eks\_cluster\_name) | value | `string` | n/a | yes | +| [helm\_charts](#input\_helm\_charts) | Settings for the required Helm charts. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| n/a | yes | +| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes | +| [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | +| [release](#input\_release) | The SPS release version | `string` | n/a | yes | +| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | +| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [airflow\_webserver\_url](#output\_airflow\_webserver\_url) | The URL of the Airflow webserver service | + diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf new file mode 100644 index 0000000..7fc417b --- /dev/null +++ b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf @@ -0,0 +1,18 @@ +data "aws_eks_cluster" "cluster" { + name = var.eks_cluster_name +} + +data "aws_eks_cluster_auth" "cluster" { + name = var.eks_cluster_name +} + +data "aws_ssm_parameter" "subnet_ids" { + name = "/unity/cs/account/network/subnet_list" +} + +data "kubernetes_ingress_v1" "airflow_ingress" { + metadata { + name = kubernetes_ingress_v1.airflow_ingress.metadata[0].name + namespace = kubernetes_namespace.airflow.metadata[0].name + } +} diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf new file mode 100644 index 0000000..9ea2d0f --- /dev/null +++ b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf @@ -0,0 +1,16 @@ + +locals { + counter = var.counter != "" ? var.counter : random_id.counter.hex + common_tags = { + Name = "" + Venue = var.venue + Proj = var.project + ServiceArea = var.service_area + CapVersion = var.release + Component = "" + CreatedBy = var.service_area + Env = var.venue + mission = var.project + Stack = "" + } +} diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf new file mode 100644 index 0000000..316a6b5 --- /dev/null +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -0,0 +1,275 @@ +resource "random_id" "counter" { + byte_length = 2 +} + +resource "kubernetes_namespace" "keda" { + metadata { + name = "keda" + } +} + +resource "helm_release" "keda" { + name = "keda" + repository = var.helm_charts.keda.repository + chart = var.helm_charts.keda.chart + version = var.helm_charts.keda.version + namespace = kubernetes_namespace.keda.metadata[0].name +} + +resource "null_resource" "remove_finalizers" { + # https://keda.sh/docs/deploy/#uninstall + provisioner "local-exec" { + when = destroy + command = < 0, f"DAG {dag_id} doesn't have any tags." + for tag in dag.tags: + assert isinstance(tag, str), f"Tag {tag} in DAG {dag_id} is not a string." + + +@then("each DAG should have at least one task") +def each_dag_should_have_at_least_one_task(dagbag): + for dag_id, dag in dagbag.dags.items(): + assert len(dag.tasks) > 0, f"DAG {dag_id} does not have any tasks" + + +@then("no DAG should have a cycle") +def no_dag_should_have_a_cycle(dagbag): + for dag_id, dag in dagbag.dags.items(): + check_cycle(dag)