diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5fa86b..b16ac6af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Tags - SPS Version 2.2.0 -- OGC API Version 1.1.0 -- OGC Python Client Version 1.1.0 +- OGC API Version 2.0.0 +- OGC Python Client Version 2.0.0 ## Repositories - unity-sps: -- unity-sps-ogc-processes-api: -- unity-sps-ogc-processes-api-client-python: +- unity-sps-ogc-processes-api: +- unity-sps-ogc-processes-api-client-python: ## Epics @@ -27,8 +27,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[New Feature]: Increase ephemeral disk space for Airflow workers](https://github.com/unity-sds/unity-sps/issues/152) - [[New Feature]: Enable users to select the EC2 type to execute a workload](https://github.com/unity-sds/unity-sps/issues/153) - [[New Feature]: Set the DAG run status to "failed" if the main worker task failed](https://github.com/unity-sds/unity-sps/issues/189) + - [[New Feature]: Demonstrate use of ECR within an Airflow DAG (https://github.com/unity-sds/unity-sps/issues/186) - EPIC: `Airflow/WPS-T Integration` - [[New Feature]: Create test to deploy, execute and undeploy the CWL DAG](https://github.com/unity-sds/unity-sps/issues/131) + - [[New Feature]: Enable execution of OGC data processing requests with arbitrary parameter values](https://github.com/unity-sds/unity-sps/issues/129) - EPIC: `Production Venue Deployments` - [[New Feature]: Airflow HTTPD Proxy development and configuration](https://github.com/unity-sds/unity-sps/issues/125) - [[New Feature]: Expose SPS health check endpoints](https://github.com/unity-sds/unity-sps/issues/127) @@ -37,6 +39,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[New Feature]: Review the SPS GitBook documentation](https://github.com/unity-sds/unity-sps/issues/118) - [[New Feature]: Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132) - [[New Feature]: Parametrize the SPS Integration Tests](https://github.com/unity-sds/unity-sps/issues/155) + - [[New Feature] Upgrade SPS to latest version of Airflow 2.10.0](https://github.com/unity-sds/unity-sps/issues/195) + +## Docker Containers + +- ghcr.io/unity-sds/unity-sps/sps-airflow:2.2.0 +- ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0 +- ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api:2.0.0 + +## Documentation + +- For Administrators: + - [SPS Deployment with Terraform](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/sps-deployment-with-terraform) + - [Interacting with an Existing SPS Deployment](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/interacting-with-an-existing-sps-deployment) + - [SPS Airflow Custom Docker Image Build Instructions](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/sps-airflow-custom-docker-image-build-instructions) + - [SPS Post Deployment Operations](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/sps-post-deployment-operations) +- For Deverlopers: + - [Tutorial: Deploy, Execute, and Undeploy a Process using the OGC API - Processes](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/developers-guide/tutorial-deploy-execute-and-undeploy-a-process-using-the-ogc-api-processes) +- For Users: + - [Tutorial: Register and Execute a CWL Workflow](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/users-guide/tutorial-register-and-execute-a-cwl-workflow) # [Unity Release 24.2] - 2024-07-01 @@ -70,6 +91,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[Task] Add TESTING.md file to SPS repo](https://github.com/unity-sds/unity-sps/issues/99) - EPIC: `SPS Infrastructure` - [[New Feature] Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132) +- EPIC: `SPS Security` + - [[Bug]: Upgrade EKS 1.27 AMIs](https://github.com/unity-sds/unity-sps/issues/159) + - [[Bug]: Upgrade to EKS 1.29 AMIs](https://github.com/unity-sds/unity-sps/issues/206) ## Docker Containers diff --git a/CONVENTIONS.md b/CONVENTIONS.md new file mode 100644 index 00000000..e69de29b diff --git a/airflow/dags/cwl_dag.py b/airflow/dags/cwl_dag.py index fa52a948..016d4c10 100644 --- a/airflow/dags/cwl_dag.py +++ b/airflow/dags/cwl_dag.py @@ -26,7 +26,7 @@ POD_NAMESPACE = "sps" POD_LABEL = "cwl_task" # SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-3" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0" NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator" NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload" diff --git a/airflow/dags/docker_cwl_pod.yaml b/airflow/dags/docker_cwl_pod.yaml index 1a9f21ac..71458072 100644 --- a/airflow/dags/docker_cwl_pod.yaml +++ b/airflow/dags/docker_cwl_pod.yaml @@ -21,7 +21,7 @@ spec: containers: - name: cwl-docker - image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1 + image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0 imagePullPolicy: Always command: ["/usr/share/cwl/docker_cwl_entrypoint.sh"] securityContext: diff --git a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py index 2cba0400..962953aa 100644 --- a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py +++ b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py @@ -23,7 +23,7 @@ # The Kubernetes namespace within which the Pod is run (it must already exist) POD_NAMESPACE = "sps" POD_LABEL = "sbg_task" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0" # The path of the working directory where the CWL workflow is executed # (aka the starting directory for cwl-runner). diff --git a/airflow/dags/sbg_preprocess_cwl_dag.py b/airflow/dags/sbg_preprocess_cwl_dag.py index 2d100925..d84b4fd8 100644 --- a/airflow/dags/sbg_preprocess_cwl_dag.py +++ b/airflow/dags/sbg_preprocess_cwl_dag.py @@ -17,7 +17,7 @@ # The Kubernetes namespace within which the Pod is run (it must already exist) POD_NAMESPACE = "sps" POD_LABEL = "sbg_preprocess_task" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0" # The path of the working directory where the CWL workflow is executed # (aka the starting directory for cwl-runner). diff --git a/ogc-application-packages/cwl_dag.json b/ogc-application-packages/cwl_dag.json new file mode 100644 index 00000000..0fbf9ebb --- /dev/null +++ b/ogc-application-packages/cwl_dag.json @@ -0,0 +1,74 @@ +{ + "executionUnit": { + "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0", + "type": "docker" + }, + "processDescription": { + "description": "This process executes any CWL workflow.", + "id": "cwl_dag", + "inputs": { + "cwl_args": { + "description": "The URL of the CWL workflow's YAML parameters file", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow Parameters URL" + }, + "cwl_workflow": { + "description": "The URL of the CWL workflow", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow URL" + }, + "request_cpu": { + "description": "The number of CPU cores requested for the job", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "string" + }, + "title": "Requested CPU" + }, + "request_memory": { + "default": "8Gi", + "description": "The amount of memory requested for the job", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "string" + }, + "title": "Requested Memory" + }, + "request_storage": { + "description": "The amount of storage requested for the job", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "string" + }, + "title": "Requested Storage" + } + }, + "jobControlOptions": [ + "async-execute" + ], + "outputs": { + "result": { + "description": "The result of the SBG Preprocess Workflow execution", + "schema": { + "$ref": "some-ref" + }, + "title": "Process Result" + } + }, + "title": "Generic CWL Process", + "version": "1.0.0" + } +} diff --git a/ogc-application-packages/karpenter_test.json b/ogc-application-packages/karpenter_test.json new file mode 100644 index 00000000..253b7327 --- /dev/null +++ b/ogc-application-packages/karpenter_test.json @@ -0,0 +1,36 @@ +{ + "executionUnit": { + "image": "busybox", + "type": "docker" + }, + "processDescription": { + "description": "This process tests Karpenter node provisioning with different instance types.", + "id": "karpenter_test", + "inputs": { + "placeholder": { + "default": 1, + "description": "A placeholder parameter", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "integer" + }, + "title": "Placeholder" + } + }, + "jobControlOptions": [ + "async-execute" + ], + "outputs": { + "result": { + "description": "The result of the Karpenter test execution", + "schema": { + "$ref": "some-ref" + }, + "title": "Process Result" + } + }, + "title": "Karpenter Test Process", + "version": "1.0.0" + } +} diff --git a/ogc-application-packages/sbg_preprocess_cwl_dag.json b/ogc-application-packages/sbg_preprocess_cwl_dag.json new file mode 100644 index 00000000..95eeea42 --- /dev/null +++ b/ogc-application-packages/sbg_preprocess_cwl_dag.json @@ -0,0 +1,48 @@ +{ + "executionUnit": { + "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0", + "type": "docker" + }, + "processDescription": { + "description": "This process executes the SBG Preprocess Workflow using CWL.", + "id": "sbg_preprocess_cwl_dag", + "inputs": { + "cwl_args": { + "default": "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.dev.yml", + "description": "The SBG Pre-process YAML parameters URL", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow Parameters" + }, + "cwl_workflow": { + "default": "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl", + "description": "The SBG Pre-process CWL workflow URL", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow" + } + }, + "jobControlOptions": [ + "async-execute" + ], + "outputs": { + "result": { + "description": "The result of the SBG Preprocess Workflow execution", + "schema": { + "$ref": "some-ref" + }, + "title": "Process Result" + } + }, + "title": "SBG Preprocess CWL Workflow", + "version": "1.0.0" + } +} diff --git a/pyproject.toml b/pyproject.toml index e1d8540a..813fc647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,12 @@ build-backend = "setuptools.build_meta" [project] name = "unity-sps" -version = "2.1.0" +version = "2.2.0" authors = [ { name = "Drew Meyers", email = "drew.meyers@jpl.nasa.gov" }, + { name = "Luca Cinquini", email = "luca.cinquini@jpl.nasa.gov" }, + { name = "Nikki Tebaldi", email = "nicole.tebaldi@jpl.nasa.gov" }, + { name = "Brad Lunsford", email = "bradley.t.lunsford@jpl.nasa.gov" } ] description = "The science processing service area of Unity." classifiers = [ @@ -37,11 +40,12 @@ test = [ "pytest-bdd==7.2.0", "pytest-mock==3.14.0", "requests==2.32.3", - "apache-airflow==2.9.2", - "kubernetes==30.1.0", - "boto3==1.34.143", + "apache-airflow==2.10.0", + "kubernetes==29.0.0", + "boto3==1.34.46", "backoff==2.2.1", - "apache-airflow-providers-cncf-kubernetes==8.4.1" + "apache-airflow-providers-cncf-kubernetes==8.4.1", + "unity_sps_ogc_processes_api_python_client @ git+https://github.com/unity-sds/unity-sps-ogc-processes-api-client-python.git@2.0.0" ] experiment = [] lambda-airflow-dag-trigger = [ diff --git a/terraform-unity/modules/terraform-unity-sps-eks/README.md b/terraform-unity/modules/terraform-unity-sps-eks/README.md index 6ac116ae..24bc32aa 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/README.md +++ b/terraform-unity/modules/terraform-unity-sps-eks/README.md @@ -31,7 +31,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "t3.large"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | +| [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "t3.xlarge"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | diff --git a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf index 00768af2..3b36d1c3 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf @@ -46,7 +46,7 @@ variable "nodegroups" { })) default = { defaultGroup = { - instance_types = ["t3.large"] + instance_types = ["t3.xlarge"] min_size = 1 max_size = 1 desired_size = 1 diff --git a/utils/deploy_ogc_app_packages.py b/utils/deploy_ogc_app_packages.py new file mode 100644 index 00000000..909a6833 --- /dev/null +++ b/utils/deploy_ogc_app_packages.py @@ -0,0 +1,98 @@ +import argparse +import json +import logging +import os + +import unity_sps_ogc_processes_api_python_client +from unity_sps_ogc_processes_api_python_client.models.ogcapppkg import Ogcapppkg +from unity_sps_ogc_processes_api_python_client.rest import ApiException + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + + +def register_process(api_instance, proc, ogcapppkg_instance): + """ + Register a process with the OGC API. + + Args: + api_instance: The API client instance. + proc: The process identifier. + ogcapppkg_instance: The Ogcapppkg instance containing the process description. + + Returns: + None + """ + try: + # Deploy a process + api_instance.deploy_processes_post(w=proc, ogcapppkg=ogcapppkg_instance) + logging.info(f"Successfully registered process: {proc}") + except ApiException as e: + logging.error(f"Exception when calling DRUApi->deploy_processes_post for process {proc}: {e}") + except Exception as e: + logging.error(f"Unexpected error for process {proc}: {e}") + + +def main(): + """ + Main function to deploy processes to the OGC API. + + Parses command-line arguments, reads process descriptions from JSON files, + and registers each process with the OGC API. + + Setup: pip install -e ".[develop, test]" + + Invocation Syntax: python utils/deploy_ogc_app_packages.py + Example invocation: python utils/deploy_ogc_app_packages.py http://k8s-sps-ogcproce-XXXXXX-XXXXXXX.us-west-2.elb.amazonaws.com:5001 .../unity-sps/ogc-application-packages + + Args: + None + + Returns: + None + """ + parser = argparse.ArgumentParser(description="Deploy processes to OGC API") + parser.add_argument("ogc_api_processes", help="OGC API Processes URL") + parser.add_argument( + "ogc_app_packages_dir", help="Directory containing JSON files with application packages" + ) + args = parser.parse_args() + + OGC_API_PROCESSES = args.ogc_api_processes + ogc_app_packages_dir = args.ogc_app_packages_dir + + # Configure the API client + configuration = unity_sps_ogc_processes_api_python_client.Configuration(host=OGC_API_PROCESSES) + + with unity_sps_ogc_processes_api_python_client.ApiClient(configuration) as api_client: + api_instance = unity_sps_ogc_processes_api_python_client.DRUApi(api_client) + + # Iterate through all JSON files in the specified directory + for filename in os.listdir(ogc_app_packages_dir): + if filename.endswith(".json"): + json_file = os.path.join(ogc_app_packages_dir, filename) + try: + # Read the process description from the JSON file + with open(json_file, "r") as f: + process_data = json.load(f) + + # Extract the process ID from the JSON data + proc = process_data.get("processDescription", {}).get("id") + if not proc: + logging.error(f"Process ID not found in JSON file: {json_file}") + continue + + # Create an instance of Ogcapppkg from the JSON data + ogcapppkg_instance = Ogcapppkg.from_dict(process_data) + logging.info(f"Registering process: {proc}") + register_process(api_instance, proc, ogcapppkg_instance) + except FileNotFoundError: + logging.error(f"JSON file not found: {json_file}") + except json.JSONDecodeError: + logging.error(f"Error decoding JSON file: {json_file}") + except Exception as e: + logging.error(f"Unexpected error processing file {json_file}: {e}") + + +if __name__ == "__main__": + main() diff --git a/utils/post_deployment.sh b/utils/post_deployment.sh deleted file mode 100755 index 82aa4506..00000000 --- a/utils/post_deployment.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -#set -ex - -# Script to execute post-deployment operations -# Pre-Requisites: -# - SPS has been deployed successfully -# - AWS credentials are renewed and set in the environment -# Syntax: -# ./post_deployment.sh -# Example: -# ./post_deployment.sh http://k8s-airflow-ogcproce-944e409e1d-687289935.us-west-2.elb.amazonaws.com:5001 - -# script argument: the $WPST_API -export WPST_API=$1 - -# list of processes to be registered -declare -a procs=("cwl_dag" "karpenter_test" "sbg_preprocess_cwl_dag") - -for proc in "${procs[@]}" -do - echo " " - # register process - echo "Registering process: $proc" - curl -k -X POST -H "Content-Type: application/json; charset=utf-8" --data '{"id":"'${proc}'", "version": "1.0.0"}' "${WPST_API}/processes" - # unregister process - # echo "Unregistering process: $proc" - # curl -X DELETE -H "Content-Type: application/json; charset=utf-8" "${WPST_API}/processes/${proc}" - echo " " -done