From 46cd7b46cbbacb38612421ff6e4b8b7079096606 Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Tue, 10 Sep 2024 10:48:35 -0600 Subject: [PATCH 01/12] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5fa86b..ddd494ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[New Feature]: Increase ephemeral disk space for Airflow workers](https://github.com/unity-sds/unity-sps/issues/152) - [[New Feature]: Enable users to select the EC2 type to execute a workload](https://github.com/unity-sds/unity-sps/issues/153) - [[New Feature]: Set the DAG run status to "failed" if the main worker task failed](https://github.com/unity-sds/unity-sps/issues/189) + - [[New Feature]: Demonstrate use of ECR within an Airflow DAG (https://github.com/unity-sds/unity-sps/issues/186) - EPIC: `Airflow/WPS-T Integration` - [[New Feature]: Create test to deploy, execute and undeploy the CWL DAG](https://github.com/unity-sds/unity-sps/issues/131) - EPIC: `Production Venue Deployments` From cde32911c11d813dd543031e9b6ffa00f5065f68 Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Tue, 10 Sep 2024 13:57:12 -0600 Subject: [PATCH 02/12] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddd494ce..5b3666ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[Task] Add TESTING.md file to SPS repo](https://github.com/unity-sds/unity-sps/issues/99) - EPIC: `SPS Infrastructure` - [[New Feature] Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132) + - [[New Feature] Upgrade SPS to latest version of Airflow 2.10.0](https://github.com/unity-sds/unity-sps/issues/195) ## Docker Containers From b457e75a1798fa18445dabbdbe3b74db96b91ce9 Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Tue, 10 Sep 2024 13:58:38 -0600 Subject: [PATCH 03/12] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3666ef..1d4fb3da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[New Feature]: Review the SPS GitBook documentation](https://github.com/unity-sds/unity-sps/issues/118) - [[New Feature]: Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132) - [[New Feature]: Parametrize the SPS Integration Tests](https://github.com/unity-sds/unity-sps/issues/155) + - [[New Feature] Upgrade SPS to latest version of Airflow 2.10.0](https://github.com/unity-sds/unity-sps/issues/195) # [Unity Release 24.2] - 2024-07-01 @@ -71,7 +72,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[Task] Add TESTING.md file to SPS repo](https://github.com/unity-sds/unity-sps/issues/99) - EPIC: `SPS Infrastructure` - [[New Feature] Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132) - - [[New Feature] Upgrade SPS to latest version of Airflow 2.10.0](https://github.com/unity-sds/unity-sps/issues/195) ## Docker Containers From 6a96a25c24120243c324658361d5e66532eed9c6 Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Mon, 16 Sep 2024 08:35:16 -0600 Subject: [PATCH 04/12] Upgrading default release to 24.3 --- terraform-unity/README.md | 2 +- terraform-unity/modules/terraform-unity-sps-eks/README.md | 4 ++-- terraform-unity/modules/terraform-unity-sps-eks/variables.tf | 2 +- .../modules/terraform-unity-sps-karpenter/README.md | 2 +- .../modules/terraform-unity-sps-karpenter/variables.tf | 2 +- terraform-unity/variables.tf | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/terraform-unity/README.md b/terraform-unity/README.md index 73e0557f..7c50aa39 100644 --- a/terraform-unity/README.md +++ b/terraform-unity/README.md @@ -198,7 +198,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt | [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no | | [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.3"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "1.0.0"
  },
  "redis": {
    "name": "redis",
    "tag": "7.2.4"
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no | -| [release](#input\_release) | The software release version. | `string` | `"24.2"` | no | +| [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no | | [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes | diff --git a/terraform-unity/modules/terraform-unity-sps-eks/README.md b/terraform-unity/modules/terraform-unity-sps-eks/README.md index 1edb9487..90a054e7 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/README.md +++ b/terraform-unity/modules/terraform-unity-sps-eks/README.md @@ -18,7 +18,7 @@ | Name | Source | Version | |------|--------|---------| -| [unity-eks](#module\_unity-eks) | git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module | eks-ami-updates | +| [unity-eks](#module\_unity-eks) | git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module | unity-sps-2.2.0 | ## Resources @@ -33,7 +33,7 @@ |------|-------------|------|---------|:--------:| | [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "t3.large"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | -| [release](#input\_release) | The software release version. | `string` | `"24.2"` | no | +| [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | | [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | diff --git a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf index 3b6f13c9..00768af2 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf @@ -18,7 +18,7 @@ variable "service_area" { variable "release" { description = "The software release version." type = string - default = "24.2" + default = "24.3" } variable "nodegroups" { diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/README.md b/terraform-unity/modules/terraform-unity-sps-karpenter/README.md index 38183865..5eca7809 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter/README.md +++ b/terraform-unity/modules/terraform-unity-sps-karpenter/README.md @@ -37,7 +37,7 @@ |------|-------------|------|---------|:--------:| | [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| 
{
  "karpenter": {
    "chart": "karpenter",
    "repository": "oci://public.ecr.aws/karpenter",
    "version": "0.36.1"
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | -| [release](#input\_release) | The software release version. | `string` | `"24.2"` | no | +| [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | | [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf b/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf index 6e116f92..97efa0c4 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf @@ -18,7 +18,7 @@ variable "service_area" { variable "release" { description = "The software release version." type = string - default = "24.2" + default = "24.3" } variable "helm_charts" { diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index 48e2bc59..1ccfa1c2 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -22,7 +22,7 @@ variable "service_area" { variable "release" { description = "The software release version." type = string - default = "24.2" + default = "24.3" } variable "kubeconfig_filepath" { From 7d00b8642c0f3772b2fa5e8aca8a85b889636b4a Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Mon, 16 Sep 2024 08:48:23 -0600 Subject: [PATCH 05/12] Updating versions in pyproject.toml --- pyproject.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3a45312f..e9e8e6d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,12 @@ build-backend = "setuptools.build_meta" [project] name = "unity-sps" -version = "2.1.0" +version = "2.2.0" authors = [ { name = "Drew Meyers", email = "drew.meyers@jpl.nasa.gov" }, + { name = "Luca Cinquini", email = "luca.cinquini@jpl.nasa.gov" }, + { name = "Nikki Tebaldi", email = "nicole.tebaldi@jpl.nasa.gov" }, + { name = "Brad Lunsford", email = "bradley.t.lunsford@jpl.nasa.gov" } ] description = "The science processing service area of Unity." classifiers = [ @@ -37,11 +40,11 @@ test = [ "pytest-bdd==7.1.1", "pytest-mock==3.12.0", "requests==2.31.0", - "apache-airflow==2.9.1", + "apache-airflow==2.10.0", "kubernetes==29.0.0", "boto3==1.34.46", "backoff==2.2.1", - "apache-airflow-providers-cncf-kubernetes==8.2.0" + "apache-airflow-providers-cncf-kubernetes==8.4.1" ] experiment = [] lambda-airflow-dag-trigger = [ From 4c0d1e4c0cc97436d070cc3820bef031a66c421e Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Thu, 19 Sep 2024 07:36:36 -0600 Subject: [PATCH 06/12] Using SPS release 2.2.0-rc-1 --- airflow/dags/cwl_dag.py | 2 +- airflow/dags/docker_cwl_pod.yaml | 2 +- airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py | 2 +- airflow/dags/sbg_preprocess_cwl_dag.py | 2 +- terraform-unity/README.md | 4 ++-- terraform-unity/variables.tf | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airflow/dags/cwl_dag.py b/airflow/dags/cwl_dag.py index fa52a948..bbd25192 100644 --- a/airflow/dags/cwl_dag.py +++ b/airflow/dags/cwl_dag.py @@ -26,7 +26,7 @@ POD_NAMESPACE = "sps" POD_LABEL = "cwl_task" # SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-3" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1" NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator" NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload" diff --git a/airflow/dags/docker_cwl_pod.yaml b/airflow/dags/docker_cwl_pod.yaml index 1a9f21ac..27b5e7e3 100644 --- a/airflow/dags/docker_cwl_pod.yaml +++ b/airflow/dags/docker_cwl_pod.yaml @@ -21,7 +21,7 @@ spec: containers: - name: cwl-docker - image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1 + image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1 imagePullPolicy: Always command: ["/usr/share/cwl/docker_cwl_entrypoint.sh"] securityContext: diff --git a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py index 2cba0400..f216bda2 100644 --- a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py +++ b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py @@ -23,7 +23,7 @@ # The Kubernetes namespace within which the Pod is run (it must already exist) POD_NAMESPACE = "sps" POD_LABEL = "sbg_task" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1" # The path of the working directory where the CWL workflow is executed # (aka the starting directory for cwl-runner). diff --git a/airflow/dags/sbg_preprocess_cwl_dag.py b/airflow/dags/sbg_preprocess_cwl_dag.py index 2d100925..f6d4e360 100644 --- a/airflow/dags/sbg_preprocess_cwl_dag.py +++ b/airflow/dags/sbg_preprocess_cwl_dag.py @@ -17,7 +17,7 @@ # The Kubernetes namespace within which the Pod is run (it must already exist) POD_NAMESPACE = "sps" POD_LABEL = "sbg_preprocess_task" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1" # The path of the working directory where the CWL workflow is executed # (aka the starting directory for cwl-runner). diff --git a/terraform-unity/README.md b/terraform-unity/README.md index 7c50aa39..382c7000 100644 --- a/terraform-unity/README.md +++ b/terraform-unity/README.md @@ -187,7 +187,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | 
object({
    airflow = object({
      name = string
      tag  = string
    })
  })
| 
{
  "airflow": {
    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",
    "tag": "2.2.0-beta-1"
  }
}
| no | +| [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | 
object({
    airflow = object({
      name = string
      tag  = string
    })
  })
| 
{
  "airflow": {
    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",
    "tag": "2.2.0-rc-1"
  }
}
| no | | [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes | | [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no | | [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | 
object({
    url                 = string
    ref                 = string
    dags_directory_path = string
  })
| 
{
  "dags_directory_path": "airflow/dags",
  "ref": "develop",
  "url": "https://github.com/unity-sds/unity-sps.git"
}
| no | @@ -196,7 +196,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt | [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | 
map(object({
    requirements : list(object({
      key : string
      operator : string
      values : list(string)
    }))
    nodeClassRef : string
    limits : object({
      cpu : string
      memory : string
    })
    disruption : object({
      consolidationPolicy : string
      consolidateAfter : string
    })
  }))
| 
{
  "airflow-celery-workers": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "80",
      "memory": "320Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "9"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "32769"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-core-components": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "40",
      "memory": "160Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "32769"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "100",
      "memory": "400Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "32769"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator-high-workload": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "528",
      "memory": "1056Gi"
    },
    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "49"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "98305"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  }
}
| no | | [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | | [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no | -| [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.3"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "1.0.0"
  },
  "redis": {
    "name": "redis",
    "tag": "7.2.4"
  }
}
| no | +| [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.4"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "2.0.0-rc-1"
  },
  "redis": {
    "name": "redis",
    "tag": "7.4.0"
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no | diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index 1ccfa1c2..a785790d 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -73,7 +73,7 @@ variable "airflow_docker_images" { default = { airflow = { name = "ghcr.io/unity-sds/unity-sps/sps-airflow" - tag = "2.2.0-beta-1" + tag = "2.2.0-rc-1" } } } @@ -98,15 +98,15 @@ variable "ogc_processes_docker_images" { default = { ogc_processes_api = { name = "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api" - tag = "1.0.0" + tag = "2.0.0-rc-1" } git_sync = { name = "registry.k8s.io/git-sync/git-sync" - tag = "v4.2.3" + tag = "v4.2.4" }, redis = { name = "redis" - tag = "7.2.4" + tag = "7.4.0" } } } From 3d11bbeaf9e8503ccd7797d66183aaa7268ff2c0 Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Thu, 19 Sep 2024 07:58:52 -0600 Subject: [PATCH 07/12] Updating CHANGELOG --- CHANGELOG.md | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4fb3da..b16ac6af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Tags - SPS Version 2.2.0 -- OGC API Version 1.1.0 -- OGC Python Client Version 1.1.0 +- OGC API Version 2.0.0 +- OGC Python Client Version 2.0.0 ## Repositories - unity-sps: -- unity-sps-ogc-processes-api: -- unity-sps-ogc-processes-api-client-python: +- unity-sps-ogc-processes-api: +- unity-sps-ogc-processes-api-client-python: ## Epics @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[New Feature]: Demonstrate use of ECR within an Airflow DAG (https://github.com/unity-sds/unity-sps/issues/186) - EPIC: `Airflow/WPS-T Integration` - [[New Feature]: Create test to deploy, execute and undeploy the CWL DAG](https://github.com/unity-sds/unity-sps/issues/131) + - [[New Feature]: Enable execution of OGC data processing requests with arbitrary parameter values](https://github.com/unity-sds/unity-sps/issues/129) - EPIC: `Production Venue Deployments` - [[New Feature]: Airflow HTTPD Proxy development and configuration](https://github.com/unity-sds/unity-sps/issues/125) - [[New Feature]: Expose SPS health check endpoints](https://github.com/unity-sds/unity-sps/issues/127) @@ -40,6 +41,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[New Feature]: Parametrize the SPS Integration Tests](https://github.com/unity-sds/unity-sps/issues/155) - [[New Feature] Upgrade SPS to latest version of Airflow 2.10.0](https://github.com/unity-sds/unity-sps/issues/195) +## Docker Containers + +- ghcr.io/unity-sds/unity-sps/sps-airflow:2.2.0 +- ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0 +- ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api:2.0.0 + +## Documentation + +- For Administrators: + - [SPS Deployment with Terraform](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/sps-deployment-with-terraform) + - [Interacting with an Existing SPS Deployment](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/interacting-with-an-existing-sps-deployment) + - [SPS Airflow Custom Docker Image Build Instructions](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/sps-airflow-custom-docker-image-build-instructions) + - [SPS Post Deployment Operations](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/admin-guide/sps-post-deployment-operations) +- For Deverlopers: + - [Tutorial: Deploy, Execute, and Undeploy a Process using the OGC API - Processes](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/developers-guide/tutorial-deploy-execute-and-undeploy-a-process-using-the-ogc-api-processes) +- For Users: + - [Tutorial: Register and Execute a CWL Workflow](https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/science-processing/docs/users-guide/tutorial-register-and-execute-a-cwl-workflow) + # [Unity Release 24.2] - 2024-07-01 ## Tags @@ -72,6 +91,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[Task] Add TESTING.md file to SPS repo](https://github.com/unity-sds/unity-sps/issues/99) - EPIC: `SPS Infrastructure` - [[New Feature] Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132) +- EPIC: `SPS Security` + - [[Bug]: Upgrade EKS 1.27 AMIs](https://github.com/unity-sds/unity-sps/issues/159) + - [[Bug]: Upgrade to EKS 1.29 AMIs](https://github.com/unity-sds/unity-sps/issues/206) ## Docker Containers From 1eac653e5dce4ed4ae69b05c4c4aaee0bf736034 Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Thu, 19 Sep 2024 08:40:17 -0600 Subject: [PATCH 08/12] Increasing size of EKS server to t3.xlarge --- terraform-unity/modules/terraform-unity-sps-eks/README.md | 2 +- terraform-unity/modules/terraform-unity-sps-eks/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform-unity/modules/terraform-unity-sps-eks/README.md b/terraform-unity/modules/terraform-unity-sps-eks/README.md index bc2eacbc..8005392b 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/README.md +++ b/terraform-unity/modules/terraform-unity-sps-eks/README.md @@ -31,7 +31,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "t3.large"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | +| [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "t3.xlarge"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | diff --git a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf index 00768af2..3b36d1c3 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf @@ -46,7 +46,7 @@ variable "nodegroups" { })) default = { defaultGroup = { - instance_types = ["t3.large"] + instance_types = ["t3.xlarge"] min_size = 1 max_size = 1 desired_size = 1 From 1a58c069a2b07f82db89360bf879924c0e82e013 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 19 Sep 2024 10:11:54 -0700 Subject: [PATCH 09/12] feat: python script to deploy OGC App Packages --- CONVENTIONS.md | 0 ogc-application-packages/cwl_dag.json | 74 +++++++++++++++ ogc-application-packages/karpenter_test.json | 36 +++++++ .../sbg_preprocess_cwl_dag.json | 48 ++++++++++ pyproject.toml | 4 +- utils/deploy_ogc_app_packages.py | 93 +++++++++++++++++++ utils/post_deployment.sh | 29 ------ 7 files changed, 254 insertions(+), 30 deletions(-) create mode 100644 CONVENTIONS.md create mode 100644 ogc-application-packages/cwl_dag.json create mode 100644 ogc-application-packages/karpenter_test.json create mode 100644 ogc-application-packages/sbg_preprocess_cwl_dag.json create mode 100644 utils/deploy_ogc_app_packages.py delete mode 100755 utils/post_deployment.sh diff --git a/CONVENTIONS.md b/CONVENTIONS.md new file mode 100644 index 00000000..e69de29b diff --git a/ogc-application-packages/cwl_dag.json b/ogc-application-packages/cwl_dag.json new file mode 100644 index 00000000..0fbf9ebb --- /dev/null +++ b/ogc-application-packages/cwl_dag.json @@ -0,0 +1,74 @@ +{ + "executionUnit": { + "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0", + "type": "docker" + }, + "processDescription": { + "description": "This process executes any CWL workflow.", + "id": "cwl_dag", + "inputs": { + "cwl_args": { + "description": "The URL of the CWL workflow's YAML parameters file", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow Parameters URL" + }, + "cwl_workflow": { + "description": "The URL of the CWL workflow", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow URL" + }, + "request_cpu": { + "description": "The number of CPU cores requested for the job", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "string" + }, + "title": "Requested CPU" + }, + "request_memory": { + "default": "8Gi", + "description": "The amount of memory requested for the job", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "string" + }, + "title": "Requested Memory" + }, + "request_storage": { + "description": "The amount of storage requested for the job", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "string" + }, + "title": "Requested Storage" + } + }, + "jobControlOptions": [ + "async-execute" + ], + "outputs": { + "result": { + "description": "The result of the SBG Preprocess Workflow execution", + "schema": { + "$ref": "some-ref" + }, + "title": "Process Result" + } + }, + "title": "Generic CWL Process", + "version": "1.0.0" + } +} diff --git a/ogc-application-packages/karpenter_test.json b/ogc-application-packages/karpenter_test.json new file mode 100644 index 00000000..253b7327 --- /dev/null +++ b/ogc-application-packages/karpenter_test.json @@ -0,0 +1,36 @@ +{ + "executionUnit": { + "image": "busybox", + "type": "docker" + }, + "processDescription": { + "description": "This process tests Karpenter node provisioning with different instance types.", + "id": "karpenter_test", + "inputs": { + "placeholder": { + "default": 1, + "description": "A placeholder parameter", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "type": "integer" + }, + "title": "Placeholder" + } + }, + "jobControlOptions": [ + "async-execute" + ], + "outputs": { + "result": { + "description": "The result of the Karpenter test execution", + "schema": { + "$ref": "some-ref" + }, + "title": "Process Result" + } + }, + "title": "Karpenter Test Process", + "version": "1.0.0" + } +} diff --git a/ogc-application-packages/sbg_preprocess_cwl_dag.json b/ogc-application-packages/sbg_preprocess_cwl_dag.json new file mode 100644 index 00000000..76959fbe --- /dev/null +++ b/ogc-application-packages/sbg_preprocess_cwl_dag.json @@ -0,0 +1,48 @@ +{ + "executionUnit": { + "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1", + "type": "docker" + }, + "processDescription": { + "description": "This process executes the SBG Preprocess Workflow using CWL.", + "id": "sbg_preprocess_cwl_dag", + "inputs": { + "cwl_args": { + "default": "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.dev.yml", + "description": "The SBG Pre-process YAML parameters URL", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow Parameters" + }, + "cwl_workflow": { + "default": "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl", + "description": "The SBG Pre-process CWL workflow URL", + "maxOccurs": 1, + "minOccurs": 1, + "schema": { + "format": "uri", + "type": "string" + }, + "title": "CWL Workflow" + } + }, + "jobControlOptions": [ + "async-execute" + ], + "outputs": { + "result": { + "description": "The result of the SBG Preprocess Workflow execution", + "schema": { + "$ref": "some-ref" + }, + "title": "Process Result" + } + }, + "title": "SBG Preprocess CWL Workflow", + "version": "1.0.0" + } +} diff --git a/pyproject.toml b/pyproject.toml index e9e8e6d2..451e5207 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,9 @@ test = [ "kubernetes==29.0.0", "boto3==1.34.46", "backoff==2.2.1", - "apache-airflow-providers-cncf-kubernetes==8.4.1" + "apache-airflow-providers-cncf-kubernetes==8.4.1", + "unity_sps_ogc_processes_api_python_client @ git+https://github.com/unity-sds/unity-sps-ogc-processes-api-client-python.git@129-arbitrary-parameters#egg=129-arbitrary-parameters" + ] experiment = [] lambda-airflow-dag-trigger = [ diff --git a/utils/deploy_ogc_app_packages.py b/utils/deploy_ogc_app_packages.py new file mode 100644 index 00000000..bae933b5 --- /dev/null +++ b/utils/deploy_ogc_app_packages.py @@ -0,0 +1,93 @@ +import argparse +import json +import logging +import os + +import unity_sps_ogc_processes_api_python_client +from unity_sps_ogc_processes_api_python_client.models.ogcapppkg import Ogcapppkg +from unity_sps_ogc_processes_api_python_client.rest import ApiException + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + + +def register_process(api_instance, proc, ogcapppkg_instance): + """ + Register a process with the OGC API. + + Args: + api_instance: The API client instance. + proc: The process identifier. + ogcapppkg_instance: The Ogcapppkg instance containing the process description. + + Returns: + None + """ + try: + # Deploy a process + api_instance.deploy_processes_post(w=proc, ogcapppkg=ogcapppkg_instance) + logging.info(f"Successfully registered process: {proc}") + except ApiException as e: + logging.error(f"Exception when calling DRUApi->deploy_processes_post for process {proc}: {e}") + except Exception as e: + logging.error(f"Unexpected error for process {proc}: {e}") + + +def main(): + """ + Main function to deploy processes to the OGC API. + + Parses command-line arguments, reads process descriptions from JSON files, + and registers each process with the OGC API. + + Args: + None + + Returns: + None + """ + parser = argparse.ArgumentParser(description="Deploy processes to OGC API") + parser.add_argument("ogc_api_processes", help="OGC API Processes URL") + parser.add_argument( + "ogc_app_packages_dir", help="Directory containing JSON files with application packages" + ) + args = parser.parse_args() + + OGC_API_PROCESSES = args.ogc_api_processes + ogc_app_packages_dir = args.ogc_app_packages_dir + + # Configure the API client + configuration = unity_sps_ogc_processes_api_python_client.Configuration(host=OGC_API_PROCESSES) + + with unity_sps_ogc_processes_api_python_client.ApiClient(configuration) as api_client: + api_instance = unity_sps_ogc_processes_api_python_client.DRUApi(api_client) + + # Iterate through all JSON files in the specified directory + for filename in os.listdir(ogc_app_packages_dir): + if filename.endswith(".json"): + json_file = os.path.join(ogc_app_packages_dir, filename) + try: + # Read the process description from the JSON file + with open(json_file, "r") as f: + process_data = json.load(f) + + # Extract the process ID from the JSON data + proc = process_data.get("processDescription", {}).get("id") + if not proc: + logging.error(f"Process ID not found in JSON file: {json_file}") + continue + + # Create an instance of Ogcapppkg from the JSON data + ogcapppkg_instance = Ogcapppkg.from_dict(process_data) + logging.info(f"Registering process: {proc}") + register_process(api_instance, proc, ogcapppkg_instance) + except FileNotFoundError: + logging.error(f"JSON file not found: {json_file}") + except json.JSONDecodeError: + logging.error(f"Error decoding JSON file: {json_file}") + except Exception as e: + logging.error(f"Unexpected error processing file {json_file}: {e}") + + +if __name__ == "__main__": + main() diff --git a/utils/post_deployment.sh b/utils/post_deployment.sh deleted file mode 100755 index 82aa4506..00000000 --- a/utils/post_deployment.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -#set -ex - -# Script to execute post-deployment operations -# Pre-Requisites: -# - SPS has been deployed successfully -# - AWS credentials are renewed and set in the environment -# Syntax: -# ./post_deployment.sh -# Example: -# ./post_deployment.sh http://k8s-airflow-ogcproce-944e409e1d-687289935.us-west-2.elb.amazonaws.com:5001 - -# script argument: the $WPST_API -export WPST_API=$1 - -# list of processes to be registered -declare -a procs=("cwl_dag" "karpenter_test" "sbg_preprocess_cwl_dag") - -for proc in "${procs[@]}" -do - echo " " - # register process - echo "Registering process: $proc" - curl -k -X POST -H "Content-Type: application/json; charset=utf-8" --data '{"id":"'${proc}'", "version": "1.0.0"}' "${WPST_API}/processes" - # unregister process - # echo "Unregistering process: $proc" - # curl -X DELETE -H "Content-Type: application/json; charset=utf-8" "${WPST_API}/processes/${proc}" - echo " " -done From 955cd1443c42d51fd3fdb275cfed27e4cd853a7a Mon Sep 17 00:00:00 2001 From: Luca Cinquini Date: Thu, 19 Sep 2024 11:53:56 -0600 Subject: [PATCH 10/12] Adding example invocation --- utils/deploy_ogc_app_packages.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/deploy_ogc_app_packages.py b/utils/deploy_ogc_app_packages.py index bae933b5..909a6833 100644 --- a/utils/deploy_ogc_app_packages.py +++ b/utils/deploy_ogc_app_packages.py @@ -40,6 +40,11 @@ def main(): Parses command-line arguments, reads process descriptions from JSON files, and registers each process with the OGC API. + Setup: pip install -e ".[develop, test]" + + Invocation Syntax: python utils/deploy_ogc_app_packages.py + Example invocation: python utils/deploy_ogc_app_packages.py http://k8s-sps-ogcproce-XXXXXX-XXXXXXX.us-west-2.elb.amazonaws.com:5001 .../unity-sps/ogc-application-packages + Args: None From fde07f12ad69385dee65da964369456c5c4314a4 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 19 Sep 2024 12:10:06 -0700 Subject: [PATCH 11/12] chore: bump versions --- airflow/dags/cwl_dag.py | 2 +- airflow/dags/docker_cwl_pod.yaml | 2 +- airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py | 2 +- airflow/dags/sbg_preprocess_cwl_dag.py | 2 +- ogc-application-packages/sbg_preprocess_cwl_dag.json | 2 +- terraform-unity/README.md | 4 ++-- terraform-unity/variables.tf | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/airflow/dags/cwl_dag.py b/airflow/dags/cwl_dag.py index bbd25192..016d4c10 100644 --- a/airflow/dags/cwl_dag.py +++ b/airflow/dags/cwl_dag.py @@ -26,7 +26,7 @@ POD_NAMESPACE = "sps" POD_LABEL = "cwl_task" # SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0" NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator" NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload" diff --git a/airflow/dags/docker_cwl_pod.yaml b/airflow/dags/docker_cwl_pod.yaml index 27b5e7e3..71458072 100644 --- a/airflow/dags/docker_cwl_pod.yaml +++ b/airflow/dags/docker_cwl_pod.yaml @@ -21,7 +21,7 @@ spec: containers: - name: cwl-docker - image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1 + image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0 imagePullPolicy: Always command: ["/usr/share/cwl/docker_cwl_entrypoint.sh"] securityContext: diff --git a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py index f216bda2..962953aa 100644 --- a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py +++ b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py @@ -23,7 +23,7 @@ # The Kubernetes namespace within which the Pod is run (it must already exist) POD_NAMESPACE = "sps" POD_LABEL = "sbg_task" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0" # The path of the working directory where the CWL workflow is executed # (aka the starting directory for cwl-runner). diff --git a/airflow/dags/sbg_preprocess_cwl_dag.py b/airflow/dags/sbg_preprocess_cwl_dag.py index f6d4e360..d84b4fd8 100644 --- a/airflow/dags/sbg_preprocess_cwl_dag.py +++ b/airflow/dags/sbg_preprocess_cwl_dag.py @@ -17,7 +17,7 @@ # The Kubernetes namespace within which the Pod is run (it must already exist) POD_NAMESPACE = "sps" POD_LABEL = "sbg_preprocess_task" -SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1" +SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0" # The path of the working directory where the CWL workflow is executed # (aka the starting directory for cwl-runner). diff --git a/ogc-application-packages/sbg_preprocess_cwl_dag.json b/ogc-application-packages/sbg_preprocess_cwl_dag.json index 76959fbe..95eeea42 100644 --- a/ogc-application-packages/sbg_preprocess_cwl_dag.json +++ b/ogc-application-packages/sbg_preprocess_cwl_dag.json @@ -1,6 +1,6 @@ { "executionUnit": { - "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-rc-1", + "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0", "type": "docker" }, "processDescription": { diff --git a/terraform-unity/README.md b/terraform-unity/README.md index 382c7000..3dce1dad 100644 --- a/terraform-unity/README.md +++ b/terraform-unity/README.md @@ -187,7 +187,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | 
object({
    airflow = object({
      name = string
      tag  = string
    })
  })
| 
{
  "airflow": {
    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",
    "tag": "2.2.0-rc-1"
  }
}
| no | +| [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | 
object({
    airflow = object({
      name = string
      tag  = string
    })
  })
| 
{
  "airflow": {
    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",
    "tag": "2.2.0"
  }
}
| no | | [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes | | [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no | | [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | 
object({
    url                 = string
    ref                 = string
    dags_directory_path = string
  })
| 
{
  "dags_directory_path": "airflow/dags",
  "ref": "develop",
  "url": "https://github.com/unity-sds/unity-sps.git"
}
| no | @@ -196,7 +196,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt | [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | 
map(object({
    requirements : list(object({
      key : string
      operator : string
      values : list(string)
    }))
    nodeClassRef : string
    limits : object({
      cpu : string
      memory : string
    })
    disruption : object({
      consolidationPolicy : string
      consolidateAfter : string
    })
  }))
| 
{
  "airflow-celery-workers": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "80",
      "memory": "320Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "9"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "32769"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-core-components": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "40",
      "memory": "160Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "32769"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "100",
      "memory": "400Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "32769"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator-high-workload": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "528",
      "memory": "1056Gi"
    },
    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "49"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "8191"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "98305"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  }
}
| no | | [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | | [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no | -| [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.4"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "2.0.0-rc-1"
  },
  "redis": {
    "name": "redis",
    "tag": "7.4.0"
  }
}
| no | +| [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.4"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "2.0.0"
  },
  "redis": {
    "name": "redis",
    "tag": "7.4.0"
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"24.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no | diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index a785790d..eed6e71a 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -73,7 +73,7 @@ variable "airflow_docker_images" { default = { airflow = { name = "ghcr.io/unity-sds/unity-sps/sps-airflow" - tag = "2.2.0-rc-1" + tag = "2.2.0" } } } @@ -98,7 +98,7 @@ variable "ogc_processes_docker_images" { default = { ogc_processes_api = { name = "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api" - tag = "2.0.0-rc-1" + tag = "2.0.0" } git_sync = { name = "registry.k8s.io/git-sync/git-sync" From 5eab03df257a90f10a243c672233229b85586995 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 19 Sep 2024 12:20:26 -0700 Subject: [PATCH 12/12] chore: bump ogc python client version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 451e5207..b6f948a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ test = [ "boto3==1.34.46", "backoff==2.2.1", "apache-airflow-providers-cncf-kubernetes==8.4.1", - "unity_sps_ogc_processes_api_python_client @ git+https://github.com/unity-sds/unity-sps-ogc-processes-api-client-python.git@129-arbitrary-parameters#egg=129-arbitrary-parameters" + "unity_sps_ogc_processes_api_python_client @ git+https://github.com/unity-sds/unity-sps-ogc-processes-api-client-python.git@2.0.0" ] experiment = []