Merge pull request #239 from unity-sds/238-upgrade-airflow-to-2.10.3

Upgrading airflow to latest version 2.10.3 and SPS to new version 2.3.0

airflow/dags/cwl_dag.py

@@ -25,7 +25,7 @@
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "sps"
 POD_LABEL = "cwl_task"
-SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0"
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.3.0"
 
 NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator"
 NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload"

airflow/dags/sbg_preprocess_cwl_dag.py

@@ -17,7 +17,7 @@
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "sps"
 POD_LABEL = "sbg_preprocess_task"
-SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0"
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.3.0"
 
 # The path of the working directory where the CWL workflow is executed
 # (aka the starting directory for cwl-runner).

airflow/docker/custom_airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.10.0-python3.11
+FROM apache/airflow:2.10.3-python3.11
 
 RUN pip install cwltool==3.1.20240112164112
 RUN pip install boto3==1.34.89

airflow/plugins/unity_sps_utils.py

@@ -8,7 +8,7 @@
 # Shared constants
 POD_NAMESPACE = "sps"  # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_LABEL = "cwl_task"
-SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0"
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.3.0"
 
 NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator"
 NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload"

ogc-application-packages/sbg_preprocess_cwl_dag.json

@@ -1,6 +1,6 @@
 {
   "executionUnit": {
-    "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0",
+    "image": "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.3.0",
     "type": "docker"
   },
   "processDescription": {

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unity-sps"
-version = "2.2.0"
+version = "2.3.0"
 authors = [
     { name = "Drew Meyers", email = "[email protected]" },
     { name = "Luca Cinquini", email = "[email protected]" },
@@ -40,7 +40,7 @@ test = [
     "pytest-bdd==7.2.0",
     "pytest-mock==3.14.0",
     "requests==2.32.3",
-    "apache-airflow==2.10.0",
+    "apache-airflow==2.10.3",
     "kubernetes==29.0.0",
     "boto3==1.34.46",
     "backoff==2.2.1",

terraform-unity/README.md

@@ -187,7 +187,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_airflow_docker_images"></a> [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.2.0"<br>  }<br>}</pre> | no |
+| <a name="input_airflow_docker_images"></a> [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.3.0"<br>  }<br>}</pre> | no |
 | <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
 | <a name="input_airflow_webserver_username"></a> [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no |
 | <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br>    url                 = string<br>    ref                 = string<br>    dags_directory_path = string<br>  })</pre> | <pre>{<br>  "dags_directory_path": "airflow/dags",<br>  "ref": "2.2.0",<br>  "url": "https://github.com/unity-sds/unity-sps.git"<br>}</pre> | no |

terraform-unity/variables.tf

@@ -73,7 +73,7 @@ variable "airflow_docker_images" {
   default = {
     airflow = {
       name = "ghcr.io/unity-sds/unity-sps/sps-airflow"
-      tag  = "2.2.0"
+      tag  = "2.3.0"
     }
   }
 }