From d0318f53458a6cefeb6dfbf325a8420e5fcef253 Mon Sep 17 00:00:00 2001
From: Luca Cinquini <luca.cinquini@jpl.nasa.gov>
Date: Mon, 26 Feb 2024 08:05:36 -0700
Subject: [PATCH] Adding aws debug step

---
 airflow/dags/cwl_dag.py                     | 18 ++++++++++++++++++
 airflow/docker/cwl/docker_cwl_entrypoint.sh |  5 ++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/airflow/dags/cwl_dag.py b/airflow/dags/cwl_dag.py
index ddc881b..51e4397 100644
--- a/airflow/dags/cwl_dag.py
+++ b/airflow/dags/cwl_dag.py
@@ -13,6 +13,7 @@
 # The Kubernetes Pod that executes the CWL-Docker container
 # Must use elevated privileges to start/stop the Docker engine
 POD_TEMPLATE_FILE = "/opt/airflow/dags/docker_cwl_pod.yaml"
+POD_TEMPLATE_FILE2 = "/opt/airflow/dags/aws_cli_pod.yaml"
 
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "airflow"
@@ -42,6 +43,21 @@
               "args_as_json": Param(default_args_as_json, type="string"),
           })
 
+awscli_task = KubernetesPodOperator(
+    namespace=POD_NAMESPACE,
+    name="awscli-task",
+    is_delete_operator_pod=True,
+    hostnetwork=False,
+    startup_timeout_seconds=1000,
+    get_logs=True,
+    task_id="awscli-task",
+    full_pod_spec=k8s.V1Pod(
+        metadata=k8s.V1ObjectMeta(
+            name='awscli-pod-' + uuid.uuid4().hex),
+    ),
+    pod_template_file=POD_TEMPLATE_FILE2,
+    dag=dag)
+
 # Environment variables
 default_env_vars = {}
 
@@ -61,3 +77,5 @@
     pod_template_file=POD_TEMPLATE_FILE,
     arguments=["{{ params.cwl_workflow }}", "{{ params.args_as_json }}"],
     dag=dag)
+
+awscli_task >> cwl_task
\ No newline at end of file
diff --git a/airflow/docker/cwl/docker_cwl_entrypoint.sh b/airflow/docker/cwl/docker_cwl_entrypoint.sh
index a4ef54e..637388c 100755
--- a/airflow/docker/cwl/docker_cwl_entrypoint.sh
+++ b/airflow/docker/cwl/docker_cwl_entrypoint.sh
@@ -27,8 +27,11 @@ do
   sleep 1
 done
 
+# run sample AWS command
+aws s3 ls
+
 # wait for 60 minutes
-sleep 3600
+# sleep 3600
 
 # Execute CWL workflow
 source /usr/share/cwl/venv/bin/activate