Adding aws debug step

airflow/dags/cwl_dag.py

@@ -13,6 +13,7 @@
 # The Kubernetes Pod that executes the CWL-Docker container
 # Must use elevated privileges to start/stop the Docker engine
 POD_TEMPLATE_FILE = "/opt/airflow/dags/docker_cwl_pod.yaml"
+POD_TEMPLATE_FILE2 = "/opt/airflow/dags/aws_cli_pod.yaml"
 
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "airflow"
@@ -42,6 +43,21 @@
               "args_as_json": Param(default_args_as_json, type="string"),
           })
 
+awscli_task = KubernetesPodOperator(
+    namespace=POD_NAMESPACE,
+    name="awscli-task",
+    is_delete_operator_pod=True,
+    hostnetwork=False,
+    startup_timeout_seconds=1000,
+    get_logs=True,
+    task_id="awscli-task",
+    full_pod_spec=k8s.V1Pod(
+        metadata=k8s.V1ObjectMeta(
+            name='awscli-pod-' + uuid.uuid4().hex),
+    ),
+    pod_template_file=POD_TEMPLATE_FILE2,
+    dag=dag)
+
 # Environment variables
 default_env_vars = {}
 
@@ -61,3 +77,5 @@
     pod_template_file=POD_TEMPLATE_FILE,
     arguments=["{{ params.cwl_workflow }}", "{{ params.args_as_json }}"],
     dag=dag)
+
+awscli_task >> cwl_task

airflow/docker/cwl/docker_cwl_entrypoint.sh

@@ -27,8 +27,11 @@ do
   sleep 1
 done
 
+# run sample AWS command
+aws s3 ls
+
 # wait for 60 minutes
-sleep 3600
+# sleep 3600
 
 # Execute CWL workflow
 source /usr/share/cwl/venv/bin/activate