#925: Fixed docker command parameters and empty ES index exception ha…

…ndling
nasa · Jul 23, 2024 · ebff25a · ebff25a
1 parent 3de4de7
commit ebff25a
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 18 deletions.
diff --git a/data_subscriber/cslc_utils.py b/data_subscriber/cslc_utils.py
@@ -7,6 +7,7 @@
 import boto3
 import logging
 from functools import cache
+import elasticsearch
 
 from util import datasets_json_util
 from util.conf_util import SettingsConf
@@ -338,18 +339,25 @@ def save_blocked_download_job(eu, release_version, product_type, params, job_que
     )
 
 def get_pending_download_jobs(es):
-    return es.query(
-        index=PENDING_CSLC_DOWNLOADS_ES_INDEX_NAME,
-        body={"query": {
-                "bool": {
-                    "must": [
-                        {"term": {"submitted": False}},
-                        {"match": {"job_type": PENDING_TYPE_CSLC_DOWNLOAD}}
-                    ]
+    '''Retrieve all pending cslc download jobs from the ES index'''
+
+    try:
+        result =  es.query(
+            index=PENDING_CSLC_DOWNLOADS_ES_INDEX_NAME,
+            body={"query": {
+                    "bool": {
+                        "must": [
+                            {"term": {"submitted": False}},
+                            {"match": {"job_type": PENDING_TYPE_CSLC_DOWNLOAD}}
+                        ]
+                    }
                 }
             }
-        }
-    )
+        )
+    except elasticsearch.exceptions.NotFoundError as e:
+        return []
+
+    return result
 
 def mark_pending_download_job_submitted(es, doc_id, download_job_id):
     return es.update_document(

diff --git a/data_subscriber/submit_pending_jobs.py b/data_subscriber/submit_pending_jobs.py
@@ -39,9 +39,6 @@ def configure_logger():
 
 def run(argv: list[str]):
     logger.info(f"{argv=}")
-    #parser = create_parser()
-    #args = parser.parse_args(argv[1:])
-    #validate_args(args)
 
     job_submission_tasks = []
     disp_burst_map, burst_to_frames, datetime_to_frames = localize_disp_frame_burst_hist()

diff --git a/docker/hysds-io.json.cslc_query b/docker/hysds-io.json.cslc_query
@@ -57,16 +57,14 @@
       "from": "submitter",
       "placeholder": "e.g. --k=<k>",
       "type": "text",
-      "default": "--k=2",
-      "optional": true
+      "default": "--k=15"
     },
     {
       "name": "m",
       "from": "submitter",
       "placeholder": "e.g. --m=<m>",
       "type": "text",
-      "default": "--m=2",
-      "optional": true
+      "default": "--m=5"
     },
     {
       "name": "grace_mins",

diff --git a/docker/hysds-io.json.cslc_query_frame_range b/docker/hysds-io.json.cslc_query_frame_range
@@ -105,7 +105,28 @@
     {
       "name": "frame_id",
       "from": "submitter",
-      "placeholder": "e.g. --frame-id",
+      "placeholder": "e.g. --frame-id"
+    },
+    {
+      "name": "k",
+      "from": "submitter",
+      "placeholder": "e.g. --k=<k>",
+      "type": "text",
+      "default": "--k=15"
+    },
+    {
+      "name": "m",
+      "from": "submitter",
+      "placeholder": "e.g. --m=<m>",
+      "type": "text",
+      "default": "--m=5"
+    },
+    {
+      "name": "grace_mins",
+      "from": "submitter",
+      "placeholder": "e.g. --grace-mins=<grace_mins>",
+      "type": "text",
+      "default": "--grace-mins=120",
       "optional": true
     },
     {

diff --git a/docker/job-spec.json.cslc_query_frame_range b/docker/job-spec.json.cslc_query_frame_range
@@ -79,6 +79,18 @@
       "name": "frame_id",
       "destination": "positional"
     },
+    {
+      "name": "k",
+      "destination": "positional"
+    },
+    {
+      "name": "m",
+      "destination": "positional"
+    },
+    {
+      "name": "grace_mins",
+      "destination": "positional"
+    },
     {
       "name": "transfer_protocol",
       "destination": "positional"