diff --git a/data_subscriber/cslc_utils.py b/data_subscriber/cslc_utils.py index 106f817b..eb0fa54c 100644 --- a/data_subscriber/cslc_utils.py +++ b/data_subscriber/cslc_utils.py @@ -7,6 +7,7 @@ import boto3 import logging from functools import cache +import elasticsearch from util import datasets_json_util from util.conf_util import SettingsConf @@ -338,18 +339,25 @@ def save_blocked_download_job(eu, release_version, product_type, params, job_que ) def get_pending_download_jobs(es): - return es.query( - index=PENDING_CSLC_DOWNLOADS_ES_INDEX_NAME, - body={"query": { - "bool": { - "must": [ - {"term": {"submitted": False}}, - {"match": {"job_type": PENDING_TYPE_CSLC_DOWNLOAD}} - ] + '''Retrieve all pending cslc download jobs from the ES index''' + + try: + result = es.query( + index=PENDING_CSLC_DOWNLOADS_ES_INDEX_NAME, + body={"query": { + "bool": { + "must": [ + {"term": {"submitted": False}}, + {"match": {"job_type": PENDING_TYPE_CSLC_DOWNLOAD}} + ] + } } } - } - ) + ) + except elasticsearch.exceptions.NotFoundError as e: + return [] + + return result def mark_pending_download_job_submitted(es, doc_id, download_job_id): return es.update_document( diff --git a/data_subscriber/submit_pending_jobs.py b/data_subscriber/submit_pending_jobs.py index 9ed8446a..9511e66a 100644 --- a/data_subscriber/submit_pending_jobs.py +++ b/data_subscriber/submit_pending_jobs.py @@ -39,9 +39,6 @@ def configure_logger(): def run(argv: list[str]): logger.info(f"{argv=}") - #parser = create_parser() - #args = parser.parse_args(argv[1:]) - #validate_args(args) job_submission_tasks = [] disp_burst_map, burst_to_frames, datetime_to_frames = localize_disp_frame_burst_hist() diff --git a/docker/hysds-io.json.cslc_query b/docker/hysds-io.json.cslc_query index 7dd333d5..86e70a02 100644 --- a/docker/hysds-io.json.cslc_query +++ b/docker/hysds-io.json.cslc_query @@ -57,16 +57,14 @@ "from": "submitter", "placeholder": "e.g. --k=", "type": "text", - "default": "--k=2", - "optional": true + "default": "--k=15" }, { "name": "m", "from": "submitter", "placeholder": "e.g. --m=", "type": "text", - "default": "--m=2", - "optional": true + "default": "--m=5" }, { "name": "grace_mins", diff --git a/docker/hysds-io.json.cslc_query_frame_range b/docker/hysds-io.json.cslc_query_frame_range index 1f84c1e4..2a3828bf 100644 --- a/docker/hysds-io.json.cslc_query_frame_range +++ b/docker/hysds-io.json.cslc_query_frame_range @@ -105,7 +105,28 @@ { "name": "frame_id", "from": "submitter", - "placeholder": "e.g. --frame-id", + "placeholder": "e.g. --frame-id" + }, + { + "name": "k", + "from": "submitter", + "placeholder": "e.g. --k=", + "type": "text", + "default": "--k=15" + }, + { + "name": "m", + "from": "submitter", + "placeholder": "e.g. --m=", + "type": "text", + "default": "--m=5" + }, + { + "name": "grace_mins", + "from": "submitter", + "placeholder": "e.g. --grace-mins=", + "type": "text", + "default": "--grace-mins=120", "optional": true }, { diff --git a/docker/job-spec.json.cslc_query_frame_range b/docker/job-spec.json.cslc_query_frame_range index 2ae945b7..4e969e18 100644 --- a/docker/job-spec.json.cslc_query_frame_range +++ b/docker/job-spec.json.cslc_query_frame_range @@ -79,6 +79,18 @@ "name": "frame_id", "destination": "positional" }, + { + "name": "k", + "destination": "positional" + }, + { + "name": "m", + "destination": "positional" + }, + { + "name": "grace_mins", + "destination": "positional" + }, { "name": "transfer_protocol", "destination": "positional"