diff --git a/data_subscriber/asf_cslc_download.py b/data_subscriber/asf_cslc_download.py index 02379b59..622b7541 100644 --- a/data_subscriber/asf_cslc_download.py +++ b/data_subscriber/asf_cslc_download.py @@ -101,6 +101,7 @@ def run_download(self, args, token, es_conn, netloc, username, password, cmr, jo try: head_object = boto3.client("s3").head_object(Bucket=bucket, Key=key) + logger.info(f"Adding CSLC file: {p}") except Exception as e: logger.error("Failed when accessing the S3 object:" + p) raise e @@ -191,7 +192,9 @@ def run_download(self, args, token, es_conn, netloc, username, password, cmr, jo raise Exception(f"Failed to get compressed cslc for frame {frame_id} and day index {latest_acq_cycle_index}") for ccslc in ccslcs: - c_cslc_s3paths.extend(ccslc["_source"]["metadata"]["product_s3_paths"]) + cslc_path = ccslc["_source"]["metadata"]["product_s3_paths"] + c_cslc_s3paths.extend(cslc_path) + logger.info(f"Adding {cslc_path} to c_cslc_s3paths") # Now acquire the Ionosphere files for the reference dates of the Compressed CSLC products logger.info(f"Downloading Ionosphere files for Compressed CSLCs") diff --git a/data_subscriber/cslc/cslc_query.py b/data_subscriber/cslc/cslc_query.py index 4037ad72..1ec32570 100644 --- a/data_subscriber/cslc/cslc_query.py +++ b/data_subscriber/cslc/cslc_query.py @@ -166,6 +166,7 @@ def determine_download_granules(self, granules): download_granules.append(granule) for granule in unsubmitted: + logger.info(f"Merging in unsubmitted granule {granule['unique_id']}: {granule['granule_id']} for triggering consideration") download_batch = by_download_batch_id[granule["download_batch_id"]] if granule["unique_id"] not in download_batch: download_batch[granule["unique_id"]] = granule diff --git a/data_subscriber/cslc_utils.py b/data_subscriber/cslc_utils.py index 7c440b19..33fa88d6 100644 --- a/data_subscriber/cslc_utils.py +++ b/data_subscriber/cslc_utils.py @@ -283,16 +283,18 @@ def get_dependent_compressed_cslcs(self, frame_id, day_index, eu): for mm in range(0, m - 1): # m parameter is inclusive of the current frame at hand for burst_id in self.frame_to_bursts[frame_id].burst_ids: ccslc_m_index = get_dependent_ccslc_index(prev_day_indices, mm, self.k, burst_id) - ccslcs = eu.query( + ccslc = eu.query( index=_C_CSLC_ES_INDEX_PATTERNS, body={"query": {"bool": {"must": [ {"term": {"metadata.ccslc_m_index.keyword": ccslc_m_index}}]}}}) # Should have exactly one compressed cslc per acq cycle per burst - if len(ccslcs) != 1: + if len(ccslc) != 1: logger.info("Compressed CSLCs for ccslc_m_index: %s was not found in GRQ ES", ccslc_m_index) return False + ccslcs.extend(ccslc) + logger.info("All Compresseed CSLSs for frame %s at day index %s found in GRQ ES", frame_id, day_index) return ccslcs def get_dependent_ccslc_index(prev_day_indices, mm, k, burst_id):