Merge pull request #20 from emit-sds/red_id

Complex Redelivery Prevention

delivery_plume_tiler.py

@@ -192,6 +192,7 @@ def main(input_args=None):
     parser.add_argument('--visions_delivery', type=int, choices=[0,1,2],default=0)
     parser.add_argument('--n_cores', type=int, default=1)
     parser.add_argument('--overwrite', action='store_true')
+    parser.add_argument('--previous_plume_file', type=str, default='visions_delivery/plume_list.txt')
     parser.add_argument('--loglevel', type=str, default='DEBUG', help='logging verbosity')    
     parser.add_argument('--logfile', type=str, default=None, help='output file to write log to')    
     args = parser.parse_args(input_args)
@@ -201,6 +202,10 @@ def main(input_args=None):
 
     tile_dir = os.path.join(args.dest_dir, 'ch4_plume_tiles')
 
+    with open(args.previous_plume_file,'r') as f:
+        delivered_plume_names = f.read().splitlines()
+        delivered_plume_ids = [f'CH4_PlumeComplex-{x.split("_")[-1]}' for x in delivered_plume_names]
+
 
     all_plume_meta = json.load(open(f'{args.manual_del_dir}/combined_plume_metadata.json'))
     unique_fids = np.unique([sublist for feat in all_plume_meta['features'] for sublist in feat['properties']['Scene FIDs']])
@@ -249,23 +254,26 @@ def main(input_args=None):
     extra_metadata['title'] = "EMIT"
     extra_metadata['Units']= 'ppm m'
 
+    print(delivered_plume_ids)
 
     if args.visions_delivery != 2:
+
         jobs = []
         for _feat, feat in enumerate(all_plume_meta['features']):
             if _feat not in valid_plume_idx:
                 continue
+            complex_id= feat['properties']['Plume ID'].split('-')[-1]
+            if f'CH4_PlumeComplex-{complex_id.zfill(6)}' in delivered_plume_ids:
+                continue
             logging.info(f'Processing plume {_feat+1}/{len(all_plume_meta["features"])}')
 
-
             if feat['geometry']['type'] == 'Polygon':
                 outdir=os.path.join(args.dest_dir, feat['properties']['Scene FIDs'][0][4:12], 'l2bch4plm')
                 if os.path.isdir(outdir) is False:
                     subprocess.call(f'mkdir -p {outdir}',shell=True)
                 output_base = os.path.join(outdir, feat['properties']['Scene FIDs'][0] + '_' + feat['properties']['Plume ID'])
-                if args.overwrite or os.path.isfile(output_base) is False:
+                if args.overwrite or os.path.isfile(output_base + '.tif') is False:
                     jobs.append(single_plume_proc.remote(all_plume_meta, _feat, output_base, args.manual_del_dir, args.source_dir, extra_metadata))
-
         rreturn = [ray.get(jid) for jid in jobs]
 
 

get_plume_list.py

@@ -0,0 +1,57 @@
+
+import requests
+import math
+import time
+import argparse
+import numpy as np
+
+CMR_OPS = 'https://cmr.earthdata.nasa.gov/search' # CMR API Endpoint
+url = f'{CMR_OPS}/{"granules"}'
+collections = 'C2748088093-LPCLOUD' # L2BCH4PLM
+datetime_range = '2022-08-01T00:00:00Z,2024-12-31T08:00:00Z' # Overall date range of granules to be searched
+page_size=2000
+
+# open arguments
+parser = argparse.ArgumentParser(description='Get a list of granules from a CMR collection.')
+parser.add_argument('token', type=str, help='CMR token')
+parser.add_argument('outfile', type=str, help='output file')
+args = parser.parse_args()
+
+print(args.token)
+
+timelist = [] 
+for n in range(1,3):
+    print(url)
+    response = requests.get(url, 
+                            params={'concept_id': collections, 
+                                    'temporal': datetime_range, 
+                                    'page_size': page_size, 
+                                    'page_num': n,
+                                    },
+                            headers={
+                                'Accept': 'application/json',
+                                'Authorization':f'Bearer {args.token}'
+                                }
+                           )
+    print(response.status_code)
+    granules = response.json()['feed']['entry']
+
+    for g in granules:
+        timelist.extend([(g['title'],g['updated'])])
+# Show the count of how many granules meet the above parameters
+print(f"Number of granules found: {response.headers['CMR-Hits']}") # Resulting quantity of granules/items.
+
+
+
+
+updated = [x[1] for x in timelist]
+order = np.argsort(updated)
+
+timelist = [timelist[i] for i in order[::-1]]
+
+out_fidlist = [x[0] for x in timelist]
+
+with open(args.outfile,'w') as f:
+    for item in out_fidlist:
+        f.write("%s\n" % item)
+

parallel_mf.py

@@ -162,7 +162,7 @@ def main(input_args=None):
         saturation = None
         if args.l1b_bandmask_file is not None:
             logging.debug("loading pixel mask")
-            dilated_saturation, saturation = calculate_saturation_mask(args.l1b_bandmask_file, chunk_edges=[ce,chunk_edges[_ce+1]])
+            dilated_saturation, saturation = calculate_saturation_mask(args.l1b_bandmask_file, radiance, chunk_edges=[ce,chunk_edges[_ce+1]])
             good_pixel_mask[dilated_saturation] = False
 
         logging.debug("adding flare mask")

scrape_refine_upload.py

@@ -170,6 +170,10 @@ def add_fids(manual_annotations, coverage, manual_annotations_previous):
         msg = f'Deleting entry due to bad metadata - check input {manual_annotations_fid["features"][td]["properties"]}'
         logging.warning(msg)
         manual_annotations_fid['features'].pop(td)
+    logging.warning('Bad metadata for the following plumes:')
+    for td in np.array(todel)[::-1]:
+        msg = f'{manual_annotations_fid["features"][td]["properties"]["Plume ID"]}'
+        logging.warning(msg)
 
     updated_plumes = np.array([x for x in updated_plumes if x not in todel]) # shouldn't be necessary anymore, deosn't hurt
     for td in np.array(todel)[::-1]: