fix(sat-etl): Fix removal of downloaded scans

openclimatefix · Aug 21, 2024 · 3835720 · 3835720
1 parent 5912fbb
commit 3835720
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/containers/sat/download_process_sat.py b/containers/sat/download_process_sat.py
@@ -627,15 +627,14 @@ def run(args: argparse.Namespace) -> None:
     if len(scan_times) > cpu_count():
         log.debug(f"Concurrency: {cpu_count()}")
         pool = Pool(max(cpu_count(), 10))  # EUMDAC only allows for 10 concurrent requests
-        raw_paths = pool.starmap(
+        results: list[list[pathlib.Path]] = pool.starmap(
             download_scans,
             [(sat_config, folder, scan_time, token) for scan_time in scan_times],
         )
         pool.close()
         pool.join()
-        raw_paths = list(itertools.chain(raw_paths))
+        raw_paths.extend(list(itertools.chain(*results)))
     else:
-        raw_paths = []
         for scan_time in scan_times:
             result: list[pathlib.Path] = download_scans(sat_config, folder, scan_time, token)
             if len(result) > 0: