Skip to content

Commit

Permalink
fix(sat): Prevent error with existing zarr coord store (#122)
Browse files Browse the repository at this point in the history
* fix(sat): Prevent error with existing zarr coord store

* fix(sat-etl): Fix removal of downloaded scans
  • Loading branch information
devsjc authored Aug 27, 2024
1 parent ade5656 commit 0d953e9
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions containers/sat/download_process_sat.py
Original file line number Diff line number Diff line change
Expand Up @@ -541,7 +541,7 @@ def _rewrite_zarr_times(output_name: str) -> None:
# Need to remove these encodings to avoid chunking
del ds.time.encoding["chunks"]
del ds.time.encoding["preferred_chunks"]
ds.to_zarr(f"{output_name.split('.zarr')[0]}_coord.zarr", consolidated=True)
ds.to_zarr(f"{output_name.split('.zarr')[0]}_coord.zarr", consolidated=True, mode="w")
# Remove current time ones
shutil.rmtree(f"{output_name}/time/")
# Add new time ones
Expand Down Expand Up @@ -627,15 +627,14 @@ def run(args: argparse.Namespace) -> None:
if len(scan_times) > cpu_count():
log.debug(f"Concurrency: {cpu_count()}")
pool = Pool(max(cpu_count(), 10)) # EUMDAC only allows for 10 concurrent requests
raw_paths = pool.starmap(
results: list[list[pathlib.Path]] = pool.starmap(
download_scans,
[(sat_config, folder, scan_time, token) for scan_time in scan_times],
)
pool.close()
pool.join()
raw_paths = list(itertools.chain(raw_paths))
raw_paths.extend(list(itertools.chain(*results)))
else:
raw_paths = []
for scan_time in scan_times:
result: list[pathlib.Path] = download_scans(sat_config, folder, scan_time, token)
if len(result) > 0:
Expand Down

0 comments on commit 0d953e9

Please sign in to comment.