Skip to content

Commit

Permalink
add logging to know the timestamp used and skip early
Browse files Browse the repository at this point in the history
  • Loading branch information
ladrians committed Aug 27, 2024
1 parent fbec74e commit 594760c
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion amazon_s3/s3reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ def get_files_from_url(self) -> list[str]:

temp_dir = self.download_dir
skip_count = 0
logging.getLogger().info(f"Downloading files from '{self.bucket}' to {temp_dir}")
logging.getLogger().info(f"Downloading files from '{self.bucket}' to {temp_dir} since {min_filter_date}")

for f_item in f_values:
f_item_name = f_item.get('name', None)
Expand Down Expand Up @@ -328,6 +328,8 @@ def get_files_from_url(self) -> list[str]:
complete_url = f"{base_url}{next_url_href}" if next_url_href is not None else None

self.skip_count = skip_count
if len(self.element_ids) <= 0:
return []
if self.verbose:
_ = self.save_debug(self.element_dict, prefix='denodo')

Expand Down

0 comments on commit 594760c

Please sign in to comment.