From 51b7203bf95ad84f1f668d3a51476854660483de Mon Sep 17 00:00:00 2001 From: ErnestaP Date: Fri, 6 Oct 2023 14:50:48 +0200 Subject: [PATCH] comments --- dags/common/pull_ftp.py | 16 ++++++++-------- dags/elsevier/repository.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dags/common/pull_ftp.py b/dags/common/pull_ftp.py index 3cf137de..7ae86dfe 100644 --- a/dags/common/pull_ftp.py +++ b/dags/common/pull_ftp.py @@ -15,30 +15,30 @@ def migrate_files( - filenames, + archives_names, s_ftp: SFTP_FTP_TYPE, repo: IRepository, logger: PrintLogger, ): - logger.msg("Processing files.", filenames=filenames) + logger.msg("Processing files.", filenames=archives_names) extracted_filenames = [] - for file_name in filenames: - logger.msg("Getting file from SFTP.", file=file_name) - file_bytes = s_ftp.get_file(file_name) + for archive_name in archives_names: + logger.msg("Getting file from SFTP.", file=archive_name) + file_bytes = s_ftp.get_file(archive_name) if zipfile.is_zipfile(file_bytes) or tarfile.is_tarfile(file_bytes): for (archive_file_content, s3_filename) in process_archive( - file_bytes=file_bytes, file_name=file_name + file_bytes=file_bytes, file_name=archive_name ): repo.save(s3_filename, io.BytesIO(archive_file_content)) if repo.is_meta(s3_filename): extracted_filenames.append("extracted/" + s3_filename) - repo.save(file_name, file_bytes) + repo.save(archive_name, file_bytes) else: logger.info( - "File is not zip or tar, processing the next one", file_name=file_name + "File is not zip or tar, processing the next one", file_name=archive_name ) continue diff --git a/dags/elsevier/repository.py b/dags/elsevier/repository.py index 69341e82..bf2964ce 100644 --- a/dags/elsevier/repository.py +++ b/dags/elsevier/repository.py @@ -53,7 +53,7 @@ def save(self, filename: str, obj: IO): def delete_all(self): self.s3.objects.all().delete() - def __find_all_extracted_files(self): + def _find_all_extracted_files(self): return [ f.key for f in self.s3.objects.filter(Prefix=self.EXTRACTED_DIR).all()