From f817d712e6efab6a4605cf01a579f0c379a14a64 Mon Sep 17 00:00:00 2001 From: ErnestaP Date: Mon, 2 Oct 2023 17:01:50 +0200 Subject: [PATCH] tests passing --- dags/common/pull_ftp.py | 5 +++-- dags/elsevier/{elsevier.py => elsevier_pull_ftp.py} | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) rename dags/elsevier/{elsevier.py => elsevier_pull_ftp.py} (97%) diff --git a/dags/common/pull_ftp.py b/dags/common/pull_ftp.py index c6feed19..3ddbf7c0 100644 --- a/dags/common/pull_ftp.py +++ b/dags/common/pull_ftp.py @@ -36,7 +36,7 @@ def migrate_files( repo.save(s3_filename, io.BytesIO(zip_file_content)) if repo.is_meta(s3_filename): extracted_filenames.append("extracted/" + s3_filename) - repo.save(s3_filename, io.BytesIO(zip_file_content)) + repo.save(_file, file_bytes) elif tarfile.is_tarfile(file_bytes): file_bytes.seek(0) @@ -48,7 +48,8 @@ def migrate_files( repo.save(s3_filename, io.BytesIO(tar_file_content)) if repo.is_meta(s3_filename): extracted_filenames.append("extracted/" + s3_filename) - repo.save(s3_filename, io.BytesIO(tar_file_content)) + repo.save(_file, file_bytes) + else: logger.info( "File is not zip or tar, processing the next one", file_name=_file diff --git a/dags/elsevier/elsevier.py b/dags/elsevier/elsevier_pull_ftp.py similarity index 97% rename from dags/elsevier/elsevier.py rename to dags/elsevier/elsevier_pull_ftp.py index f00e2ba5..dfd6738c 100644 --- a/dags/elsevier/elsevier.py +++ b/dags/elsevier/elsevier_pull_ftp.py @@ -47,7 +47,7 @@ def trigger_file_processing( publisher="elsevier", repo=repo, logger=logger, filenames=filenames or [] ) - # filenames = migrate_from_ftp() + filenames = migrate_from_ftp() # trigger_file_processing(filenames=filenames)