Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

files: update extensions in files, update ACL in repo S3 #204

Merged
merged 2 commits into from
Apr 25, 2024
Merged

files: update extensions in files, update ACL in repo S3 #204

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e54a5a6
dags: file upload - adds extensions to files
pamfilos Apr 24, 2024
3af17a8
s3: update ACL on copy to repo bucket
pamfilos Apr 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions dags/common/scoap3_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,18 @@

logger = get_logger()

FILE_EXTENSIONS = {
"pdf": ".pdf",
"xml": ".xml",
"pdfa": ".a_2b.pdf"
}

def update_filename_extension(filename, type):
extension = FILE_EXTENSIONS.get(type, "")
if filename.endsWith(extension):
return filename
elif extension:
return f"{filename}{extension}"

class Scoap3Repository(IRepository):
def __init__(self):
Expand All @@ -18,7 +30,7 @@ def __init__(self):
self.s3 = S3Service(self.bucket)
self.client = self.s3.meta.client

def copy_file(self, source_bucket, source_key, prefix=None):
def copy_file(self, source_bucket, source_key, prefix=None, type=None):
if not self.upload_enabled:
return ""

Expand All @@ -27,6 +39,7 @@ def copy_file(self, source_bucket, source_key, prefix=None):

copy_source = {"Bucket": source_bucket, "Key": source_key}
filename = os.path.basename(source_key)
filename = update_filename_extension(filename, type)
destination_key = f"{self.upload_dir}/{prefix}/{filename}"

logger.info("Copying file from", copy_source=copy_source)
Expand All @@ -40,6 +53,7 @@ def copy_file(self, source_bucket, source_key, prefix=None):
"source_key": source_key,
},
"MetadataDirective": "REPLACE",
"ACL": "public-read"
},
)
logger.info(
Expand All @@ -51,7 +65,7 @@ def copy_files(self, bucket, files, prefix=None):
copied_files = {}
for type, path in files.items():
try:
copied_files[type] = self.copy_file(bucket, path, prefix=prefix)
copied_files[type] = self.copy_file(bucket, path, prefix=prefix, type=type)
except Exception as e:
logger.error("Failed to copy file.", error=str(e), type=type, path=path)
return copied_files
Expand All @@ -65,7 +79,7 @@ def download_files(self, files, prefix=None):
for type, url in files.items():
try:
downloaded_files[type] = self.download_and_upload_to_s3(
url, prefix=prefix
url, prefix=prefix, type=type
)
logger.info("Downloaded file", type=type, url=url)
except Exception as e:
Expand All @@ -86,7 +100,7 @@ def download_files_for_aps(self, files, prefix=None):
}
try:
downloaded_files[type] = self.download_and_upload_to_s3(
url, prefix=prefix, headers=headers
url, prefix=prefix, headers=headers, type=type
)
logger.info("Downloaded file", type=type, url=url)
except Exception as e:
Expand All @@ -95,14 +109,15 @@ def download_files_for_aps(self, files, prefix=None):
)
return downloaded_files

def download_and_upload_to_s3(self, url, prefix=None, headers=None):
def download_and_upload_to_s3(self, url, prefix=None, headers=None, type=None):
if not self.upload_enabled:
return ""

if not prefix:
prefix = str(uuid4())

filename = os.path.basename(url)
filename = update_filename_extension(filename, type)
destination_key = f"{self.upload_dir}/{prefix}/{filename}"

response = requests.get(url, headers=headers)
Expand Down
Loading