From d91eab0ea5d11dab2d3d75ca8589aa46e7064391 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 31 Jul 2024 16:59:21 -0700 Subject: [PATCH 1/3] Blank out metadata to make zip builds deterministic --- stack/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stack/utils.py b/stack/utils.py index 8c516bbf..b1bbba9b 100644 --- a/stack/utils.py +++ b/stack/utils.py @@ -143,6 +143,10 @@ def create_zip( # Store the file relative to the directory specified archive_name = os.path.relpath(full_path, dir_to_zip) zipf.write(full_path, archive_name) + + # Blank out metadata to make zip builds deterministic + zinfo = zipf.getinfo(archive_name) + zinfo.date_time = (1980, 1, 1, 0, 0, 0) return zip_filepath From 680fe0ed6847b8c0f40d01deecc71f7c12e58bfc Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 1 Aug 2024 14:56:53 -0700 Subject: [PATCH 2/3] Rework for better deterministic writes --- stack/utils.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/stack/utils.py b/stack/utils.py index b1bbba9b..34294738 100644 --- a/stack/utils.py +++ b/stack/utils.py @@ -126,9 +126,7 @@ def create_zip( :param compression: The compression type to use for the zip file (default is ZIP_DEFLATED) """ if not zip_filepath: - _, zip_filepath = mkstemp( - suffix=".zip", - ) + _, zip_filepath = mkstemp(suffix=".zip") else: zip_filepath = os.path.abspath(zip_filepath) @@ -142,11 +140,21 @@ def create_zip( ): # Store the file relative to the directory specified archive_name = os.path.relpath(full_path, dir_to_zip) - zipf.write(full_path, archive_name) - # Blank out metadata to make zip builds deterministic - zinfo = zipf.getinfo(archive_name) + # Create ZipInfo object to set the date_time manually + zinfo = zipfile.ZipInfo.from_file(full_path, archive_name) + # Set date_time to January 1, 1980 zinfo.date_time = (1980, 1, 1, 0, 0, 0) + + # Write file data to zip in chunks to be memory efficient + with open(full_path, "rb") as f: + with zipf.open(zinfo, "w") as zf: + while True: + chunk = f.read(1024 * 1024) # Read 1MB at a time + if not chunk: + break + zf.write(chunk) + return zip_filepath From 329bee58129dd2ca4eb4084a6fb28de950dfc7f7 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 31 Jul 2024 17:00:32 -0700 Subject: [PATCH 3/3] Make filename stable across deployments --- stack/cloud_function_ais_analysis.py | 4 +--- stack/cloud_function_historical_run.py | 4 +--- stack/cloud_function_scene_relevancy.py | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/stack/cloud_function_ais_analysis.py b/stack/cloud_function_ais_analysis.py index fc55050f..2ed78a8c 100644 --- a/stack/cloud_function_ais_analysis.py +++ b/stack/cloud_function_ais_analysis.py @@ -1,7 +1,5 @@ """cloud function to find slick culprits from AIS tracks""" -import time - import database import git import pulumi @@ -58,7 +56,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-ais"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=bucket.name, source=archive, ) diff --git a/stack/cloud_function_historical_run.py b/stack/cloud_function_historical_run.py index 8c7d1436..a8b0d871 100644 --- a/stack/cloud_function_historical_run.py +++ b/stack/cloud_function_historical_run.py @@ -1,7 +1,5 @@ """cloud function to select appropriate scenes (over water and IW) from SNS notification""" -import time - import cloud_function_scene_relevancy import cloud_run_orchestrator import database @@ -37,7 +35,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-historical-run"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=cloud_function_scene_relevancy.bucket.name, source=archive, ) diff --git a/stack/cloud_function_scene_relevancy.py b/stack/cloud_function_scene_relevancy.py index b8a1aa69..d5425f5f 100644 --- a/stack/cloud_function_scene_relevancy.py +++ b/stack/cloud_function_scene_relevancy.py @@ -1,7 +1,5 @@ """cloud function to select appropriate scenes (over water and IW) from SNS notification""" -import time - import cloud_run_orchestrator import database import pulumi @@ -60,7 +58,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-sr"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=bucket.name, source=archive, )