From d63fc7963b1e099af85dcc2ad9e835c2397e393f Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 31 Jul 2024 16:59:21 -0700 Subject: [PATCH 1/8] Blank out metadata to make zip builds deterministic --- stack/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stack/utils.py b/stack/utils.py index 8c516bbf..b1bbba9b 100644 --- a/stack/utils.py +++ b/stack/utils.py @@ -143,6 +143,10 @@ def create_zip( # Store the file relative to the directory specified archive_name = os.path.relpath(full_path, dir_to_zip) zipf.write(full_path, archive_name) + + # Blank out metadata to make zip builds deterministic + zinfo = zipf.getinfo(archive_name) + zinfo.date_time = (1980, 1, 1, 0, 0, 0) return zip_filepath From 8980e42939b6ea660d252f68c2727827b0161f8b Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 1 Aug 2024 14:56:53 -0700 Subject: [PATCH 2/8] Rework for better deterministic writes --- stack/utils.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/stack/utils.py b/stack/utils.py index b1bbba9b..34294738 100644 --- a/stack/utils.py +++ b/stack/utils.py @@ -126,9 +126,7 @@ def create_zip( :param compression: The compression type to use for the zip file (default is ZIP_DEFLATED) """ if not zip_filepath: - _, zip_filepath = mkstemp( - suffix=".zip", - ) + _, zip_filepath = mkstemp(suffix=".zip") else: zip_filepath = os.path.abspath(zip_filepath) @@ -142,11 +140,21 @@ def create_zip( ): # Store the file relative to the directory specified archive_name = os.path.relpath(full_path, dir_to_zip) - zipf.write(full_path, archive_name) - # Blank out metadata to make zip builds deterministic - zinfo = zipf.getinfo(archive_name) + # Create ZipInfo object to set the date_time manually + zinfo = zipfile.ZipInfo.from_file(full_path, archive_name) + # Set date_time to January 1, 1980 zinfo.date_time = (1980, 1, 1, 0, 0, 0) + + # Write file data to zip in chunks to be memory efficient + with open(full_path, "rb") as f: + with zipf.open(zinfo, "w") as zf: + while True: + chunk = f.read(1024 * 1024) # Read 1MB at a time + if not chunk: + break + zf.write(chunk) + return zip_filepath From 806147d2c7a62ae0cc7b78eb14404f767a53d28b Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 31 Jul 2024 17:00:32 -0700 Subject: [PATCH 3/8] Make filename stable across deployments --- stack/cloud_function_ais_analysis.py | 4 +--- stack/cloud_function_historical_run.py | 4 +--- stack/cloud_function_scene_relevancy.py | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/stack/cloud_function_ais_analysis.py b/stack/cloud_function_ais_analysis.py index fc55050f..2ed78a8c 100644 --- a/stack/cloud_function_ais_analysis.py +++ b/stack/cloud_function_ais_analysis.py @@ -1,7 +1,5 @@ """cloud function to find slick culprits from AIS tracks""" -import time - import database import git import pulumi @@ -58,7 +56,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-ais"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=bucket.name, source=archive, ) diff --git a/stack/cloud_function_historical_run.py b/stack/cloud_function_historical_run.py index 8c7d1436..a8b0d871 100644 --- a/stack/cloud_function_historical_run.py +++ b/stack/cloud_function_historical_run.py @@ -1,7 +1,5 @@ """cloud function to select appropriate scenes (over water and IW) from SNS notification""" -import time - import cloud_function_scene_relevancy import cloud_run_orchestrator import database @@ -37,7 +35,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-historical-run"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=cloud_function_scene_relevancy.bucket.name, source=archive, ) diff --git a/stack/cloud_function_scene_relevancy.py b/stack/cloud_function_scene_relevancy.py index 31526bc0..4a9c206d 100644 --- a/stack/cloud_function_scene_relevancy.py +++ b/stack/cloud_function_scene_relevancy.py @@ -1,7 +1,5 @@ """cloud function to select appropriate scenes (over water and IW) from SNS notification""" -import time - import cloud_run_orchestrator import database import pulumi @@ -60,7 +58,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-sr"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=bucket.name, source=archive, ) From c01bfc22d6d15c697a14a8f5b92466601640cb2c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Dec 2024 11:05:53 -0800 Subject: [PATCH 4/8] Experiment: Cache pulumi plugins --- .github/actions/deploy_infrastructure/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/deploy_infrastructure/action.yml b/.github/actions/deploy_infrastructure/action.yml index bfc958c7..16aac24d 100644 --- a/.github/actions/deploy_infrastructure/action.yml +++ b/.github/actions/deploy_infrastructure/action.yml @@ -43,6 +43,14 @@ runs: restore-keys: | ${{ runner.os }}-pip- + - name: Cache Pulumi plugins + uses: actions/cache@v4 + with: + path: ~/.pulumi/plugins + key: pulumi-plugins-${{ runner.os }}-${{ inputs.pulumi-version }} + restore-keys: | + pulumi-plugins-${{ runner.os }}- + - name: Configure GCP Credentials uses: google-github-actions/auth@v2 with: From 2f6862351642c8ee4cc44793bb009db754a7f78c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Dec 2024 11:32:36 -0800 Subject: [PATCH 5/8] Experiment: use GHA cache for docker builds --- images/cloud_run_images.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/images/cloud_run_images.py b/images/cloud_run_images.py index fddd2437..031e8484 100644 --- a/images/cloud_run_images.py +++ b/images/cloud_run_images.py @@ -65,6 +65,8 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_offset", + cache_from={"type": "gha"}, + cache_to={"type": "gha", "mode": "max"}, target="final", ), image_name=cloud_run_offset_tile_image_url, @@ -75,6 +77,8 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_orchestrator", + cache_from={"type": "gha"}, + cache_to={"type": "gha", "mode": "max"}, target="final", ), image_name=cloud_run_orchestrator_image_url, @@ -85,6 +89,8 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_tipg", + cache_from={"type": "gha"}, + cache_to={"type": "gha", "mode": "max"}, target="final", ), image_name=cloud_run_tipg_image_url, From d8507a3928aa26d03d712f6600d165a7671fdd83 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Dec 2024 11:49:18 -0800 Subject: [PATCH 6/8] Experiment: use GHA cache for docker builds [fix] --- images/cloud_run_images.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/images/cloud_run_images.py b/images/cloud_run_images.py index 031e8484..2def4805 100644 --- a/images/cloud_run_images.py +++ b/images/cloud_run_images.py @@ -65,8 +65,11 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_offset", - cache_from={"type": "gha"}, - cache_to={"type": "gha", "mode": "max"}, + args={ + "output": "type=docker", + "cache-from": "type=local,src=/tmp/.buildx-cache", + "cache-to": "type=local,dest=/tmp/.buildx-cache", + }, target="final", ), image_name=cloud_run_offset_tile_image_url, @@ -77,8 +80,11 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_orchestrator", - cache_from={"type": "gha"}, - cache_to={"type": "gha", "mode": "max"}, + args={ + "output": "type=docker", + "cache-from": "type=local,src=/tmp/.buildx-cache", + "cache-to": "type=local,dest=/tmp/.buildx-cache", + }, target="final", ), image_name=cloud_run_orchestrator_image_url, @@ -89,8 +95,11 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_tipg", - cache_from={"type": "gha"}, - cache_to={"type": "gha", "mode": "max"}, + args={ + "output": "type=docker", + "cache-from": "type=local,src=/tmp/.buildx-cache", + "cache-to": "type=local,dest=/tmp/.buildx-cache", + }, target="final", ), image_name=cloud_run_tipg_image_url, From b476f91d1c63bebb375b158b6fdb1cb1e2634553 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Dec 2024 13:50:09 -0800 Subject: [PATCH 7/8] Experiment: Test caching from last built image --- images/cloud_run_images.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/images/cloud_run_images.py b/images/cloud_run_images.py index 2def4805..90fd9ad9 100644 --- a/images/cloud_run_images.py +++ b/images/cloud_run_images.py @@ -65,11 +65,9 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_offset", - args={ - "output": "type=docker", - "cache-from": "type=local,src=/tmp/.buildx-cache", - "cache-to": "type=local,dest=/tmp/.buildx-cache", - }, + cache_from=docker.CacheFromArgs( + images=[cloud_run_offset_tile_image_url.apply(lambda url: f"{url}:latest")] + ), target="final", ), image_name=cloud_run_offset_tile_image_url, @@ -80,11 +78,9 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_orchestrator", - args={ - "output": "type=docker", - "cache-from": "type=local,src=/tmp/.buildx-cache", - "cache-to": "type=local,dest=/tmp/.buildx-cache", - }, + cache_from=docker.CacheFromArgs( + images=[cloud_run_orchestrator_image_url.apply(lambda url: f"{url}:latest")] + ), target="final", ), image_name=cloud_run_orchestrator_image_url, @@ -95,11 +91,9 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_tipg", - args={ - "output": "type=docker", - "cache-from": "type=local,src=/tmp/.buildx-cache", - "cache-to": "type=local,dest=/tmp/.buildx-cache", - }, + cache_from=docker.CacheFromArgs( + images=[cloud_run_tipg_image_url.apply(lambda url: f"{url}:latest")] + ), target="final", ), image_name=cloud_run_tipg_image_url, From aa94939d28f04cc215cc79b7e33f5ce10a4c831a Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Dec 2024 16:48:05 -0800 Subject: [PATCH 8/8] Set deployment concurrency --- .github/workflows/test_and_deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index a6bf52c4..91360b5f 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -111,6 +111,7 @@ jobs: name: Deploy [TEST] runs-on: ubuntu-20.04 environment: test + concurrency: test needs: [tests] if: github.event_name == 'workflow_dispatch' steps: @@ -133,6 +134,7 @@ jobs: name: Deploy [STAGING] runs-on: ubuntu-20.04 environment: staging + concurrency: staging needs: [tests] if: github.ref == 'refs/heads/main' steps: @@ -155,6 +157,7 @@ jobs: name: Deploy [PRODUCTION] runs-on: ubuntu-20.04 environment: prod20240903 + concurrency: prod20240903 needs: [tests] if: startsWith(github.event.ref, 'refs/tags') steps: