diff --git a/.github/actions/deploy_infrastructure/action.yml b/.github/actions/deploy_infrastructure/action.yml index bfc958c7..16aac24d 100644 --- a/.github/actions/deploy_infrastructure/action.yml +++ b/.github/actions/deploy_infrastructure/action.yml @@ -43,6 +43,14 @@ runs: restore-keys: | ${{ runner.os }}-pip- + - name: Cache Pulumi plugins + uses: actions/cache@v4 + with: + path: ~/.pulumi/plugins + key: pulumi-plugins-${{ runner.os }}-${{ inputs.pulumi-version }} + restore-keys: | + pulumi-plugins-${{ runner.os }}- + - name: Configure GCP Credentials uses: google-github-actions/auth@v2 with: diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index a6bf52c4..91360b5f 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -111,6 +111,7 @@ jobs: name: Deploy [TEST] runs-on: ubuntu-20.04 environment: test + concurrency: test needs: [tests] if: github.event_name == 'workflow_dispatch' steps: @@ -133,6 +134,7 @@ jobs: name: Deploy [STAGING] runs-on: ubuntu-20.04 environment: staging + concurrency: staging needs: [tests] if: github.ref == 'refs/heads/main' steps: @@ -155,6 +157,7 @@ jobs: name: Deploy [PRODUCTION] runs-on: ubuntu-20.04 environment: prod20240903 + concurrency: prod20240903 needs: [tests] if: startsWith(github.event.ref, 'refs/tags') steps: diff --git a/images/cloud_run_images.py b/images/cloud_run_images.py index fddd2437..90fd9ad9 100644 --- a/images/cloud_run_images.py +++ b/images/cloud_run_images.py @@ -65,6 +65,9 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_offset", + cache_from=docker.CacheFromArgs( + images=[cloud_run_offset_tile_image_url.apply(lambda url: f"{url}:latest")] + ), target="final", ), image_name=cloud_run_offset_tile_image_url, @@ -75,6 +78,9 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_orchestrator", + cache_from=docker.CacheFromArgs( + images=[cloud_run_orchestrator_image_url.apply(lambda url: f"{url}:latest")] + ), target="final", ), image_name=cloud_run_orchestrator_image_url, @@ -85,6 +91,9 @@ def construct_name(resource_name: str) -> str: build=docker.DockerBuildArgs( context="../", dockerfile="../Dockerfiles/Dockerfile.cloud_run_tipg", + cache_from=docker.CacheFromArgs( + images=[cloud_run_tipg_image_url.apply(lambda url: f"{url}:latest")] + ), target="final", ), image_name=cloud_run_tipg_image_url, diff --git a/stack/cloud_function_ais_analysis.py b/stack/cloud_function_ais_analysis.py index fc55050f..2ed78a8c 100644 --- a/stack/cloud_function_ais_analysis.py +++ b/stack/cloud_function_ais_analysis.py @@ -1,7 +1,5 @@ """cloud function to find slick culprits from AIS tracks""" -import time - import database import git import pulumi @@ -58,7 +56,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-ais"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=bucket.name, source=archive, ) diff --git a/stack/cloud_function_historical_run.py b/stack/cloud_function_historical_run.py index 8c7d1436..a8b0d871 100644 --- a/stack/cloud_function_historical_run.py +++ b/stack/cloud_function_historical_run.py @@ -1,7 +1,5 @@ """cloud function to select appropriate scenes (over water and IW) from SNS notification""" -import time - import cloud_function_scene_relevancy import cloud_run_orchestrator import database @@ -37,7 +35,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-historical-run"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=cloud_function_scene_relevancy.bucket.name, source=archive, ) diff --git a/stack/cloud_function_scene_relevancy.py b/stack/cloud_function_scene_relevancy.py index 31526bc0..4a9c206d 100644 --- a/stack/cloud_function_scene_relevancy.py +++ b/stack/cloud_function_scene_relevancy.py @@ -1,7 +1,5 @@ """cloud function to select appropriate scenes (over water and IW) from SNS notification""" -import time - import cloud_run_orchestrator import database import pulumi @@ -60,7 +58,7 @@ # source code. ("main.py" and "requirements.txt".) source_archive_object = storage.BucketObject( construct_name("source-cf-sr"), - name=f"handler.py-{time.time():f}", + name="handler.py", bucket=bucket.name, source=archive, ) diff --git a/stack/utils.py b/stack/utils.py index 8c516bbf..34294738 100644 --- a/stack/utils.py +++ b/stack/utils.py @@ -126,9 +126,7 @@ def create_zip( :param compression: The compression type to use for the zip file (default is ZIP_DEFLATED) """ if not zip_filepath: - _, zip_filepath = mkstemp( - suffix=".zip", - ) + _, zip_filepath = mkstemp(suffix=".zip") else: zip_filepath = os.path.abspath(zip_filepath) @@ -142,7 +140,21 @@ def create_zip( ): # Store the file relative to the directory specified archive_name = os.path.relpath(full_path, dir_to_zip) - zipf.write(full_path, archive_name) + + # Create ZipInfo object to set the date_time manually + zinfo = zipfile.ZipInfo.from_file(full_path, archive_name) + # Set date_time to January 1, 1980 + zinfo.date_time = (1980, 1, 1, 0, 0, 0) + + # Write file data to zip in chunks to be memory efficient + with open(full_path, "rb") as f: + with zipf.open(zinfo, "w") as zf: + while True: + chunk = f.read(1024 * 1024) # Read 1MB at a time + if not chunk: + break + zf.write(chunk) + return zip_filepath