From 5daf0e8107f85c7db373750572f35517304b326b Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 11:49:18 +0100
Subject: [PATCH 1/8] bump flake.lock
---
flake.lock | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/flake.lock b/flake.lock
index db54019..21f2ceb 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,11 +2,11 @@
"nodes": {
"nixpkgs": {
"locked": {
- "lastModified": 1713532629,
- "narHash": "sha256-8iwNoSDOCKFnDF7f8XReiztpESA0GyFieKhWAaG7jrw=",
+ "lastModified": 1735036762,
+ "narHash": "sha256-CDGLmnmuAFFBplTgrILFkYViO0OUUAyRU4V02WYggnE=",
"owner": "NixOS",
"repo": "nixpkgs",
- "rev": "7f62671ffcb37436b3df7d6ae44dfdca9e5a069d",
+ "rev": "2f15fd55c63346aa315ede30927f8e1f66a2d800",
"type": "github"
},
"original": {
From 337b0e75fd4368a06f49936e3a8fb3a84576ca1f Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 11:49:28 +0100
Subject: [PATCH 2/8] Rename delete_images to delete_images_by_name
---
upload-ami/pyproject.toml | 2 +-
.../upload_ami/{delete_images.py => delete_images_by_name.py} | 0
2 files changed, 1 insertion(+), 1 deletion(-)
rename upload-ami/src/upload_ami/{delete_images.py => delete_images_by_name.py} (100%)
diff --git a/upload-ami/pyproject.toml b/upload-ami/pyproject.toml
index 1b157de..baa3ec9 100644
--- a/upload-ami/pyproject.toml
+++ b/upload-ami/pyproject.toml
@@ -17,6 +17,6 @@ disable-image-block-public-access = "upload_ami.disable_image_block_public_acces
enable-regions = "upload_ami.enable_regions:main"
request-public-ami-quota-increase = "upload_ami.request_public_ami_quota_increase:main"
describe-images = "upload_ami.describe_images:main"
-delete-images = "upload_ami.delete_images:main"
+delete-images-by-name = "upload_ami.delete_images_by_name:main"
[tool.mypy]
strict=true
diff --git a/upload-ami/src/upload_ami/delete_images.py b/upload-ami/src/upload_ami/delete_images_by_name.py
similarity index 100%
rename from upload-ami/src/upload_ami/delete_images.py
rename to upload-ami/src/upload_ami/delete_images_by_name.py
From a9860b964e97bb29e01bce2ea63effb6241d8be2 Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 12:16:24 +0100
Subject: [PATCH 3/8] add garbage collection
---
.envrc | 2 +-
upload-ami/pyproject.toml | 2 +
.../upload_ami/delete_deprecated_images.py | 72 +++++++++++++++++++
.../upload_ami/delete_orphaned_snapshots.py | 52 ++++++++++++++
4 files changed, 127 insertions(+), 1 deletion(-)
create mode 100644 upload-ami/src/upload_ami/delete_deprecated_images.py
create mode 100644 upload-ami/src/upload_ami/delete_orphaned_snapshots.py
diff --git a/.envrc b/.envrc
index 901323e..2a58f13 100644
--- a/.envrc
+++ b/.envrc
@@ -2,4 +2,4 @@ source $(direnv fetchurl "https://raw.githubusercontent.com/numtide/prj-spec/mai
export AWS_CONFIG_FILE=$PRJ_CONFIG_HOME/aws/config
-use flake
+use flake .#upload-ami
diff --git a/upload-ami/pyproject.toml b/upload-ami/pyproject.toml
index baa3ec9..84f2971 100644
--- a/upload-ami/pyproject.toml
+++ b/upload-ami/pyproject.toml
@@ -18,5 +18,7 @@ enable-regions = "upload_ami.enable_regions:main"
request-public-ami-quota-increase = "upload_ami.request_public_ami_quota_increase:main"
describe-images = "upload_ami.describe_images:main"
delete-images-by-name = "upload_ami.delete_images_by_name:main"
+delete-deprecated-images = "upload_ami.delete_deprecated_images:main"
+delete-orphaned-snapshots = "upload_ami.delete_orphaned_snapshots:main"
[tool.mypy]
strict=true
diff --git a/upload-ami/src/upload_ami/delete_deprecated_images.py b/upload-ami/src/upload_ami/delete_deprecated_images.py
new file mode 100644
index 0000000..e6d7258
--- /dev/null
+++ b/upload-ami/src/upload_ami/delete_deprecated_images.py
@@ -0,0 +1,72 @@
+import logging
+import boto3
+from mypy_boto3_ec2 import EC2Client
+import argparse
+import botocore.exceptions
+import datetime
+
+logger = logging.getLogger(__name__)
+
+
+def delete_deprecated_images(ec2: EC2Client, dry_run: bool) -> None:
+ """
+ Delete an image by its name.
+
+ Name can be a filter
+
+ Idempotent, unlike nuke
+ """
+
+ images_paginator = ec2.get_paginator("describe_images")
+ images_iterator = images_paginator.paginate(Owners=["self"])
+ for pages in images_iterator:
+ for image in pages["Images"]:
+ deprecation_time = image.get("DeprecationTime")
+ if deprecation_time:
+ current_time = datetime.datetime.now(deprecation_time.tzinfo)
+ deprecation_time = datetime.datetime.strptime(
+ deprecation_time, "%Y-%m-%dT%H:%M:%SZ"
+ )
+ if current_time >= deprecation_time:
+ logger.info(f"Deleting image {image['ImageId']}")
+ try:
+ ec2.deregister_image(ImageId=image["ImageId"], DryRun=dry_run)
+ except botocore.exceptions.ClientError as e:
+ if "DryRunOperation" in str(e):
+ logger.info(f"Would have deleted image {image['ImageId']}")
+ else:
+ raise
+ snapshot_id = image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"]
+ logger.info(f"Deleting snapshot {snapshot_id}")
+ try:
+ ec2.delete_snapshot(SnapshotId=snapshot_id, DryRun=dry_run)
+ except botocore.exceptions.ClientError as e:
+ if "DryRunOperation" in str(e):
+ logger.info(f"Would have deleted snapshot {snapshot_id}")
+ else:
+ raise
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Do not actually delete anything, just log what would be deleted",
+ )
+ logging.basicConfig(level=logging.INFO)
+ ec2: EC2Client = boto3.client("ec2")
+
+ args = parser.parse_args()
+ regions = ec2.describe_regions()["Regions"]
+ for region in regions:
+ assert "RegionName" in region
+ ec2r = boto3.client("ec2", region_name=region["RegionName"])
+ logger.info(
+ f"Deleting image by name {args.image_name} in {region['RegionName']}"
+ )
+ delete_deprecated_images(ec2r, args.dry_run)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/upload-ami/src/upload_ami/delete_orphaned_snapshots.py b/upload-ami/src/upload_ami/delete_orphaned_snapshots.py
new file mode 100644
index 0000000..949bb42
--- /dev/null
+++ b/upload-ami/src/upload_ami/delete_orphaned_snapshots.py
@@ -0,0 +1,52 @@
+import logging
+import boto3
+from mypy_boto3_ec2 import EC2Client
+import argparse
+import botocore.exceptions
+import datetime
+
+
+def delete_orphaned_snapshots(ec2: EC2Client, dry_run: bool) -> None:
+ snapshot_paginator = ec2.get_paginator("describe_snapshots")
+ snapshot_iterator = snapshot_paginator.paginate(
+ OwnerIds=["self"], Filters=[{"Name": "tag:ManagedBy", "Values": ["NixOS/amis"]}]
+ )
+ for pages in snapshot_iterator:
+ for snapshot in pages["Snapshots"]:
+ snapshot_id = snapshot["SnapshotId"]
+ images = ec2.describe_images(
+ Filters=[
+ {
+ "Name": "block-device-mapping.snapshot-id",
+ "Values": [snapshot_id],
+ }
+ ],
+ MaxResults=1,
+ )
+ if len(images["Images"]) == 0:
+ logging.info(f"Deleting orphaned snapshot {snapshot_id}")
+ try:
+ ec2.delete_snapshot(SnapshotId=snapshot_id, DryRun=dry_run)
+ except botocore.exceptions.ClientError as e:
+ if "DryRunOperation" in str(e):
+ logging.info(
+ f"Would have deleted orphaned snapshot {snapshot_id}"
+ )
+ else:
+ raise
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Do not actually delete anything, just log what would be deleted",
+ )
+ logging.basicConfig(level=logging.INFO)
+ ec2: EC2Client = boto3.client("ec2")
+ args = parser.parse_args()
+ regions = ec2.describe_regions()["Regions"]
+ for region in regions:
+ ec2 = boto3.client("ec2", region_name=region["RegionName"])
+ delete_orphaned_snapshots(ec2, args.dry_run)
From 594f2de22c2931c4b53b734f05edbd8f0c361fad Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 12:34:28 +0100
Subject: [PATCH 4/8] use 24.11 because awscli2 build failure
https://github.com/NixOS/nixpkgs/issues/367876
---
.envrc | 2 +-
flake.lock | 7 ++++---
flake.nix | 2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/.envrc b/.envrc
index 2a58f13..2db9579 100644
--- a/.envrc
+++ b/.envrc
@@ -2,4 +2,4 @@ source $(direnv fetchurl "https://raw.githubusercontent.com/numtide/prj-spec/mai
export AWS_CONFIG_FILE=$PRJ_CONFIG_HOME/aws/config
-use flake .#upload-ami
+use flake
diff --git a/flake.lock b/flake.lock
index 21f2ceb..b812975 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,15 +2,16 @@
"nodes": {
"nixpkgs": {
"locked": {
- "lastModified": 1735036762,
- "narHash": "sha256-CDGLmnmuAFFBplTgrILFkYViO0OUUAyRU4V02WYggnE=",
+ "lastModified": 1734875076,
+ "narHash": "sha256-Pzyb+YNG5u3zP79zoi8HXYMs15Q5dfjDgwCdUI5B0nY=",
"owner": "NixOS",
"repo": "nixpkgs",
- "rev": "2f15fd55c63346aa315ede30927f8e1f66a2d800",
+ "rev": "1807c2b91223227ad5599d7067a61665c52d1295",
"type": "github"
},
"original": {
"owner": "NixOS",
+ "ref": "nixos-24.11",
"repo": "nixpkgs",
"type": "github"
}
diff --git a/flake.nix b/flake.nix
index 74f952a..74ca1e2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
description = "A very basic flake";
inputs = {
- nixpkgs.url = "github:NixOS/nixpkgs";
+ nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.11";
};
outputs = { self, nixpkgs, ... }:
From 072e5f93dafe56ae5cd8898a281d7e2d2bd10fa4 Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 12:59:49 +0100
Subject: [PATCH 5/8] Fix
---
flake.nix | 52 +++++++++----------
upload-ami/default.nix | 4 ++
.../upload_ami/delete_deprecated_images.py | 30 ++++++-----
.../upload_ami/delete_orphaned_snapshots.py | 7 +--
4 files changed, 52 insertions(+), 41 deletions(-)
diff --git a/flake.nix b/flake.nix
index 74ca1e2..4189671 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,19 +1,19 @@
{
description = "A very basic flake";
- inputs = {
- nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.11";
- };
+ inputs = { nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.11"; };
outputs = { self, nixpkgs, ... }:
- let inherit (nixpkgs) lib; in
+ let inherit (nixpkgs) lib;
- {
+ in {
nixosModules = {
ec2-instance-connect = ./modules/ec2-instance-connect.nix;
- legacyAmazonProfile = nixpkgs + "nixos/modules/virtualisation/amazon-image.nix";
- legacyAmazonImage = nixpkgs + "/nixos/maintainers/scripts/ec2/amazon-image.nix";
+ legacyAmazonProfile = nixpkgs
+ + "nixos/modules/virtualisation/amazon-image.nix";
+ legacyAmazonImage = nixpkgs
+ + "/nixos/maintainers/scripts/ec2/amazon-image.nix";
amazonProfile = ./modules/amazon-profile.nix;
amazonImage = ./modules/amazon-image.nix;
@@ -27,11 +27,14 @@
};
};
- lib.supportedSystems = [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" ];
+ lib.supportedSystems =
+ [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" ];
packages = lib.genAttrs self.lib.supportedSystems (system:
- let pkgs = nixpkgs.legacyPackages.${system}; in {
- ec2-instance-connect = pkgs.callPackage ./packages/ec2-instance-connect.nix { };
+ let pkgs = nixpkgs.legacyPackages.${system};
+ in {
+ ec2-instance-connect =
+ pkgs.callPackage ./packages/ec2-instance-connect.nix { };
amazon-ec2-metadata-mock = pkgs.buildGoModule rec {
pname = "amazon-ec2-metadata-mock";
version = "1.11.2";
@@ -64,7 +67,10 @@
boot.loader.grub.enable = false;
boot.loader.systemd-boot.enable = true;
}
- { ec2.efi = true; amazonImage.sizeMB = "auto"; }
+ {
+ ec2.efi = true;
+ amazonImage.sizeMB = "auto";
+ }
self.nixosModules.version
];
}).config.system.build.amazonImage;
@@ -74,11 +80,12 @@
apps = lib.genAttrs self.lib.supportedSystems (system:
let
upload-ami = self.packages.${system}.upload-ami;
- mkApp = name: _: { type = "app"; program = "${upload-ami}/bin/${name}"; };
- in
- lib.mapAttrs mkApp self.packages.${system}.upload-ami.passthru.pyproject.project.scripts
- );
-
+ mkApp = name: _: {
+ type = "app";
+ program = "${upload-ami}/bin/${name}";
+ };
+ in lib.mapAttrs mkApp
+ self.packages.${system}.upload-ami.passthru.pyproject.project.scripts);
# TODO: unfortunately I don't have access to a aarch64-linux hardware with virtualisation support
checks = lib.genAttrs [ "x86_64-linux" ] (system:
@@ -98,8 +105,7 @@
};
};
- in
- {
+ in {
resize-partition = lib.nixos.runTest {
hostPkgs = pkgs;
imports = [ config ./tests/resize-partition.nix ];
@@ -110,13 +116,7 @@
};
});
- devShells = lib.genAttrs [ "x86_64-linux" "aarch64-darwin" ] (system: {
- default = let pkgs = nixpkgs.legacyPackages.${system}; in pkgs.mkShell {
- nativeBuildInputs = [
- pkgs.awscli2
- pkgs.opentofu
- ];
- };
- });
+ devShells = lib.genAttrs [ "x86_64-linux" "aarch64-darwin" ]
+ (system: { default = self.packages.${system}.upload-ami; });
};
}
diff --git a/upload-ami/default.nix b/upload-ami/default.nix
index c4f16ae..2663af9 100644
--- a/upload-ami/default.nix
+++ b/upload-ami/default.nix
@@ -1,5 +1,7 @@
{ buildPythonApplication
, python3Packages
+, awscli2
+, opentofu
, lib
}:
@@ -33,6 +35,8 @@ buildPythonApplication {
pyproject = true;
nativeBuildInputs =
map (name: python3Packages.${name}) pyproject.build-system.requires ++ [
+ opentofu
+ awscli2
python3Packages.mypy
python3Packages.black
];
diff --git a/upload-ami/src/upload_ami/delete_deprecated_images.py b/upload-ami/src/upload_ami/delete_deprecated_images.py
index e6d7258..382b408 100644
--- a/upload-ami/src/upload_ami/delete_deprecated_images.py
+++ b/upload-ami/src/upload_ami/delete_deprecated_images.py
@@ -21,14 +21,19 @@ def delete_deprecated_images(ec2: EC2Client, dry_run: bool) -> None:
images_iterator = images_paginator.paginate(Owners=["self"])
for pages in images_iterator:
for image in pages["Images"]:
- deprecation_time = image.get("DeprecationTime")
- if deprecation_time:
- current_time = datetime.datetime.now(deprecation_time.tzinfo)
- deprecation_time = datetime.datetime.strptime(
- deprecation_time, "%Y-%m-%dT%H:%M:%SZ"
+ if "DeprecationTime" in image:
+ # HACK: As python can not parse ISO8601 strings with
+ # milliseconds, but it **can** produce them, instead of parsing
+ # the datetime from the API, we format the current time as an
+ # ISO8601 string and compare the strings. This works because
+ # ISO8601 strings are lexicographically comparable.
+ current_time = datetime.datetime.isoformat(
+ datetime.datetime.now(), timespec="milliseconds"
)
- if current_time >= deprecation_time:
- logger.info(f"Deleting image {image['ImageId']}")
+ if current_time >= image["DeprecationTime"]:
+ assert "ImageId" in image
+ assert "Name" in image
+ logger.info(f"Deleting image {image['Name']} : {image['ImageId']}. DeprecationTime: {image['DeprecationTime']}")
try:
ec2.deregister_image(ImageId=image["ImageId"], DryRun=dry_run)
except botocore.exceptions.ClientError as e:
@@ -36,6 +41,9 @@ def delete_deprecated_images(ec2: EC2Client, dry_run: bool) -> None:
logger.info(f"Would have deleted image {image['ImageId']}")
else:
raise
+ assert "BlockDeviceMappings" in image
+ assert "Ebs" in image["BlockDeviceMappings"][0]
+ assert "SnapshotId" in image["BlockDeviceMappings"][0]["Ebs"]
snapshot_id = image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"]
logger.info(f"Deleting snapshot {snapshot_id}")
try:
@@ -55,16 +63,14 @@ def main() -> None:
help="Do not actually delete anything, just log what would be deleted",
)
logging.basicConfig(level=logging.INFO)
- ec2: EC2Client = boto3.client("ec2")
+ ec2: EC2Client = boto3.client("ec2") # type: ignore
args = parser.parse_args()
regions = ec2.describe_regions()["Regions"]
for region in regions:
assert "RegionName" in region
- ec2r = boto3.client("ec2", region_name=region["RegionName"])
- logger.info(
- f"Deleting image by name {args.image_name} in {region['RegionName']}"
- )
+ ec2r = boto3.client("ec2", region_name=region["RegionName"]) # type: ignore
+ logging.info(f"Checking region {region['RegionName']}")
delete_deprecated_images(ec2r, args.dry_run)
diff --git a/upload-ami/src/upload_ami/delete_orphaned_snapshots.py b/upload-ami/src/upload_ami/delete_orphaned_snapshots.py
index 949bb42..06402bc 100644
--- a/upload-ami/src/upload_ami/delete_orphaned_snapshots.py
+++ b/upload-ami/src/upload_ami/delete_orphaned_snapshots.py
@@ -3,7 +3,6 @@
from mypy_boto3_ec2 import EC2Client
import argparse
import botocore.exceptions
-import datetime
def delete_orphaned_snapshots(ec2: EC2Client, dry_run: bool) -> None:
@@ -13,6 +12,7 @@ def delete_orphaned_snapshots(ec2: EC2Client, dry_run: bool) -> None:
)
for pages in snapshot_iterator:
for snapshot in pages["Snapshots"]:
+ assert "SnapshotId" in snapshot
snapshot_id = snapshot["SnapshotId"]
images = ec2.describe_images(
Filters=[
@@ -44,9 +44,10 @@ def main() -> None:
help="Do not actually delete anything, just log what would be deleted",
)
logging.basicConfig(level=logging.INFO)
- ec2: EC2Client = boto3.client("ec2")
+ ec2: EC2Client = boto3.client("ec2") # type: ignore 0
args = parser.parse_args()
regions = ec2.describe_regions()["Regions"]
for region in regions:
- ec2 = boto3.client("ec2", region_name=region["RegionName"])
+ assert "RegionName" in region
+ ec2 = boto3.client("ec2", region_name=region["RegionName"]) # type: ignore
delete_orphaned_snapshots(ec2, args.dry_run)
From 91c2a8ab0b40cfc306841ae9508edd1d5479970c Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 13:00:19 +0100
Subject: [PATCH 6/8] wihtespace
---
.envrc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.envrc b/.envrc
index 2db9579..901323e 100644
--- a/.envrc
+++ b/.envrc
@@ -2,4 +2,4 @@ source $(direnv fetchurl "https://raw.githubusercontent.com/numtide/prj-spec/mai
export AWS_CONFIG_FILE=$PRJ_CONFIG_HOME/aws/config
-use flake
+use flake
From b75287c40544508ff3462263191751470f9b8a9f Mon Sep 17 00:00:00 2001
From: Arian van Putten
Date: Tue, 24 Dec 2024 13:31:47 +0100
Subject: [PATCH 7/8] Update docs and actively deprecate
---
.github/workflows/upload-legacy-ami.yml | 5 +
site/index.html | 111 ++++++++----------
.../upload_ami/delete_deprecated_images.py | 4 +-
.../upload_ami/delete_orphaned_snapshots.py | 8 +-
4 files changed, 64 insertions(+), 64 deletions(-)
diff --git a/.github/workflows/upload-legacy-ami.yml b/.github/workflows/upload-legacy-ami.yml
index d3b385d..43e30f4 100644
--- a/.github/workflows/upload-legacy-ami.yml
+++ b/.github/workflows/upload-legacy-ami.yml
@@ -86,6 +86,11 @@ jobs:
--copy-to-regions \
--public
+ - name: Delete deprecated AMIs
+ if: github.ref == 'refs/heads/main'
+ run: |
+ nix run .#delete-deprecated-images
+
deploy-pages:
name: Deploy images page
if: github.ref == 'refs/heads/main'
diff --git a/site/index.html b/site/index.html
index 88a1cca..087ab52 100644
--- a/site/index.html
+++ b/site/index.html
@@ -1,12 +1,11 @@
-
-
+
NixOS Amazon Images / AMIs
-
-
-
+
+
Amazon Images / AMIs
- NixOS can be deployed to Amazon EC2 using our official AMI. We publish
+ NixOS can be deployed to Amazon EC2 using our official AMI. We publish
AMIs to all AWS regions for both `x86_64` and `arm64` on a weekly basis.
- We will start deprecating and garbage collecting images older than 90 days
- in the future.
+
We deprecate and garbage collecting images older than 90 days.
This is why we suggest using a terraform data source or the AWS API to query
- for the latest AMI.
-
- NixOS images are published under AWS Account ID
-
+ for the latest AMI.
+ NixOS images are published under AWS Account ID
+
Terraform / OpenTofu
You can use terraform to query for the latest image
@@ -249,52 +245,49 @@ AWS CLI
aws ec2 describe-images --owners _OWNER_ID_ --filter 'Name=name,Values=nixos/24.11*' 'Name=architecture,Values=arm64' --query 'sort_by(Images, &CreationDate)'
-
AMI table
Here are the latest NixOS images available in the Amazon cloud.
-
-
-
+
+