fix: perpetual tag drift on aws resources

Panfactum · Mar 20, 2024 · c0516a1 · c0516a1
1 parent 4a8fa5f
commit c0516a1
Show file tree

Hide file tree

Showing 9 changed files with 53 additions and 33 deletions.
diff --git a/packages/nix/mkDevShells/common/setup/files/terragrunt/providers/aws.tftpl b/packages/nix/mkDevShells/common/setup/files/terragrunt/providers/aws.tftpl
@@ -9,8 +9,7 @@ provider "aws" {
   ignore_tags {
     key_prefixes = [
       "kubernetes.io",
-      "karpenter.sh",
-      "panfactum.com"
+      "karpenter.sh"
     ]
   }
 }
diff --git a/packages/nix/mkDevShells/common/setup/files/terragrunt/providers/aws_global.tftpl b/packages/nix/mkDevShells/common/setup/files/terragrunt/providers/aws_global.tftpl
@@ -10,8 +10,7 @@ provider "aws" {
   ignore_tags {
     key_prefixes = [
       "kubernetes.io",
-      "karpenter.sh",
-      "panfactum.com"
+      "karpenter.sh"
     ]
   }
 }
diff --git a/packages/nix/mkDevShells/common/setup/files/terragrunt/providers/aws_secondary.tftpl b/packages/nix/mkDevShells/common/setup/files/terragrunt/providers/aws_secondary.tftpl
@@ -10,8 +10,7 @@ provider "aws" {
   ignore_tags {
     key_prefixes = [
       "kubernetes.io",
-      "karpenter.sh",
-      "panfactum.com"
+      "karpenter.sh"
     ]
   }
 }
diff --git a/packages/reference/environments/providers/aws.tftpl b/packages/reference/environments/providers/aws.tftpl
@@ -7,6 +7,9 @@ provider "aws" {
   allowed_account_ids = ["${aws_account_id}"]
   profile             = "${aws_profile}"
   ignore_tags {
-    key_prefixes = ["kubernetes.io", "karpenter.sh"]
+    key_prefixes = [
+      "kubernetes.io",
+      "karpenter.sh"
+    ]
   }
 }
diff --git a/packages/reference/environments/providers/aws_global.tftpl b/packages/reference/environments/providers/aws_global.tftpl
@@ -8,6 +8,9 @@ provider "aws" {
   allowed_account_ids = ["${aws_account_id}"]
   profile             = "${aws_profile}"
   ignore_tags {
-    key_prefixes = ["kubernetes.io", "karpenter.sh"]
+    key_prefixes = [
+      "kubernetes.io",
+      "karpenter.sh"
+    ]
   }
 }
diff --git a/packages/reference/environments/providers/aws_secondary.tftpl b/packages/reference/environments/providers/aws_secondary.tftpl
@@ -8,6 +8,9 @@ provider "aws" {
   allowed_account_ids = ["${aws_account_id}"]
   profile             = "${aws_profile}"
   ignore_tags {
-    key_prefixes = ["kubernetes.io", "karpenter.sh"]
+    key_prefixes = [
+      "kubernetes.io",
+      "karpenter.sh"
+    ]
   }
 }
diff --git a/packages/reference/flake.lock b/packages/reference/flake.lock
diff --git a/packages/terraform/aws_eks/main.tf b/packages/terraform/aws_eks/main.tf
@@ -15,16 +15,7 @@ terraform {
 }
 
 locals {
-  vpc_id = values(data.aws_subnet.control_plane_subnets)[0].vpc_id // a bit hacky but we can just assume all subnets are in the same aws_vpc
-  common_tags = merge({
-    environment    = var.environment
-    pf_root_module = var.pf_root_module
-    region         = var.region
-    terraform      = "true"
-    },
-    {
-      "kubernetes.io/cluster/${var.cluster_name}" = "owned"
-  })
+  vpc_id                       = values(data.aws_subnet.control_plane_subnets)[0].vpc_id // a bit hacky but we can just assume all subnets are in the same aws_vpc
   controller_nodes_description = "Nodes for cluster-critical components and bootstrapping processes. Not autoscaled."
 }
 
@@ -128,15 +119,22 @@ resource "aws_security_group" "control_plane" {
   description = "Security group for the ${var.cluster_name} EKS control plane."
   vpc_id      = local.vpc_id
   tags = merge(module.tags.tags, {
-    Name                                        = var.cluster_name
-    "kubernetes.io/cluster/${var.cluster_name}" = "owned"
-    description                                 = "Security group for the ${var.cluster_name} EKS control plane."
+    Name        = var.cluster_name
+    description = "Security group for the ${var.cluster_name} EKS control plane."
   })
   lifecycle {
     prevent_destroy = true
   }
 }
 
+// This needs to be managed separately because they are included in the ignore_tags provider configuration
+resource "aws_ec2_tag" "control_plane_kubernetes" {
+  resource_id = aws_security_group.control_plane.id
+  key         = "kubernetes.io/cluster/${var.cluster_name}"
+  value       = "owned"
+}
+
+
 resource "aws_security_group_rule" "control_plane_nodes" {
   type                     = "ingress"
   description              = "Allow nodes to talk with API server."
@@ -288,14 +286,14 @@ resource "aws_launch_template" "controller" {
 
   tag_specifications {
     resource_type = "instance"
-    tags = merge(module.tags.tags, local.common_tags, {
+    tags = merge(module.tags.tags, {
       Name        = "${var.cluster_name}-controller"
       description = local.controller_nodes_description
       eks-managed = "true"
     })
   }
 
-  tags = merge(module.tags.tags, local.common_tags, {
+  tags = merge(module.tags.tags, {
     description = local.controller_nodes_description
   })
 
@@ -327,7 +325,7 @@ resource "aws_eks_node_group" "controllers" {
   }
 
   capacity_type = "ON_DEMAND"
-  tags = merge(module.tags.tags, local.common_tags, {
+  tags = merge(module.tags.tags, {
     description = local.controller_nodes_description
   })
   labels = {
@@ -362,17 +360,29 @@ resource "aws_security_group" "all_nodes" {
   vpc_id      = local.vpc_id
 
   tags = merge(module.tags.tags, {
-    "Name"                                      = "${var.cluster_name}-nodes"
-    "kubernetes.io/cluster/${var.cluster_name}" = "owned"
-    description                                 = "Security group for all nodes in the ${var.cluster_name} EKS cluster"
-    "karpenter.sh/discovery"                    = var.cluster_name
+    Name        = "${var.cluster_name}-nodes"
+    description = "Security group for all nodes in the ${var.cluster_name} EKS cluster"
   })
 
   lifecycle {
     prevent_destroy = true
   }
 }
 
+// These need to be managed separately because they are included in the ignore_tags provider configuration
+resource "aws_ec2_tag" "all_nodes_kubernetes" {
+  resource_id = aws_security_group.all_nodes.id
+  key         = "kubernetes.io/cluster/${var.cluster_name}"
+  value       = "owned"
+}
+
+resource "aws_ec2_tag" "all_nodes_karpenter" {
+  resource_id = aws_security_group.all_nodes.id
+  key         = "karpenter.sh/discovery"
+  value       = var.cluster_name
+}
+
+
 resource "aws_security_group_rule" "ingress_self" {
   security_group_id = aws_security_group.all_nodes.id
   type              = "ingress"

diff --git a/packages/terraform/aws_vpc/main.tf b/packages/terraform/aws_vpc/main.tf
@@ -69,6 +69,10 @@ resource "aws_subnet" "subnets" {
     Name                 = each.key
     "panfactum.com/type" = each.value.public ? "public" : contains(keys(var.nat_associations), each.key) ? "private" : "isolated"
   })
+
+  lifecycle {
+    ignore_changes = [tags["panfactum.com/public-ip"]]
+  }
 }
 
 ##########################################################################