From 80c8e2fdbfb6a4841779981a33c014932ea29dfe Mon Sep 17 00:00:00 2001 From: Arnaud Meukam Date: Wed, 9 Aug 2023 01:35:02 +0200 Subject: [PATCH] AWS: add EKS cluster for kOps CI Related to: - https://github.com/kubernetes/k8s.io/issues/5127 Add an EKS cluster that will be used as build cluster for kOps Signed-off-by: Arnaud Meukam --- infra/aws/terraform/kops-infra-ci/eks.tf | 190 ++++++++++++++++++ .../aws/terraform/kops-infra-ci/variables.tf | 12 +- infra/aws/terraform/kops-infra-ci/vpc.tf | 6 +- 3 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 infra/aws/terraform/kops-infra-ci/eks.tf diff --git a/infra/aws/terraform/kops-infra-ci/eks.tf b/infra/aws/terraform/kops-infra-ci/eks.tf new file mode 100644 index 00000000000..383f815e100 --- /dev/null +++ b/infra/aws/terraform/kops-infra-ci/eks.tf @@ -0,0 +1,190 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +module "eks" { + providers = { aws = aws.kops-infra-ci } + source = "terraform-aws-modules/eks/aws" + version = "19.16.0" + + cluster_name = "${local.prefix}-prow-build" + cluster_version = var.eks_version + cluster_endpoint_public_access = true + + cluster_ip_family = "ipv4" + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets + + cluster_enabled_log_types = [ + "audit", + "authenticator", + "api", + "controllerManager", + "scheduler" + ] + + cloudwatch_log_group_retention_in_days = 30 + + cluster_addons = { + coredns = { + most_recent = true + resolve_conflicts = "OVERWRITE" + } + kube-proxy = { + most_recent = true + } + vpc-cni = { + most_recent = true + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + } + aws-ebs-csi-driver = { + most_recent = true + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.ebs_csi_irsa.iam_role_arn + } + } + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] + + iam_role_attach_cni_policy = true + } + + eks_managed_node_groups = { + prow-build = { + name = "prow-build" + description = "EKS managed node group used to run kops jobs" + use_name_prefix = true + + dataplane_wait_duration = "600s" + + subnet_ids = module.vpc.private_subnets + + min_size = 3 + max_size = 100 + desired_size = 3 + + # Force version update if existing pods are unable to be drained due to a PodDisruptionBudget issue + force_update_version = true + update_config = { + max_unavailable = 1 + } + + capacity_type = "ON_DEMAND" + instance_types = ["r6i.2xlarge"] + ami_type = "BOTTLEROCKET_x86_64" + platform = "bottlerocket" + + ebs_optimized = true + enable_monitoring = true + + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 100 + volume_type = "gp3" + iops = "3000" #https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html + encrypted = false + delete_on_termination = true + } + } + } + + metadata_options = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "enabled" + } + + iam_role_additional_policies = { + AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + tags = merge(var.tags, { + "region" = "${data.aws_region.current.name}" + }) + } + } + + tags = merge(var.tags, { + "region" = "${data.aws_region.current.name}" + }) +} + + +module "vpc_cni_irsa" { + providers = { aws = aws.kops-infra-ci } + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name_prefix = "VPC-CNI-IRSA" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + # We use IPv4-based EKS cluster, so we don't need this + vpc_cni_enable_ipv6 = false + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } + + tags = var.tags +} + +module "ebs_csi_irsa" { + providers = { aws = aws.kops-infra-ci } + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name_prefix = "EBS-CSI-IRSA" + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } + + tags = var.tags +} + +module "cluster_autoscaler_irsa_role" { + providers = { aws = aws.kops-infra-ci } + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name = "cluster-autoscaler" + attach_cluster_autoscaler_policy = true + cluster_autoscaler_cluster_ids = [module.eks.cluster_name] + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:cluster-autoscaler"] + } + } + + tags = var.tags +} diff --git a/infra/aws/terraform/kops-infra-ci/variables.tf b/infra/aws/terraform/kops-infra-ci/variables.tf index 9f446e8861b..924325a666c 100644 --- a/infra/aws/terraform/kops-infra-ci/variables.tf +++ b/infra/aws/terraform/kops-infra-ci/variables.tf @@ -14,17 +14,23 @@ See the License for the specific language governing permissions and limitations under the License. */ +variable "eks_version" { + type = string + default = "1.26" +} + variable "tags" { type = map(string) default = { - "managed-by" = "Terraform", - "group" = "sig-cluster-lifecycle", + "managed-by" = "Terraform", + "group" = "sig-cluster-lifecycle", "subproject" = "kops" + "githubRepo" = "git.k8s.io/k8s.io" } } variable "region" { - type = string + type = string default = "us-east-2" } diff --git a/infra/aws/terraform/kops-infra-ci/vpc.tf b/infra/aws/terraform/kops-infra-ci/vpc.tf index 1c463aed986..3c12d9344f5 100644 --- a/infra/aws/terraform/kops-infra-ci/vpc.tf +++ b/infra/aws/terraform/kops-infra-ci/vpc.tf @@ -48,14 +48,14 @@ resource "aws_vpc_ipam_pool" "main" { resource "aws_vpc_ipam_pool_cidr" "main" { - provider = aws.kops-infra-ci + provider = aws.kops-infra-ci ipam_pool_id = aws_vpc_ipam_pool.main.id cidr = var.vpc_cidr } resource "aws_vpc_ipam_preview_next_cidr" "main" { - provider = aws.kops-infra-ci - ipam_pool_id = aws_vpc_ipam_pool.main.id + provider = aws.kops-infra-ci + ipam_pool_id = aws_vpc_ipam_pool.main.id netmask_length = 20 // a 18 netmask length is considered as too big for the CIDR pool }