diff --git a/.github/workflows/terraform-deploy-aws-curvenote.yml b/.github/workflows/terraform-deploy-aws-curvenote.yml new file mode 100644 index 000000000..768cba1d9 --- /dev/null +++ b/.github/workflows/terraform-deploy-aws-curvenote.yml @@ -0,0 +1,142 @@ +# See terraform/aws/curvenote/README.md +name: Terraform aws-curvenote + +on: + push: + branches: + - main + paths: + - "terraform/aws/curvenote/**" + - .github/workflows/terraform-deploy-aws-curvenote.yml + workflow_dispatch: + +# Only allow one workflow to run at a time +concurrency: terraform-deploy-aws-curvenote + +env: + TFPLAN: aws-curvenote.tfplan + AWS_DEPLOYMENT_ROLE: arn:aws:iam::166088433508:role/binderhub-github-oidc-mybinderorgdeploy-terraform + AWS_REGION: us-east-2 + WORKDIR: ./terraform/aws/curvenote + +jobs: + terraform-plan: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + defaults: + run: + working-directory: ${{ env.WORKDIR }} + outputs: + apply: ${{ steps.terraform-plan.outputs.apply }} + + steps: + - uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.AWS_DEPLOYMENT_ROLE }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: terraform-plan + + # Capture the console output of terraform plan to a file, so we can include + # it as a job summary in the Actions view: + # https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/ + - name: Terraform plan + id: terraform-plan + run: | + terraform init + terraform plan -out="${TFPLAN}" -detailed-exitcode -no-color | tee tfplan.stdout + # Get the exit code of the terraform plan command, not the tee command. + EXIT_CODE="${PIPESTATUS[0]}" + if [ $EXIT_CODE -eq 0 ]; then + echo "No changes" + echo "apply=false" >> "$GITHUB_OUTPUT" + elif [ $EXIT_CODE -eq 2 ]; then + echo "Changes found" + echo "apply=true" >> "$GITHUB_OUTPUT" + else + echo "Terraform plan failed" + exit $EXIT_CODE + fi + + # Skip the first bit of the terraform plan stdout + # https://unix.stackexchange.com/a/205680 + - name: Set job summary + if: steps.terraform-plan.outputs.apply == 'true' + run: | + echo '### Terraform Plan summary' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + sed -n '/Terraform will perform the following/,$p' tfplan.stdout >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: Install age + if: steps.terraform-plan.outputs.apply == 'true' + run: | + sudo apt-get update -y -q + sudo apt-get install -y -q age + + - name: Encrypt plan + if: steps.terraform-plan.outputs.apply == 'true' + run: | + echo ${{ secrets.TFPLAN_ARTIFACT_SECRET_KEY }} > tfplan.key + age --identity tfplan.key --encrypt --output "${TFPLAN}.enc" "${TFPLAN}" + + - name: Upload plan + if: steps.terraform-plan.outputs.apply == 'true' + uses: actions/upload-artifact@v3 + with: + name: ${{ env.TFPLAN }} + path: ${{ env.WORKDIR }}/${{ env.TFPLAN }}.enc + if-no-files-found: error + + terraform-apply: + needs: + - terraform-plan + runs-on: ubuntu-22.04 + timeout-minutes: 60 + # This environment requires approval before the deploy is run. + environment: aws-curvenote + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + defaults: + run: + working-directory: ${{ env.WORKDIR }} + if: needs.terraform-plan.outputs.apply == 'true' + + steps: + - uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.AWS_DEPLOYMENT_ROLE }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: terraform-apply + + - name: Download plan + uses: actions/download-artifact@v3 + with: + name: ${{ env.TFPLAN }} + path: ${{ env.WORKDIR }} + + - name: Install age + run: | + sudo apt-get update -y -q + sudo apt-get install -y -q age + + - name: Decrypt plan + run: | + echo ${{ secrets.TFPLAN_ARTIFACT_SECRET_KEY }} > tfplan.key + age --identity tfplan.key --decrypt --output "${TFPLAN}" "${TFPLAN}.enc" + + - name: Terraform apply + run: | + terraform init + terraform apply "${TFPLAN}" diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 000000000..f94f843d8 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,33 @@ +name: Terraform static checks + +on: + pull_request: + paths: + - "terraform/**" + push: + paths: + - "terraform/**" + workflow_dispatch: + +# We can't run CI tests on Terraform, so use as many static linters as possible + +jobs: + terraform-pre-commit: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: ".python-version" + + - name: Install dependencies + run: pip install pre-commit + + # https://github.com/terraform-linters/setup-tflint + - name: Install tflint + uses: terraform-linters/setup-tflint@v3.0.0 + with: + tflint_version: v0.47.0 + + - name: Run terraform pre-commit + run: pre-commit run --all --config .pre-commit-config-terraform.yaml diff --git a/.gitignore b/.gitignore index 340b738a8..6f108494c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ env .terraform .terraform.lock.hcl +# Keep .terraform.lock.hcl to ensure reproducible deployments +!terraform/aws/curvenote/.terraform.lock.hcl diff --git a/.pre-commit-config-terraform.yaml b/.pre-commit-config-terraform.yaml new file mode 100644 index 000000000..0d18cee2c --- /dev/null +++ b/.pre-commit-config-terraform.yaml @@ -0,0 +1,22 @@ +# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level +# +# Common tasks +# +# - Run on all files: pre-commit run --all --config .pre-commit-config-terraform.yaml +# +# Prerequisites: +# - terraform +# - tflint + +# Currently only aws/ is checked +files: "^terraform/aws/" +exclude: "^terraform/aws/pangeo/" + +repos: + # We can't run any CI tests on production Terraform code, so use as many static linters as possible + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.83.0 + hooks: + - id: terraform_fmt + - id: terraform_tflint + - id: terraform_validate diff --git a/terraform/aws/binder-eks/README.md b/terraform/aws/binder-eks/README.md new file mode 100644 index 000000000..e73b381ca --- /dev/null +++ b/terraform/aws/binder-eks/README.md @@ -0,0 +1,7 @@ +# BinderHub on AWS EKS + +This module deploys an AWS EKS cluster with IRSA roles to support BinderHub ECR access. + +The module has optional support for using a limited non-administrative AWS role with a permissions boundary to deploy the cluster. + +For an example see [curvenote](../curvenote/README.md) diff --git a/terraform/aws/binder-eks/eks-cluster.tf b/terraform/aws/binder-eks/eks-cluster.tf new file mode 100644 index 000000000..1f3d5fb1f --- /dev/null +++ b/terraform/aws/binder-eks/eks-cluster.tf @@ -0,0 +1,111 @@ +# https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/19.15.2 +# Full example: +# https://github.com/terraform-aws-modules/terraform-aws-eks/blame/v19.14.0/examples/complete/main.tf +# https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v19.14.0/docs/compute_resources.md + +data "aws_caller_identity" "current" {} + +locals { + permissions_boundary_arn = ( + var.permissions_boundary_name != null ? + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_name}" : + null + ) +} + +# This assumes the EKS service linked role is already created (or the current user has permissions to create it) +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.15.3" + cluster_name = var.cluster_name + cluster_version = var.k8s_version + subnet_ids = module.vpc.public_subnets + + cluster_endpoint_private_access = true + cluster_endpoint_public_access = true + cluster_endpoint_public_access_cidrs = var.k8s_api_cidrs + + vpc_id = module.vpc.vpc_id + + # Allow all allowed roles to access the KMS key + kms_key_enable_default_policy = true + # This duplicates the above, but the default is the current user/role so this will avoid + # a deployment change when run by different users/roles + kms_key_administrators = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root", + ] + + enable_irsa = var.enable_irsa + iam_role_permissions_boundary = local.permissions_boundary_arn + + eks_managed_node_group_defaults = { + capacity_type = "SPOT" + iam_role_permissions_boundary = local.permissions_boundary_arn + } + + eks_managed_node_groups = { + worker_group_1 = { + name = "${var.cluster_name}-wg1" + instance_types = [var.instance_type_wg1] + ami_type = var.use_bottlerocket ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" + platform = var.use_bottlerocket ? "bottlerocket" : "linux" + + # additional_userdata = "echo foo bar" + vpc_security_group_ids = [ + aws_security_group.all_worker_mgmt.id, + aws_security_group.worker_group_all.id, + ] + desired_size = var.wg1_size + min_size = 1 + max_size = var.wg1_max_size + + # Disk space can't be set with the default custom launch template + # disk_size = 100 + block_device_mappings = [ + { + # https://github.com/bottlerocket-os/bottlerocket/discussions/2011 + device_name = var.use_bottlerocket ? "/dev/xvdb" : "/dev/xvda" + ebs = { + # Uses default alias/aws/ebs key + encrypted = true + volume_size = var.root_volume_size + volume_type = "gp3" + } + } + ] + + subnet_ids = slice(module.vpc.public_subnets, 0, var.number_azs) + }, + # Add more worker groups here + } + + manage_aws_auth_configmap = true + # Anyone in the AWS account with sufficient permissions can access the cluster + aws_auth_accounts = [ + data.aws_caller_identity.current.account_id, + ] + aws_auth_roles = [ + { + # GitHub OIDC role + rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-${var.github_oidc_role_suffix}" + username = "binderhub-github-oidc" + groups = ["system:masters"] + }, + { + # GitHub OIDC terraform role + rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-${var.github_oidc_role_suffix}-terraform" + username = "binderhub-github-oidc" + groups = ["system:masters"] + }, + { + # BinderHub admins role + rolearn = aws_iam_role.eks_access.arn + username = "binderhub-admin" + groups = ["system:masters"] + } + ] +} + +data "aws_eks_cluster_auth" "binderhub" { + name = var.cluster_name +} diff --git a/terraform/aws/binder-eks/k8s-access.tf b/terraform/aws/binder-eks/k8s-access.tf new file mode 100644 index 000000000..7a0281b14 --- /dev/null +++ b/terraform/aws/binder-eks/k8s-access.tf @@ -0,0 +1,116 @@ +# Roles to allow access to EKS + +# Allow GitHub workflows to access AWS using OIDC (no hardcoded credentials) +# https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services + +locals { + create_github_roles = (var.enable_irsa || var.oidc_created) ? 1 : 0 +} + +data "aws_iam_openid_connect_provider" "github_oidc_provider" { + count = local.create_github_roles + url = "https://token.actions.githubusercontent.com" +} + +resource "aws_iam_policy" "eks_access" { + name = "${var.cluster_name}-eks-access" + description = "Kubernetes EKS access to ${var.cluster_name}" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = ["eks:DescribeCluster"] + Effect = "Allow" + Resource = module.eks.cluster_arn + } + ] + }) +} + +resource "aws_iam_role" "github_oidc_mybinderorgdeploy" { + count = local.create_github_roles + name = "${var.cluster_name}-${var.github_oidc_role_suffix}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRoleWithWebIdentity" + Effect = "Allow" + Principal = { + Federated = data.aws_iam_openid_connect_provider.github_oidc_provider[0].arn + } + Condition = { + StringLike = { + "token.actions.githubusercontent.com:sub" = [ + # GitHub repositories and refs allowed to use this role + "repo:jupyterhub/mybinder.org-deploy:ref:refs/heads/main", + ] + } + } + } + ] + }) + inline_policy {} + managed_policy_arns = [ + aws_iam_policy.eks_access.arn, + ] + permissions_boundary = local.permissions_boundary_arn +} + +resource "aws_iam_role" "github_oidc_mybinderorgdeploy_terraform" { + count = local.create_github_roles + name = "${var.cluster_name}-${var.github_oidc_role_suffix}-terraform" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRoleWithWebIdentity" + Effect = "Allow" + Principal = { + Federated = data.aws_iam_openid_connect_provider.github_oidc_provider[0].arn + } + Condition = { + StringLike = { + "token.actions.githubusercontent.com:sub" = [ + # GitHub repositories and refs allowed to use this role + "repo:jupyterhub/mybinder.org-deploy:ref:refs/heads/main", + # Can't have branch and environment in the same condition + # https://github.com/aws-actions/configure-aws-credentials/issues/746 + "repo:jupyterhub/mybinder.org-deploy:environment:aws-curvenote", + ] + } + } + } + ] + }) + inline_policy {} + managed_policy_arns = [ + local.permissions_boundary_arn, + ] + permissions_boundary = local.permissions_boundary_arn +} + +# IAM role that can be assumed by anyone in the AWS account (assuming they have sufficient permissions) +resource "aws_iam_role" "eks_access" { + name = "${var.cluster_name}-eks-access" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" + } + } + ] + }) + inline_policy {} + managed_policy_arns = [ + aws_iam_policy.eks_access.arn, + ] + permissions_boundary = local.permissions_boundary_arn +} diff --git a/terraform/aws/binder-eks/openid_connect_providers.tf.example b/terraform/aws/binder-eks/openid_connect_providers.tf.example new file mode 100644 index 000000000..a4dcc8fd5 --- /dev/null +++ b/terraform/aws/binder-eks/openid_connect_providers.tf.example @@ -0,0 +1,33 @@ +resource "aws_iam_openid_connect_provider" "binderhub_eks_oidc_provider" { + client_id_list = [ + "sts.amazonaws.com", + ] + tags = { + "Name" = "binderhub-eks-irsa" + } + thumbprint_list = [ + # Use output from cluster_tls_certificate_sha1_fingerprint + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + ] + # Use output from cluster_oidc_issuer_url + url = "https://oidc.eks.us-east-2.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +} + +# https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services +# Use in conjunction with a role, and +# https://github.com/aws-actions/configure-aws-credentials +resource "aws_iam_openid_connect_provider" "github_oidc" { + client_id_list = [ + "sts.amazonaws.com", + ] + tags = { + "Name" = "github-oidc" + } + thumbprint_list = [ + "6938fd4d98bab03faadb97b34396831e3780aea1" + ] + url = "https://token.actions.githubusercontent.com" +} diff --git a/terraform/aws/binder-eks/outputs.tf b/terraform/aws/binder-eks/outputs.tf new file mode 100644 index 000000000..7e422776c --- /dev/null +++ b/terraform/aws/binder-eks/outputs.tf @@ -0,0 +1,32 @@ +output "cluster_name" { + description = "Kubernetes Cluster Name" + value = var.cluster_name +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = module.eks.cluster_oidc_issuer_url +} + +data "tls_certificate" "cluster_tls_certificate" { + url = module.eks.cluster_oidc_issuer_url +} + +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = data.tls_certificate.cluster_tls_certificate.certificates[*].sha1_fingerprint +} + +output "cluster_endpoint" { + description = "The endpoint for the EKS Kubernetes API" + value = module.eks.cluster_endpoint +} + +output "cluster_ca_certificate" { + description = "The EKS certificate authority data" + value = base64decode(module.eks.cluster_certificate_authority_data) +} +output "eks_token" { + description = "The EKS token" + value = data.aws_eks_cluster_auth.binderhub.token +} diff --git a/terraform/aws/binder-eks/provider.tf b/terraform/aws/binder-eks/provider.tf new file mode 100644 index 000000000..b71e2c168 --- /dev/null +++ b/terraform/aws/binder-eks/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.5" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.21" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + } + + required_version = ">= 1.4.6" +} diff --git a/terraform/aws/binder-eks/security-groups.tf b/terraform/aws/binder-eks/security-groups.tf new file mode 100644 index 000000000..f1dc29833 --- /dev/null +++ b/terraform/aws/binder-eks/security-groups.tf @@ -0,0 +1,43 @@ +# https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v18.26.6/docs/network_connectivity.md + +resource "aws_security_group" "worker_group_all" { + name_prefix = "worker_group_all_ports" + vpc_id = module.vpc.vpc_id + description = "Allow all ports for worker group" + + ingress { + description = "Allow all inbound traffic" + protocol = "-1" + from_port = 0 + to_port = 0 + self = true + } + egress { + description = "Allow all outbound traffic" + protocol = "-1" + from_port = 0 + to_port = 0 + # self = true + cidr_blocks = ["0.0.0.0/0"] + } + +} + +resource "aws_security_group" "all_worker_mgmt" { + name_prefix = "all_worker_management" + vpc_id = module.vpc.vpc_id + description = "Worker nodes internal access" + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + + cidr_blocks = [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ] + } +} diff --git a/terraform/aws/binder-eks/serviceaccounts.tf b/terraform/aws/binder-eks/serviceaccounts.tf new file mode 100644 index 000000000..7029cec9a --- /dev/null +++ b/terraform/aws/binder-eks/serviceaccounts.tf @@ -0,0 +1,161 @@ +# IAM roles for Kubernetes Service Accounts + +# https://registry.terraform.io/modules/terraform-aws-modules/iam/aws/latest/submodules/iam-role-for-service-accounts-eks +# https://github.com/terraform-aws-modules/terraform-aws-iam/tree/v5.2.0/modules/iam-role-for-service-accounts-eks + +locals { + create_irsa_roles = (var.enable_irsa || var.oidc_created) ? 1 : 0 + eks_oidc_provider_arn = (local.create_irsa_roles == 1) ? data.aws_iam_openid_connect_provider.binderhub_eks_oidc_provider[0].arn : null +} + +data "aws_iam_openid_connect_provider" "binderhub_eks_oidc_provider" { + count = local.create_irsa_roles + url = module.eks.cluster_oidc_issuer_url +} + +module "irsa_eks_role_load_balancer" { + count = local.create_irsa_roles + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.19.0" + role_name = "${var.cluster_name}-IRSA-aws-load-balancer-controller" + attach_load_balancer_controller_policy = true + role_permissions_boundary_arn = local.permissions_boundary_arn + policy_name_prefix = "${var.cluster_name}-AmazonEKS_" + + oidc_providers = { + default = { + provider_arn = local.eks_oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +module "irsa_eks_role_ebs_csi" { + count = local.create_irsa_roles + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.19.0" + role_name = "${var.cluster_name}-IRSA-ebs-csi-controller-sa" + attach_ebs_csi_policy = true + role_permissions_boundary_arn = local.permissions_boundary_arn + policy_name_prefix = "${var.cluster_name}-AmazonEKS_" + + oidc_providers = { + default = { + provider_arn = local.eks_oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } +} + +# BinderHub ECR IAM role +# https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonelasticcontainerregistry.html +resource "aws_iam_policy" "binderhub-ecr" { + name = "${var.cluster_name}-ecr-policy" + path = "/" + description = "BinderHub ECR create/read/write" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ecr:CreateRepository", + "ecr:DescribeImages", + "ecr:DescribeRepositories", + + "ecr:ListImages", + + "ecr:BatchCheckLayerAvailability", + "ecr:CompleteLayerUpload", + "ecr:GetAuthorizationToken", + "ecr:InitiateLayerUpload", + "ecr:PutImage", + "ecr:UploadLayerPart", + + "ecr:BatchDeleteImage", + "ecr:DeleteLifecyclePolicy", + "ecr:DeleteRepository", + "ecr:GetLifecyclePolicy", + "ecr:PutLifecyclePolicy", + ] + Effect = "Allow" + Resource = "arn:aws:ecr:${var.region}:${data.aws_caller_identity.current.account_id}:*" + }, + { + Action = [ + "ecr:GetAuthorizationToken", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) +} + +module "irsa_eks_role_binderhub_ecr" { + count = local.create_irsa_roles + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.19.0" + role_name = "${var.cluster_name}-IRSA-aws-binderhub-ecr" + role_permissions_boundary_arn = local.permissions_boundary_arn + policy_name_prefix = "${var.cluster_name}-AmazonEKS_" + + oidc_providers = { + default = { + provider_arn = local.eks_oidc_provider_arn + namespace_service_accounts = ["curvenote:binderhub-container-registry-helper"] + } + } + + role_policy_arns = { + binderhub-ecr = aws_iam_policy.binderhub-ecr.arn + } +} + +# BinderHub ECR registry cleaner role +resource "aws_iam_policy" "binderhub-ecr-registry-cleaner" { + name = "${var.cluster_name}-ecr-cleaner-policy" + path = "/" + description = "BinderHub ECR list/delete" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ecr:DescribeImages", + "ecr:DescribeRepositories", + + "ecr:ListImages", + + "ecr:BatchDeleteImage", + "ecr:DeleteLifecyclePolicy", + "ecr:DeleteRepository", + "ecr:GetLifecyclePolicy", + ] + Effect = "Allow" + Resource = "arn:aws:ecr:${var.region}:${data.aws_caller_identity.current.account_id}:*" + }, + ] + }) +} + +module "irsa_eks_role_binderhub_ecr_registry_cleaner" { + count = local.create_irsa_roles + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.19.0" + role_name = "${var.cluster_name}-IRSA-aws-binderhub-ecr-registry-cleaner" + role_permissions_boundary_arn = local.permissions_boundary_arn + policy_name_prefix = "${var.cluster_name}-AmazonEKS_" + + oidc_providers = { + default = { + provider_arn = local.eks_oidc_provider_arn + namespace_service_accounts = ["curvenote:binderhub-ecr-registry-cleaner"] + } + } + + role_policy_arns = { + binderhub-ecr = aws_iam_policy.binderhub-ecr-registry-cleaner.arn + } +} diff --git a/terraform/aws/binder-eks/variables.tf b/terraform/aws/binder-eks/variables.tf new file mode 100644 index 000000000..9b2faf219 --- /dev/null +++ b/terraform/aws/binder-eks/variables.tf @@ -0,0 +1,105 @@ +variable "region" { + type = string + description = "AWS region" +} + +variable "cluster_name" { + type = string + description = "EKS cluster name" + default = "binderhub" +} + +variable "k8s_version" { + type = string + description = "Kubernetes cluster version" + default = "1.26" +} + +variable "k8s_api_cidrs" { + type = list(string) + default = ["0.0.0.0/0"] + description = "CIDRs that have access to the K8s API" +} + +variable "number_azs" { + type = number + # Use just one so we don't have to deal with node/volume affinity- + # can't use EBS volumes across AZs + default = 1 + description = "Number of AZs to use" +} + +variable "instance_type_wg1" { + type = string + default = "r6a.2xlarge" + description = "Worker-group-1 EC2 instance type" +} + +variable "use_bottlerocket" { + type = bool + default = false + description = "Use Bottlerocket for worker nodes" +} + +variable "root_volume_size" { + type = number + default = 100 + description = "Root volume size in GB" +} + +variable "wg1_size" { + type = number + default = 2 + description = <<-EOT + Worker-group-1 initial desired number of nodes. + Note this has no effect after the cluster is provisioned: + - https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2030 + - https://github.com/bryantbiggs/eks-desired-size-hack + Manually change the node group size in the AWS console instead. + EOT +} + +variable "wg1_max_size" { + type = number + default = 2 + description = "Worker-group-1 maximum number of nodes" +} + + +# The following configuration is needed if you are using a limited IAM role for deployment + +variable "enable_irsa" { + type = bool + default = true + description = <<-EOT + Disable if OIDC needs to be setup manually due to limited permissions. + If you have full admin access, you can set this to true. + EOT +} + +variable "oidc_created" { + type = bool + default = false + description = <<-EOT + If enable_irsa is false and the OIDC provider has been manually created using + the openid_connect_providers.tf.example file, set this to true. + EOT +} + +variable "github_oidc_role_suffix" { + type = string + description = <<-EOT + The suffix of the IAM role that will be created for the GitHub OIDC provider. + Will be joined to var.cluster_name with a hyphen. + EOT + default = "github-oidc-mybinderorgdeploy" +} + +variable "permissions_boundary_name" { + type = string + description = <<-EOT + The name of the permissions boundary to attach to all IAM roles. + Specify if you are using a limited IAM role for deployment. + EOT + default = null +} diff --git a/terraform/aws/binder-eks/vpc.tf b/terraform/aws/binder-eks/vpc.tf new file mode 100644 index 000000000..4c78dbd10 --- /dev/null +++ b/terraform/aws/binder-eks/vpc.tf @@ -0,0 +1,29 @@ +# data "aws_availability_zones" "available" {} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.0.0" + + name = var.cluster_name + cidr = "10.0.0.0/16" + # EKS requires at least two AZ (though node groups can be placed in just one) + azs = ["${var.region}a", "${var.region}b"] + public_subnets = ["10.0.1.0/24", "10.0.2.0/24"] + private_subnets = [] + enable_nat_gateway = false + single_nat_gateway = true + enable_dns_hostnames = true + map_public_ip_on_launch = true + + tags = { + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + } + + # https://repost.aws/knowledge-center/eks-load-balancer-controller-subnets + public_subnet_tags = { + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = {} +} diff --git a/terraform/aws/curvenote/.terraform.lock.hcl b/terraform/aws/curvenote/.terraform.lock.hcl new file mode 100644 index 000000000..f071bf5d6 --- /dev/null +++ b/terraform/aws/curvenote/.terraform.lock.hcl @@ -0,0 +1,105 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.5.0" + constraints = ">= 3.72.0, >= 4.0.0, >= 4.47.0, >= 4.57.0, >= 5.0.0, ~> 5.5" + hashes = [ + "h1:WOweXv4ZjePZwdxuzE2UmRWOPhhcQDNxGu2wOcpHFWY=", + "zh:10fe0ef4191323c920c1844f27dbc88114547d5f78fad915c1769c908f40d916", + "zh:565fc7c3a1f42474fa75f143cb8115e11b894ed7fd9973569b00bd429fb92b4e", + "zh:5ba6132b1d442ed679ad8ea89fb5602aa0893e8dcd002a52ab3d76591aa18c8b", + "zh:5c2580630cd5034bae800445074c17950aea17f089bcdae7af637173122f8b03", + "zh:656d77220c6053fd5adb86d3bfb57dd42f98220d81590ffd643156ffeca36608", + "zh:65c7b3e333b734ce641735a23539d4fb392a675a5a9b892e8369781b1f3386a2", + "zh:682d55b2e6e9c40e20d679aa53d561797b1f3450e5187c9f4e8c359b69f06df3", + "zh:79ebc0993d6128819d70dd896cd743e3bab3e3cdc4c02f2a2dbd138471c23179", + "zh:8d44214c738f0410f829e1c761b021c92b3364daf9fcd08097216cc84eaff997", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a0b1bc008e95c5a7285f5e7dd116ce60ba7a6c1c3bd8ac3e3b63d4e1438d8e49", + "zh:cf40fb60efc5df42fc5716c7e458868251c82fc78b623f12d1bc994b6fcc7ef2", + "zh:cfd8f3f391cddecfc5e44fe57f0633067470e9038517115ba69d8ee533d5d74e", + "zh:d6552490599e02a756e72b7091b591493cee25548ce7120ad05210b4ff2492bd", + "zh:f77dfe665fd4b3d9e36fdc989d7feff4cf6bf17161c0b1a0f25a0fcf402c779d", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.2" + constraints = ">= 2.0.0" + hashes = [ + "h1:Vl0aixAYTV/bjathX7VArC5TVNkxBCsi3Vq7R4z1uvc=", + "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", + "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", + "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", + "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f", + "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422", + "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136", + "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7", + "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc", + "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040", + "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.21.1" + constraints = ">= 2.10.0, ~> 2.21" + hashes = [ + "h1:gP8IU3gFfXYRfGZr5Qws9JryZsOGsluAVpiAoZW7eo0=", + "zh:156a437d7edd6813e9cb7bdff16ebce28cec08b07ba1b0f5e9cec029a217bc27", + "zh:1a21c255d8099e303560e252579c54e99b5f24f2efde772c7e39502c62472605", + "zh:27b2021f86e5eaf6b9ee7c77d7a9e32bc496e59dd0808fb15a5687879736acf6", + "zh:31fa284c1c873a85c3b5cfc26cf7e7214d27b3b8ba7ea5134ab7d53800894c42", + "zh:4be9cc1654e994229c0d598f4e07487fc8b513337de9719d79b45ce07fc4e123", + "zh:5f684ed161f54213a1414ac71b3971a527c3a6bfbaaf687a7c8cc39dcd68c512", + "zh:6d58f1832665c256afb68110c99c8112926406ae0b64dd5f250c2954fc26928e", + "zh:9dadfa4a019d1e90decb1fab14278ee2dbefd42e8f58fe7fa567a9bf51b01e0e", + "zh:a68ce7208a1ef4502528efb8ce9f774db56c421dcaccd3eb10ae68f1324a6963", + "zh:acdd5b45a7e80bc9d254ad0c2f9cb4715104117425f0d22409685909a790a6dd", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb451e882118fe92e1cb2e60ac2d77592f5f7282b3608b878b5bdc38bbe4fd5b", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.9.1" + constraints = ">= 0.9.0" + hashes = [ + "h1:NUv/YtEytDQncBQ2mTxnUZEy/rmDlPYmE9h2iokR0vk=", + "zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f", + "zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5", + "zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8c8094689a2bed4bb597d24a418bbbf846e15507f08be447d0a5acea67c2265a", + "zh:a6d9206e95d5681229429b406bc7a9ba4b2d9b67470bda7df88fa161508ace57", + "zh:aa299ec058f23ebe68976c7581017de50da6204883950de228ed9246f309e7f1", + "zh:b129f00f45fba1991db0aa954a6ba48d90f64a738629119bfb8e9a844b66e80b", + "zh:ef6cecf5f50cda971c1b215847938ced4cb4a30a18095509c068643b14030b00", + "zh:f1f46a4f6c65886d2dd27b66d92632232adc64f92145bf8403fe64d5ffa5caea", + "zh:f79d6155cda7d559c60d74883a24879a01c4d5f6fd7e8d1e3250f3cd215fb904", + "zh:fd59fa73074805c3575f08cd627eef7acda14ab6dac2c135a66e7a38d262201c", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + constraints = ">= 3.0.0" + hashes = [ + "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/aws/curvenote/README.md b/terraform/aws/curvenote/README.md new file mode 100644 index 000000000..c0edd26ec --- /dev/null +++ b/terraform/aws/curvenote/README.md @@ -0,0 +1,148 @@ +# AWS infrastructure on CurveNote + +This deployment is run in an [AWS account owned by Curvenote](https://github.com/jupyterhub/mybinder.org-deploy/issues/2629), with restricted user accounts provided to the mybinder team. + +## AWS account prerequisites + +IAM roles, users, and some other base infrastructure is defined in a separate private repository under the control of the AWS account administrator. +Contact the mybinder team and @stevejpurves to obtain access. + +## Manual bootstrapping (new deployment only) + +The Terraform state file is stored in a remote S3 bucket which must be created before the first deployment, along with a DynamoDB lock. +This should only be run once! + +``` +cd bootstrap +terraform init +terraform apply +cd .. +``` + +If you want to use this to create multiple deployments you **must** modify + +`terraform { backend "s3" { ... } }` + +to use a different key and/or bucket. +Failure to do this will result in the original deployment becoming unmanageable- it will not be possible to modify or even delete it with Terraform! + +## Manual deployment + +Ensure you have a recent version of Terraform. +The minimum required version is specified by `terraform { required_version } }` in [`provider.tf`](provider.tf). + +The full deployment requires an OIDC Provider that must be created by a privileged AWS administrator. + +Deploy the Kubernetes cluster without the OIDC Provider by setting the following [variables](variables.tf): + +``` +enable_irsa = false +``` + +``` +terraform init +terraform apply +``` + +Copy [`openid_connect_providers.tf.example`](../binder-eks/openid_connect_providers.tf.example) and edit the `resource "aws_iam_openid_connect_provider" "binderhub_eks_oidc_provider" { thumbprint_list, url }` fields, using the values from [`outputs.tf`](../binder-eks/outputs.tf). + +Ask the AWS account administrator to create the OIDC Provider, and retrieve the OIDC binderhub provider ARN. + +Set + +``` +oidc_created = true +``` + +and deploy again + +``` +terraform apply +``` + +## Obtaining a kubeconfig file + +You must have the AWS CLI (v2 is recommended) and kubectl in your PATH. + +You must assume the `arn:aws:iam:::role/binderhub-eks-access` role to obtain a kubeconfig file. +Assuming your AWS credentials are already setup, for example in a profile called `aws-curvenote`, the easiest way to do this is to [add a second profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html) in `~/.aws/config`: + +``` +[profile aws-curvenote-binderhub-eks-access] +region = us-east-2 +role_arn = arn:aws:iam:::role/binderhub-eks-access +source_profile = +``` + +Obtain a kubeconfig file + +``` +aws --profile=aws-curvenote-binderhub-eks-access eks update-kubeconfig --name binderhub --kubeconfig /path/to/kubeconfig + +kubectl --kubeconfig=/path/to/kubeconfig get nodes +``` + +Note: The AWS user who deployed the cluster [automatically has admin access to the cluster](https://docs.aws.amazon.com/eks/latest/userguide/add-user-role.html)- this is not configurable. +To minimise confusion always assume the `binderhub-eks-access` IAM role to create the kubeconfig. + +## Automatic deployment (GitHub workflows) + +This infrastructure deployment can be partially managed by the +[`terraform-deploy-aws-curvenote.yml` GitHub workflow](../../../.github/workflows/terraform-deploy-aws-curvenote.yml) that runs in two stages: + +1. The first job runs `terraform plan`. + If there are infrastructure changes the diff is shown in the job output and an encrypted Terraform plan artifact is uploaded. +2. The second job is linked to the environment `aws-curvenote`, and requires a + [manual approval step](https://docs.github.com/en/actions/managing-workflow-runs/reviewing-deployments). + Check the planned changes from the first job, before either approving or rejecting the job. + If approved `terraform apply` will be run. + +Since infrastructure pull requested aren't tested this allows the actual production changes to be reviewed before deployment. + +The role assumed by GitHub for this deployment does not have sufficient rights to deploy all changes, most notably it cannot create new IAM roles. +It should be sufficient for most routine changes such as Kubernetes upgrades or changes to the cluster size. +For more complex changes the deployment should be run manually first. + +### Example approval + +If `terraform-plan` found changes a request for approval will be sent: +![Request for a Terraform deployment to be approved](./docs/terraform-request-approval.png) + +Open the `terraform-plan` job: +![](./docs/terraform-plan-1.png) + +Check the planned changes in the `Terraform plan` step: +![](./docs/terraform-plan-2.png) + +If the changes are acceptable approve the deployment: +![](./docs/terraform-request-approved.png) + +This should start the `terraform-apply` job which will apply the saved plan. + +### GitHub secrets and environment setup + +The terraform plan needs to be passed between jobs as an artifact, but may contain sensitive information so must not be visible to downloaders. +[`age`](https://github.com/FiloSottile/age) is used to encrypt and decrypt the artifact. + +A GitHub secret `TFPLAN_ARTIFACT_SECRET_KEY` must be created by running + +``` +age-keygen +``` + +and saving the `AGE-SECRET-KEY-*` output as the GitHub secret. + +The secret is only needed by the GitHub workflow, it is never required outside the workflow so should be treated as a write-once read-never secret which is not recorded anywhere else. +It can be rotated at any time between deployments. + +The GitHub environment `aws-curvenote` must be created before the first deployment with the following settings: + +- Only branch `main` can deploy to this environment +- Require reviewers to approve deployments + +## Accessing Kubernetes from a GitHub workflow + +All access to the Kubernetes cluster is managed using [GitHub OIDC](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services). + +AWS secret tokens are not required. +AWS API access for BinderHub components, for example to ECR, is managed using [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). diff --git a/terraform/aws/curvenote/bootstrap/backend.tf b/terraform/aws/curvenote/bootstrap/backend.tf new file mode 100644 index 000000000..b7e94579e --- /dev/null +++ b/terraform/aws/curvenote/bootstrap/backend.tf @@ -0,0 +1,62 @@ +# Initial setup of S3 bucket to store tfstate file +variable "state_bucket_name" { + type = string + # python -c 'import random; import string; print("".join(random.choices(string.ascii_lowercase + string.digits,k=12)))' + default = "binderhub-tfstate-7rjazazm1c7k" + description = "Bucket name for Terraform state file" +} + +variable "region" { + type = string + default = "us-east-2" + description = "AWS region" +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } + required_version = ">= 1.4.6" +} + +# Configure the AWS Provider +provider "aws" { + region = var.region + default_tags { + tags = { + "owner" : "binderhub" + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = var.state_bucket_name +} + +resource "aws_s3_bucket_versioning" "bucket_versioning" { + bucket = aws_s3_bucket.bucket.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_public_access_block" "public-block" { + bucket = aws_s3_bucket.bucket.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_dynamodb_table" "tfstate-lock" { + hash_key = "LockID" + name = "dynamodb-state-locking" + attribute { + name = "LockID" + type = "S" + } + billing_mode = "PAY_PER_REQUEST" +} diff --git a/terraform/aws/curvenote/docs/terraform-plan-1.png b/terraform/aws/curvenote/docs/terraform-plan-1.png new file mode 100644 index 000000000..b345a5e5d Binary files /dev/null and b/terraform/aws/curvenote/docs/terraform-plan-1.png differ diff --git a/terraform/aws/curvenote/docs/terraform-plan-2.png b/terraform/aws/curvenote/docs/terraform-plan-2.png new file mode 100644 index 000000000..1f7235a49 Binary files /dev/null and b/terraform/aws/curvenote/docs/terraform-plan-2.png differ diff --git a/terraform/aws/curvenote/docs/terraform-request-approval.png b/terraform/aws/curvenote/docs/terraform-request-approval.png new file mode 100644 index 000000000..898b85d2a Binary files /dev/null and b/terraform/aws/curvenote/docs/terraform-request-approval.png differ diff --git a/terraform/aws/curvenote/docs/terraform-request-approved.png b/terraform/aws/curvenote/docs/terraform-request-approved.png new file mode 100644 index 000000000..56ac05d12 Binary files /dev/null and b/terraform/aws/curvenote/docs/terraform-request-approved.png differ diff --git a/terraform/aws/curvenote/main.tf b/terraform/aws/curvenote/main.tf new file mode 100644 index 000000000..ec6489923 --- /dev/null +++ b/terraform/aws/curvenote/main.tf @@ -0,0 +1,60 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.5" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.21" + } + } + + required_version = ">= 1.4.6" + + # Bootstrapping: Create the bucket and DynamoDB table using the ./bootstrap directory + backend "s3" { + bucket = "binderhub-tfstate-7rjazazm1c7k" + key = "tfstate/dev/binderhub-dev" + region = "us-east-2" + dynamodb_table = "dynamodb-state-locking" + } +} + +provider "aws" { + region = "us-east-2" + default_tags { + tags = { + "owner" : "binderhub" + } + } +} + +module "binder-eks" { + source = "../binder-eks" + region = "us-east-2" + cluster_name = "binderhub" + k8s_version = "1.26" + k8s_api_cidrs = ["0.0.0.0/0"] + number_azs = 1 + instance_type_wg1 = "r6a.2xlarge" + use_bottlerocket = false + root_volume_size = 100 + wg1_size = 2 + wg1_max_size = 2 + + # The following configuration is needed if you are using a limited IAM role for deployment + enable_irsa = false + oidc_created = true + + github_oidc_role_suffix = "github-oidc-mybinderorgdeploy" + permissions_boundary_name = "system/binderhub_policy" +} + +# Needed so that Terraform can manage the EKS auth configmap +provider "kubernetes" { + host = module.binder-eks.cluster_endpoint + cluster_ca_certificate = module.binder-eks.cluster_ca_certificate + token = module.binder-eks.eks_token +}