Skip to content

Commit

Permalink
IAM support for airflow worker
Browse files Browse the repository at this point in the history
  • Loading branch information
Drew Meyers committed Mar 1, 2024
1 parent dd4208f commit 4ecba38
Show file tree
Hide file tree
Showing 5 changed files with 85 additions and 12 deletions.
2 changes: 1 addition & 1 deletion terraform-unity/modules/terraform-eks-cluster/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes |
| <a name="input_nodegroups"></a> [nodegroups](#input\_nodegroups) | A map of node group configurations | <pre>map(object({<br> instance_types = list(string)<br> min_size = number<br> max_size = number<br> desired_size = number<br> }))</pre> | <pre>{<br> "defaultGroup": {<br> "desired_size": 1,<br> "instance_types": [<br> "m5.xlarge"<br> ],<br> "max_size": 1,<br> "min_size": 1<br> }<br>}</pre> | no |
| <a name="input_nodegroups"></a> [nodegroups](#input\_nodegroups) | A map of node group configurations | <pre>map(object({<br> create_iam_role = optional(bool)<br> iam_role_arn = optional(string)<br> ami_id = optional(string)<br> min_size = optional(number)<br> max_size = optional(number)<br> desired_size = optional(number)<br> instance_types = optional(list(string))<br> capacity_type = optional(string)<br> enable_bootstrap_user_data = optional(bool)<br> metadata_options = optional(map(any))<br> }))</pre> | <pre>{<br> "defaultGroup": {<br> "desired_size": 1,<br> "instance_types": [<br> "m5.xlarge"<br> ],<br> "max_size": 1,<br> "metadata_options": {<br> "http_endpoint": "enabled",<br> "http_put_response_hop_limit": 3<br> },<br> "min_size": 1<br> }<br>}</pre> | no |

## Outputs

Expand Down
20 changes: 16 additions & 4 deletions terraform-unity/modules/terraform-eks-cluster/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,30 @@ variable "cluster_name" {

variable "nodegroups" {
description = "A map of node group configurations"

type = map(object({
instance_types = list(string)
min_size = number
max_size = number
desired_size = number
create_iam_role = optional(bool)
iam_role_arn = optional(string)
ami_id = optional(string)
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
instance_types = optional(list(string))
capacity_type = optional(string)
enable_bootstrap_user_data = optional(bool)
metadata_options = optional(map(any))
}))

default = {
defaultGroup = {
instance_types = ["m5.xlarge"]
min_size = 1
max_size = 1
desired_size = 1
metadata_options = {
"http_endpoint" : "enabled",
"http_put_response_hop_limit" : 3,
}
}
}
}
5 changes: 4 additions & 1 deletion terraform-unity/modules/terraform-unity-sps-airflow/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ No modules.
|------|------|
| [aws_db_instance.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/db_instance) | resource |
| [aws_db_subnet_group.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/db_subnet_group) | resource |
| [aws_iam_policy.airflow_worker_policy](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/iam_policy) | resource |
| [aws_iam_role.airflow_worker_role](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/iam_role) | resource |
| [aws_iam_role_policy_attachment.airflow_worker_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/iam_role_policy_attachment) | resource |
| [aws_s3_bucket.airflow_logs](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/s3_bucket) | resource |
| [aws_secretsmanager_secret.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/secretsmanager_secret) | resource |
| [aws_secretsmanager_secret_version.airflow_db](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/secretsmanager_secret_version) | resource |
Expand All @@ -54,9 +57,9 @@ No modules.
| [random_id.airflow_webserver_secret](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/id) | resource |
| [random_id.counter](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/id) | resource |
| [random_password.airflow_db](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/password) | resource |
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/caller_identity) | data source |
| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster) | data source |
| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster_auth) | data source |
| [aws_eks_node_group.default](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_node_group) | data source |
| [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/security_group) | data source |
| [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/ssm_parameter) | data source |
| [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/data-sources/ingress_v1) | data source |
Expand Down
5 changes: 0 additions & 5 deletions terraform-unity/modules/terraform-unity-sps-airflow/data.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,3 @@ data "kubernetes_ingress_v1" "ogc_processes_api_ingress" {
namespace = kubernetes_namespace.airflow.metadata[0].name
}
}

data "aws_eks_node_group" "default" {
cluster_name = var.eks_cluster_name
node_group_name = "defaultGroup"
}
65 changes: 64 additions & 1 deletion terraform-unity/modules/terraform-unity-sps-airflow/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ resource "helm_release" "airflow" {
metadata_secret_name = "airflow-metadata-secret"
webserver_secret_name = "airflow-webserver-secret"
airflow_logs_s3_location = "s3://${aws_s3_bucket.airflow_logs.id}"
airflow_worker_role_arn = data.aws_eks_node_group.default.node_role_arn
airflow_worker_role_arn = aws_iam_role.airflow_worker_role.arn
})
]
set_sensitive {
Expand Down Expand Up @@ -369,3 +369,66 @@ resource "kubernetes_ingress_v1" "ogc_processes_api_ingress" {
}
wait_for_load_balancer = true
}

data "aws_caller_identity" "current" {}


resource "aws_iam_policy" "airflow_worker_policy" {
name = "AirflowWorkerPolicy"
description = "Policy for Airflow Workers to access AWS services"
policy = jsonencode(
{
"Version" : "2012-10-17",
"Statement" : [
{
"Effect" : "Allow",
"Action" : [
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:CreateLogGroup",
"s3:ListBucket",
"s3:GetObject",
"s3:PutObject",
"sqs:SendMessage",
"sqs:ReceiveMessage",
"sns:Publish",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"secretsmanager:GetSecretValue",
"ssm:GetParameters"
],
"Resource" : "*"
}
]
}
)
}

resource "aws_iam_role" "airflow_worker_role" {
name = "AirflowWorkerRole"
assume_role_policy = jsonencode(
{
"Version" : "2012-10-17",
"Statement" : [
{
"Effect" : "Allow",
"Principal" : {
"Federated" : "arn:aws:iam::429178552491:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/7E914336E4CD991EA4B403BA606CB778"
},
"Action" : "sts:AssumeRoleWithWebIdentity",
"Condition" : {
"StringEquals" : {
"oidc.eks.us-west-2.amazonaws.com/id/7E914336E4CD991EA4B403BA606CB778:sub" : "system:serviceaccount:airflow:airflow-worker"
}
}
}
]
}
)
permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/mcp-tenantOperator-AMI-APIG"
}

resource "aws_iam_role_policy_attachment" "airflow_worker_policy_attachment" {
role = aws_iam_role.airflow_worker_role.name
policy_arn = aws_iam_policy.airflow_worker_policy.arn
}

0 comments on commit 4ecba38

Please sign in to comment.