From b7fee1a8e76a9f7eb8a8372ab1c2c35bd4deb4c2 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:22:05 -0700 Subject: [PATCH 01/28] feat: New module for existing catalogs for new volumes --- .../Makefile | 12 +++ .../README.md | 76 ++++++++++++++++ databricks-s3-volume-existing-catalog/fogg.tf | 2 + .../grants.tf | 14 +++ databricks-s3-volume-existing-catalog/iam.tf | 90 +++++++++++++++++++ databricks-s3-volume-existing-catalog/main.tf | 56 ++++++++++++ .../outputs.tf | 11 +++ .../variables.tf | 53 +++++++++++ .../versions.tf | 11 +++ 9 files changed, 325 insertions(+) create mode 100644 databricks-s3-volume-existing-catalog/Makefile create mode 100644 databricks-s3-volume-existing-catalog/README.md create mode 100644 databricks-s3-volume-existing-catalog/fogg.tf create mode 100644 databricks-s3-volume-existing-catalog/grants.tf create mode 100644 databricks-s3-volume-existing-catalog/iam.tf create mode 100644 databricks-s3-volume-existing-catalog/main.tf create mode 100644 databricks-s3-volume-existing-catalog/outputs.tf create mode 100644 databricks-s3-volume-existing-catalog/variables.tf create mode 100644 databricks-s3-volume-existing-catalog/versions.tf diff --git a/databricks-s3-volume-existing-catalog/Makefile b/databricks-s3-volume-existing-catalog/Makefile new file mode 100644 index 00000000..bf1f5140 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/Makefile @@ -0,0 +1,12 @@ +# Auto-generated by fogg. Do not edit +# Make improvements in fogg, so that everyone can benefit. + +export TERRAFORM_VERSION := 1.3.0 +export TF_PLUGIN_CACHE_DIR := ../../..//.terraform.d/plugin-cache + +include ../../..//scripts/module.mk + + +help: ## display help for this makefile + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +.PHONY: help diff --git a/databricks-s3-volume-existing-catalog/README.md b/databricks-s3-volume-existing-catalog/README.md new file mode 100644 index 00000000..d5c783c0 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/README.md @@ -0,0 +1,76 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [databricks](#provider\_databricks) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [databricks\_bucket](#module\_databricks\_bucket) | github.com/chanzuckerberg/cztack//aws-s3-private-bucket | v0.71.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy.dbx_unity_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.dbx_unity_aws_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.dbx_unity_aws_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [databricks_catalog.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource | +| [databricks_external_location.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/external_location) | resource | +| [databricks_grant.catalog_r](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | +| [databricks_grant.catalog_rw](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | +| [databricks_grant.schema_r](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | +| [databricks_grant.schema_rw](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | +| [databricks_grant.volume_r](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | +| [databricks_grant.volume_rw](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | +| [databricks_schema.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource | +| [databricks_storage_credential.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/storage_credential) | resource | +| [databricks_volume.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/volume) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.databricks-s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.dbx_unity_aws_role_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.volume_bucket_dbx_unity_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_rw\_bucket\_grant\_arns](#input\_additional\_rw\_bucket\_grant\_arns) | (Optional) Additional AWS ARNs to grant read/write permissions to on the bucket (may be necessary for service principals, instance profiles, or users) | `list(string)` | `[]` | no | +| [bucket\_object\_ownership](#input\_bucket\_object\_ownership) | Set default owner of all objects within bucket (e.g., bucket vs. object owner) | `string` | `null` | no | +| [catalog\_name](#input\_catalog\_name) | Name of the Databricks existing catalog to add the volume to | `string` | n/a | yes | +| [catalog\_owner](#input\_catalog\_owner) | User or group name of the catalog owner | `string` | n/a | yes | +| [create\_catalog](#input\_create\_catalog) | Flag to create a new catalog or look for an existing one with the given name | `bool` | n/a | yes | +| [catalog\_r\_grant\_principals](#input\_catalog\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the catalog | `list(string)` | `[]` | no | +| [catalog\_rw\_grant\_principals](#input\_catalog\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the catalog | `list(string)` | `[]` | no | +| [metastore\_id](#input\_metastore\_id) | ID of metastore to create catalog in | `string` | n/a | yes | +| [create\_schema](#input\_create\_schema) | Flag to create a new schema or look for an existing one with the given name | `bool` | n/a | yes | +| [schema\_name](#input\_schema\_name) | Name of the Databricks existing schema to add the volume to | `string` | n/a | yes | +| [schema\_r\_grant\_principals](#input\_schema\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the schema | `list(string)` | `[]` | no | +| [schema\_rw\_grant\_principals](#input\_schema\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the schema | `list(string)` | `[]` | no | +| [tags](#input\_tags) | REQUIRED: Tags to include for this environment. | 
object({
    project : string
    env : string
    service : string
    owner : string
    managedBy : string
  })
| n/a | yes | +| [volume\_bucket](#input\_volume\_bucket) | (Optional) Name of an existing S3 bucket to use for Databricks volume. NOTE: if provided, you will need to update the bucket policy whereever it is defined to allow Databricks access | `string` | `null` | no | +| [volume\_comment](#input\_volume\_comment) | (Optional) Comment to add to the Databricks volume | `string` | `"Managed by Terraform - this is a default volume for the Databricks workspace"` | no | +| [volume\_name](#input\_volume\_name) | Name of the Databricks volume to create | `string` | n/a | yes | +| [volume\_r\_grant\_principals](#input\_volume\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the volume | `list(string)` | `[]` | no | +| [volume\_rw\_grant\_principals](#input\_volume\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the volume | `list(string)` | `[]` | no | +| [volume\_schema\_properties](#input\_volume\_schema\_properties) | Properties of the Databricks schema to add the volume to | `map(string)` | `{}` | no | +| [workspace\_name](#input\_workspace\_name) | Name of the Databricks catalog to add the volume to | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dbx\_unity\_aws\_role\_arn](#output\_dbx\_unity\_aws\_role\_arn) | n/a | +| [volume\_path](#output\_volume\_path) | n/a | +| [volume\_specific\_bucket\_name](#output\_volume\_specific\_bucket\_name) | n/a | + \ No newline at end of file diff --git a/databricks-s3-volume-existing-catalog/fogg.tf b/databricks-s3-volume-existing-catalog/fogg.tf new file mode 100644 index 00000000..8c7ad42a --- /dev/null +++ b/databricks-s3-volume-existing-catalog/fogg.tf @@ -0,0 +1,2 @@ +# Auto-generated by fogg. Do not edit +# Make improvements in fogg, so that everyone can benefit. diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf new file mode 100644 index 00000000..4d334b44 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -0,0 +1,14 @@ +# volume +resource "databricks_grant" "volume_r" { + for_each = toset(var.volume_r_grant_principals) + volume = databricks_volume.volume.id + principal = each.value + privileges = ["READ_VOLUME"] +} + +resource "databricks_grant" "volume_rw" { + for_each = toset(var.volume_rw_grant_principals) + volume = databricks_volume.volume.id + principal = each.value + privileges = ["READ_VOLUME", "WRITE_VOLUME"] +} \ No newline at end of file diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf new file mode 100644 index 00000000..2b319259 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -0,0 +1,90 @@ +## Databricks external location and IAM + +data "aws_caller_identity" "current" { + provider = aws +} + +data "aws_iam_policy_document" "volume_dbx_unity_aws_role_assume_role" { + statement { + principals { + type = "AWS" + identifiers = ["arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"] + } + + actions = ["sts:AssumeRole"] + condition { + test = "StringEquals" + variable = "sts:ExternalId" + + values = ["4a2f419c-ae7a-49f1-b774-8f3113d9834d"] + } + } + statement { + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + + actions = ["sts:AssumeRole"] + condition { + test = "ArnEquals" + variable = "aws:PrincipalArn" + values = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:role${local.path}${local.dbx_volume_aws_role_name}"] + } + } +} + +resource "aws_iam_role" "volume_dbx_unity_aws_role" { + name = local.dbx_volume_aws_role_name + path = local.path + assume_role_policy = data.aws_iam_policy_document.dbx_volume_aws_role_assume_role.json +} + + +### Policy document to access default volume bucket and assume role +data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { + statement { + sid = "dbxSCBucketAccess" + effect = "Allow" + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:GetLifecycleConfiguration", + "s3:PutLifecycleConfiguration" + ] + resources = [ + "arn:aws:s3:::${var.volume_name}" + ] + } + statement { + sid = "dbxSCObjAccess" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + ] + resources = [ + "arn:aws:s3:::${var.volume_name}/*" + ] + } + statement { + sid = "databricksAssumeRole" + effect = "Allow" + actions = [ + "sts:AssumeRole" + ] + resources = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role${local.path}${local.dbx_volume_aws_role_name}" + ] + } +} + +resource "aws_iam_policy" "volume_dbx_unity_access_policy" { + policy = data.aws_iam_policy_document.volume_bucket_dbx_unity_access.json +} + +resource "aws_iam_role_policy_attachment" "dbx_unity_aws_access" { + policy_arn = aws_iam_policy.volume_dbx_unity_access_policy.arn + role = aws_iam_role.dbx_volume_aws_role_name.name +} diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf new file mode 100644 index 00000000..20cb9222 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -0,0 +1,56 @@ +# Volume bucket (UC supported) + +// https://docs.databricks.com/administration-guide/multiworkspace/iam-role.html#language-Your%C2%A0VPC,%C2%A0custom +locals { + name = "${var.tags.project}-${var.tags.env}-${var.tags.service}" + dbx_volume_aws_role_name = "${local.name}-${var.volume_name}-role" + dbx_volume_storage_credential_name = "${local.name}-${var.volume_name}-storage-credential" + + path = "/databricks/" + databricks_aws_account = "414351767826" # Databricks' own AWS account, not CZI's. See https://docs.databricks.com/en/administration-guide/account-settings-e2/credentials.html#step-1-create-a-cross-account-iam-role +} + +### Databricks storage credential - allows workspace to access an external location. +### NOTE: names need to be unique across an account, not just a workspace + +resource "databricks_storage_credential" "volume" { + depends_on = [ + resource.aws_iam_role.volume_dbx_unity_aws_role, + resource.aws_iam_role_policy_attachment.volume_bucket_dbx_unity_access + ] + + name = local.dbx_volume_storage_credential_name + aws_iam_role { + role_arn = aws_iam_role.volume_dbx_unity_aws_role.arn + } + comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" +} + +# upstream external location sometimes takes a moment to register +resource "time_sleep" "wait_30_seconds" { + depends_on = [databricks_storage_credential.volume] + + create_duration = "30s" +} + +resource "databricks_external_location" "volume" { + depends_on = [time_sleep.wait_30_seconds] + + name = var.volume_name + url = "s3://${var.volume_name}" + credential_name = databricks_storage_credential.volume.name + comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" +} + +# New volume + +resource "databricks_volume" "volume" { + depends_on = [databricks_external_location.volume] + name = "${var.volume_name}" + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_type = "EXTERNAL" + storage_location = "s3://${var.volume_name}/${var.schema_name}" + owner = var.catalog_owner + comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" +} diff --git a/databricks-s3-volume-existing-catalog/outputs.tf b/databricks-s3-volume-existing-catalog/outputs.tf new file mode 100644 index 00000000..1f36f081 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/outputs.tf @@ -0,0 +1,11 @@ +output "dbx_unity_aws_role_arn" { + value = length(aws_iam_role.dbx_unity_aws_role) > 0 ? aws_iam_role.dbx_unity_aws_role[0].arn : null +} + +output "volume_specific_bucket_name" { + value = length(module.databricks_bucket) > 0 ? module.databricks_bucket[0].name : null +} + +output "volume_path" { + value = "${local.catalog_name}.${local.schema_name}.${local.volume_name}" +} \ No newline at end of file diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf new file mode 100644 index 00000000..7bfdecce --- /dev/null +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -0,0 +1,53 @@ + +variable "workspace_name" { + description = "Name of the Databricks catalog to add the volume to" + type = string +} + +variable "catalog_name" { + description = "Name of the Databricks existing catalog to add the volume to" + type = string +} + +variable "catalog_owner" { + description = "User or group name of the catalog owner" + type = string +} + +variable "metastore_id" { + description = "ID of metastore to create catalog in" + type = string +} + +variable "schema_name" { + description = "Name of the Databricks schema to add the volume to" + type = string +} + +variable "volume_name" { + description = "Name of the Databricks volume to create" + type = string +} + +variable "volume_r_grant_principals" { + description = "(Optional) Databricks groups to grant read-only permissions to on the volume" + type = list(string) + default = [] +} + +variable "volume_rw_grant_principals" { + description = "(Optional) Databricks groups to grant read/write permissions to on the volume" + type = list(string) + default = [] +} + +variable "tags" { + description = "REQUIRED: Tags to include for this environment." + type = object({ + project : string + env : string + service : string + owner : string + managedBy : string + }) +} \ No newline at end of file diff --git a/databricks-s3-volume-existing-catalog/versions.tf b/databricks-s3-volume-existing-catalog/versions.tf new file mode 100644 index 00000000..159e8002 --- /dev/null +++ b/databricks-s3-volume-existing-catalog/versions.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + databricks = { + source = "databricks/databricks" + } + } + required_version = ">= 1.3.0" +} From 5b9359e76fe96fc7b1195bef84efe6c760957c5b Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:25:01 -0700 Subject: [PATCH 02/28] remove --- .../Makefile | 12 --- .../README.md | 76 ------------------- databricks-s3-volume-existing-catalog/fogg.tf | 2 - .../outputs.tf | 11 --- 4 files changed, 101 deletions(-) delete mode 100644 databricks-s3-volume-existing-catalog/Makefile delete mode 100644 databricks-s3-volume-existing-catalog/README.md delete mode 100644 databricks-s3-volume-existing-catalog/fogg.tf delete mode 100644 databricks-s3-volume-existing-catalog/outputs.tf diff --git a/databricks-s3-volume-existing-catalog/Makefile b/databricks-s3-volume-existing-catalog/Makefile deleted file mode 100644 index bf1f5140..00000000 --- a/databricks-s3-volume-existing-catalog/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# Auto-generated by fogg. Do not edit -# Make improvements in fogg, so that everyone can benefit. - -export TERRAFORM_VERSION := 1.3.0 -export TF_PLUGIN_CACHE_DIR := ../../..//.terraform.d/plugin-cache - -include ../../..//scripts/module.mk - - -help: ## display help for this makefile - @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: help diff --git a/databricks-s3-volume-existing-catalog/README.md b/databricks-s3-volume-existing-catalog/README.md deleted file mode 100644 index d5c783c0..00000000 --- a/databricks-s3-volume-existing-catalog/README.md +++ /dev/null @@ -1,76 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | n/a | -| [databricks](#provider\_databricks) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [databricks\_bucket](#module\_databricks\_bucket) | github.com/chanzuckerberg/cztack//aws-s3-private-bucket | v0.71.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.dbx_unity_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.dbx_unity_aws_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.dbx_unity_aws_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [databricks_catalog.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource | -| [databricks_external_location.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/external_location) | resource | -| [databricks_grant.catalog_r](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | -| [databricks_grant.catalog_rw](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | -| [databricks_grant.schema_r](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | -| [databricks_grant.schema_rw](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | -| [databricks_grant.volume_r](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | -| [databricks_grant.volume_rw](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) | resource | -| [databricks_schema.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource | -| [databricks_storage_credential.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/storage_credential) | resource | -| [databricks_volume.volume](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/volume) | resource | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.databricks-s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.dbx_unity_aws_role_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.volume_bucket_dbx_unity_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_rw\_bucket\_grant\_arns](#input\_additional\_rw\_bucket\_grant\_arns) | (Optional) Additional AWS ARNs to grant read/write permissions to on the bucket (may be necessary for service principals, instance profiles, or users) | `list(string)` | `[]` | no | -| [bucket\_object\_ownership](#input\_bucket\_object\_ownership) | Set default owner of all objects within bucket (e.g., bucket vs. object owner) | `string` | `null` | no | -| [catalog\_name](#input\_catalog\_name) | Name of the Databricks existing catalog to add the volume to | `string` | n/a | yes | -| [catalog\_owner](#input\_catalog\_owner) | User or group name of the catalog owner | `string` | n/a | yes | -| [create\_catalog](#input\_create\_catalog) | Flag to create a new catalog or look for an existing one with the given name | `bool` | n/a | yes | -| [catalog\_r\_grant\_principals](#input\_catalog\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the catalog | `list(string)` | `[]` | no | -| [catalog\_rw\_grant\_principals](#input\_catalog\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the catalog | `list(string)` | `[]` | no | -| [metastore\_id](#input\_metastore\_id) | ID of metastore to create catalog in | `string` | n/a | yes | -| [create\_schema](#input\_create\_schema) | Flag to create a new schema or look for an existing one with the given name | `bool` | n/a | yes | -| [schema\_name](#input\_schema\_name) | Name of the Databricks existing schema to add the volume to | `string` | n/a | yes | -| [schema\_r\_grant\_principals](#input\_schema\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the schema | `list(string)` | `[]` | no | -| [schema\_rw\_grant\_principals](#input\_schema\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the schema | `list(string)` | `[]` | no | -| [tags](#input\_tags) | REQUIRED: Tags to include for this environment. | 
object({
    project : string
    env : string
    service : string
    owner : string
    managedBy : string
  })
| n/a | yes | -| [volume\_bucket](#input\_volume\_bucket) | (Optional) Name of an existing S3 bucket to use for Databricks volume. NOTE: if provided, you will need to update the bucket policy whereever it is defined to allow Databricks access | `string` | `null` | no | -| [volume\_comment](#input\_volume\_comment) | (Optional) Comment to add to the Databricks volume | `string` | `"Managed by Terraform - this is a default volume for the Databricks workspace"` | no | -| [volume\_name](#input\_volume\_name) | Name of the Databricks volume to create | `string` | n/a | yes | -| [volume\_r\_grant\_principals](#input\_volume\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the volume | `list(string)` | `[]` | no | -| [volume\_rw\_grant\_principals](#input\_volume\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the volume | `list(string)` | `[]` | no | -| [volume\_schema\_properties](#input\_volume\_schema\_properties) | Properties of the Databricks schema to add the volume to | `map(string)` | `{}` | no | -| [workspace\_name](#input\_workspace\_name) | Name of the Databricks catalog to add the volume to | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dbx\_unity\_aws\_role\_arn](#output\_dbx\_unity\_aws\_role\_arn) | n/a | -| [volume\_path](#output\_volume\_path) | n/a | -| [volume\_specific\_bucket\_name](#output\_volume\_specific\_bucket\_name) | n/a | - \ No newline at end of file diff --git a/databricks-s3-volume-existing-catalog/fogg.tf b/databricks-s3-volume-existing-catalog/fogg.tf deleted file mode 100644 index 8c7ad42a..00000000 --- a/databricks-s3-volume-existing-catalog/fogg.tf +++ /dev/null @@ -1,2 +0,0 @@ -# Auto-generated by fogg. Do not edit -# Make improvements in fogg, so that everyone can benefit. diff --git a/databricks-s3-volume-existing-catalog/outputs.tf b/databricks-s3-volume-existing-catalog/outputs.tf deleted file mode 100644 index 1f36f081..00000000 --- a/databricks-s3-volume-existing-catalog/outputs.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "dbx_unity_aws_role_arn" { - value = length(aws_iam_role.dbx_unity_aws_role) > 0 ? aws_iam_role.dbx_unity_aws_role[0].arn : null -} - -output "volume_specific_bucket_name" { - value = length(module.databricks_bucket) > 0 ? module.databricks_bucket[0].name : null -} - -output "volume_path" { - value = "${local.catalog_name}.${local.schema_name}.${local.volume_name}" -} \ No newline at end of file From 418c339590f40d949bb7dcf1b9fff135d7c8fb02 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:26:33 -0700 Subject: [PATCH 03/28] remove --- databricks-s3-volume-existing-catalog/variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index 7bfdecce..2802a98f 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -14,11 +14,6 @@ variable "catalog_owner" { type = string } -variable "metastore_id" { - description = "ID of metastore to create catalog in" - type = string -} - variable "schema_name" { description = "Name of the Databricks schema to add the volume to" type = string From 3081256f7c7f9640f297fba36c999e550f16ff7e Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:30:52 -0700 Subject: [PATCH 04/28] fix --- databricks-s3-volume-existing-catalog/iam.tf | 4 ++-- databricks-s3-volume-existing-catalog/main.tf | 4 ++-- databricks-s3-volume-existing-catalog/variables.tf | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf index 2b319259..aa29945e 100644 --- a/databricks-s3-volume-existing-catalog/iam.tf +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -53,7 +53,7 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { "s3:PutLifecycleConfiguration" ] resources = [ - "arn:aws:s3:::${var.volume_name}" + "arn:aws:s3:::${var.volume_bucket}" ] } statement { @@ -65,7 +65,7 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { "s3:DeleteObject", ] resources = [ - "arn:aws:s3:::${var.volume_name}/*" + "arn:aws:s3:::${var.volume_bucket}/*" ] } statement { diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index 20cb9222..d6cd5d67 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -37,7 +37,7 @@ resource "databricks_external_location" "volume" { depends_on = [time_sleep.wait_30_seconds] name = var.volume_name - url = "s3://${var.volume_name}" + url = "s3://${var.volume_bucket}" credential_name = databricks_storage_credential.volume.name comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" } @@ -50,7 +50,7 @@ resource "databricks_volume" "volume" { catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${var.volume_name}/${var.schema_name}" + storage_location = "s3://${var.volume_bucket}/${var.schema_name}" owner = var.catalog_owner comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" } diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index 2802a98f..c8feaf53 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -24,6 +24,11 @@ variable "volume_name" { type = string } +variable "volume_bucket" { + description = "Name of the S3 bucket to use for the volume" + type = string +} + variable "volume_r_grant_principals" { description = "(Optional) Databricks groups to grant read-only permissions to on the volume" type = list(string) From aaeb1b5eb81b1160a7c82a95be1297d2e3ed5273 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:31:32 -0700 Subject: [PATCH 05/28] remove --- databricks-s3-volume-existing-catalog/variables.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index c8feaf53..e174f1e7 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -1,9 +1,3 @@ - -variable "workspace_name" { - description = "Name of the Databricks catalog to add the volume to" - type = string -} - variable "catalog_name" { description = "Name of the Databricks existing catalog to add the volume to" type = string From 84b5163e4f082e58804c02a772f19aeedb819ce4 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:44:14 -0700 Subject: [PATCH 06/28] fix --- databricks-s3-volume-existing-catalog/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf index aa29945e..f0ea2932 100644 --- a/databricks-s3-volume-existing-catalog/iam.tf +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -37,7 +37,7 @@ data "aws_iam_policy_document" "volume_dbx_unity_aws_role_assume_role" { resource "aws_iam_role" "volume_dbx_unity_aws_role" { name = local.dbx_volume_aws_role_name path = local.path - assume_role_policy = data.aws_iam_policy_document.dbx_volume_aws_role_assume_role.json + assume_role_policy = data.aws_iam_policy_document.volume_dbx_unity_aws_role_assume_role.json } From f8f15de522d291e18b88d61c0d7486facb377e74 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:44:35 -0700 Subject: [PATCH 07/28] fix --- databricks-s3-volume-existing-catalog/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf index f0ea2932..8998a3d1 100644 --- a/databricks-s3-volume-existing-catalog/iam.tf +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -86,5 +86,5 @@ resource "aws_iam_policy" "volume_dbx_unity_access_policy" { resource "aws_iam_role_policy_attachment" "dbx_unity_aws_access" { policy_arn = aws_iam_policy.volume_dbx_unity_access_policy.arn - role = aws_iam_role.dbx_volume_aws_role_name.name + role = aws_iam_role.volume_dbx_unity_aws_role.name } From f4dd876cafb4d5c0bb9f5ccb43a274ea7b95ca8e Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 14:49:31 -0700 Subject: [PATCH 08/28] fix --- databricks-s3-volume-existing-catalog/iam.tf | 2 +- databricks-s3-volume-existing-catalog/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf index 8998a3d1..aafcabbc 100644 --- a/databricks-s3-volume-existing-catalog/iam.tf +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -84,7 +84,7 @@ resource "aws_iam_policy" "volume_dbx_unity_access_policy" { policy = data.aws_iam_policy_document.volume_bucket_dbx_unity_access.json } -resource "aws_iam_role_policy_attachment" "dbx_unity_aws_access" { +resource "aws_iam_role_policy_attachment" "volume_dbx_unity_aws_access" { policy_arn = aws_iam_policy.volume_dbx_unity_access_policy.arn role = aws_iam_role.volume_dbx_unity_aws_role.name } diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index d6cd5d67..a2d9139c 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -16,7 +16,7 @@ locals { resource "databricks_storage_credential" "volume" { depends_on = [ resource.aws_iam_role.volume_dbx_unity_aws_role, - resource.aws_iam_role_policy_attachment.volume_bucket_dbx_unity_access + resource.aws_iam_role_policy_attachment.volume_dbx_unity_aws_access ] name = local.dbx_volume_storage_credential_name From bb6c2c5316a02a75b0fb497914389d4c35f06688 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 15:11:49 -0700 Subject: [PATCH 09/28] make name same as bucket if no name is provided --- databricks-s3-volume-existing-catalog/main.tf | 2 +- databricks-s3-volume-existing-catalog/variables.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index a2d9139c..a0ffc033 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -46,7 +46,7 @@ resource "databricks_external_location" "volume" { resource "databricks_volume" "volume" { depends_on = [databricks_external_location.volume] - name = "${var.volume_name}" + name = var.volume_name catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index e174f1e7..7a5d8593 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -16,6 +16,7 @@ variable "schema_name" { variable "volume_name" { description = "Name of the Databricks volume to create" type = string + default = var.volume_bucket } variable "volume_bucket" { From ab25ff0e3eea273e30247dd81bce8a802480b585 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 15:19:21 -0700 Subject: [PATCH 10/28] dynamic policy --- databricks-s3-volume-existing-catalog/iam.tf | 4 ++-- databricks-s3-volume-existing-catalog/main.tf | 19 ++++++++++--------- .../variables.tf | 10 ++-------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf index aafcabbc..a1e0b2ae 100644 --- a/databricks-s3-volume-existing-catalog/iam.tf +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -53,7 +53,7 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { "s3:PutLifecycleConfiguration" ] resources = [ - "arn:aws:s3:::${var.volume_bucket}" + for bucket in var.volume_buckets : "arn:aws:s3:::${bucket}" ] } statement { @@ -65,7 +65,7 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { "s3:DeleteObject", ] resources = [ - "arn:aws:s3:::${var.volume_bucket}/*" + for bucket in var.volume_buckets : "arn:aws:s3:::${bucket}/*" ] } statement { diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index a0ffc033..d58f7da1 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -19,11 +19,11 @@ resource "databricks_storage_credential" "volume" { resource.aws_iam_role_policy_attachment.volume_dbx_unity_aws_access ] - name = local.dbx_volume_storage_credential_name + name = "${var.catalog_name}-volumes-role" aws_iam_role { role_arn = aws_iam_role.volume_dbx_unity_aws_role.arn } - comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" + comment = "Managed by Terraform - access for the volumes in ${var.catalog_name}" } # upstream external location sometimes takes a moment to register @@ -34,23 +34,24 @@ resource "time_sleep" "wait_30_seconds" { } resource "databricks_external_location" "volume" { + for_each = var.volume_buckets depends_on = [time_sleep.wait_30_seconds] - name = var.volume_name - url = "s3://${var.volume_bucket}" + name = "${each.key}-external-location" + url = "s3://${each.key}" credential_name = databricks_storage_credential.volume.name - comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" + comment = "Managed by Terraform - access for the volume named ${each.key} in ${var.catalog_name}" } # New volume - resource "databricks_volume" "volume" { + for_each = var.volume_buckets depends_on = [databricks_external_location.volume] - name = var.volume_name + name = each.key catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${var.volume_bucket}/${var.schema_name}" + storage_location = "s3://${each.key}/${var.schema_name}" owner = var.catalog_owner - comment = "Managed by Terraform - access for the volume named ${var.volume_name} in ${var.catalog_name}" + comment = "Managed by Terraform - access for the volume named ${each.key} in ${var.catalog_name}" } diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index 7a5d8593..a7850ff6 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -13,15 +13,9 @@ variable "schema_name" { type = string } -variable "volume_name" { - description = "Name of the Databricks volume to create" - type = string - default = var.volume_bucket -} - -variable "volume_bucket" { +variable "volume_buckets" { description = "Name of the S3 bucket to use for the volume" - type = string + type = list(string) } variable "volume_r_grant_principals" { From 45bcde828ab9487895c91dd4a9abf6bab5f84724 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 15:52:35 -0700 Subject: [PATCH 11/28] fix names --- databricks-s3-volume-existing-catalog/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index d58f7da1..a14a7a36 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -2,10 +2,6 @@ // https://docs.databricks.com/administration-guide/multiworkspace/iam-role.html#language-Your%C2%A0VPC,%C2%A0custom locals { - name = "${var.tags.project}-${var.tags.env}-${var.tags.service}" - dbx_volume_aws_role_name = "${local.name}-${var.volume_name}-role" - dbx_volume_storage_credential_name = "${local.name}-${var.volume_name}-storage-credential" - path = "/databricks/" databricks_aws_account = "414351767826" # Databricks' own AWS account, not CZI's. See https://docs.databricks.com/en/administration-guide/account-settings-e2/credentials.html#step-1-create-a-cross-account-iam-role } From 9c434f5af6f88d1a66d34927e747029fb360bc01 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 16:00:27 -0700 Subject: [PATCH 12/28] volumes role --- databricks-s3-volume-existing-catalog/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index a14a7a36..f2cad651 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -2,6 +2,7 @@ // https://docs.databricks.com/administration-guide/multiworkspace/iam-role.html#language-Your%C2%A0VPC,%C2%A0custom locals { + dbx_volume_aws_role_name = "${var.catalog_name}-volumes-role" path = "/databricks/" databricks_aws_account = "414351767826" # Databricks' own AWS account, not CZI's. See https://docs.databricks.com/en/administration-guide/account-settings-e2/credentials.html#step-1-create-a-cross-account-iam-role } From 3bda70176f32db006641fe6e56c3b2c1bc77313a Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 16:08:58 -0700 Subject: [PATCH 13/28] list of volumes --- .../grants.tf | 21 ++++++++++++------- .../variables.tf | 20 ++++++------------ 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index 4d334b44..b0e8e448 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -1,14 +1,19 @@ -# volume resource "databricks_grant" "volume_r" { - for_each = toset(var.volume_r_grant_principals) - volume = databricks_volume.volume.id - principal = each.value + for_each = { + for bucket in var.volume_buckets : "${bucket.bucket_name}_${bucket.volume_r_grant_principals}" => bucket + if length(bucket.volume_r_grant_principals) > 0 + } + volume = each.value.bucket_name + principal = each.value.volume_r_grant_principals[count.index] privileges = ["READ_VOLUME"] } resource "databricks_grant" "volume_rw" { - for_each = toset(var.volume_rw_grant_principals) - volume = databricks_volume.volume.id - principal = each.value + for_each = { + for bucket in var.volume_buckets : "${bucket.bucket_name}_${bucket.volume_rw_grant_principals}" => bucket + if length(bucket.volume_rw_grant_principals) > 0 + } + volume = each.value.bucket_name + principal = each.value.volume_rw_grant_principals[count.index] privileges = ["READ_VOLUME", "WRITE_VOLUME"] -} \ No newline at end of file +} diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index a7850ff6..5933f695 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -14,20 +14,12 @@ variable "schema_name" { } variable "volume_buckets" { - description = "Name of the S3 bucket to use for the volume" - type = list(string) -} - -variable "volume_r_grant_principals" { - description = "(Optional) Databricks groups to grant read-only permissions to on the volume" - type = list(string) - default = [] -} - -variable "volume_rw_grant_principals" { - description = "(Optional) Databricks groups to grant read/write permissions to on the volume" - type = list(string) - default = [] + description = "List of external buckets and their corresponding groups that should have r/rw access to it" + type = list(object({ + bucket_name : string + volume_r_grant_principals: optional(list(string), []) + volume_rw_grant_principals: optional(list(string), []) + })) } variable "tags" { From a3625edf8061087c1980480d854eb549649eb37e Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 16:09:40 -0700 Subject: [PATCH 14/28] bucket names --- databricks-s3-volume-existing-catalog/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index f2cad651..19129191 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -34,21 +34,21 @@ resource "databricks_external_location" "volume" { for_each = var.volume_buckets depends_on = [time_sleep.wait_30_seconds] - name = "${each.key}-external-location" - url = "s3://${each.key}" + name = "${each.value.bucket_name}-external-location" + url = "s3://${each.value.bucket_name}" credential_name = databricks_storage_credential.volume.name - comment = "Managed by Terraform - access for the volume named ${each.key} in ${var.catalog_name}" + comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" } # New volume resource "databricks_volume" "volume" { for_each = var.volume_buckets depends_on = [databricks_external_location.volume] - name = each.key + name = each.value.bucket_name catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${each.key}/${var.schema_name}" + storage_location = "s3://${each.value.bucket_name}/${var.schema_name}" owner = var.catalog_owner - comment = "Managed by Terraform - access for the volume named ${each.key} in ${var.catalog_name}" + comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" } From 4a55197550bfa5b66cb653b299cc96dd0523aa1c Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 16:46:19 -0700 Subject: [PATCH 15/28] flatten --- .../grants.tf | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index b0e8e448..915173a2 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -1,19 +1,37 @@ +locals { + volume_r_grants = flatten([ + for bucket in var.volume_buckets : [ + for principal in bucket.volume_r_grant_principals : { + bucket_name = bucket.bucket_name + principal = principal + } + ] + ]) + + volume_rw_grants = flatten([ + for bucket in var.volume_buckets : [ + for principal in bucket.volume_rw_grant_principals : { + bucket_name = bucket.bucket_name + principal = principal + } + ] + ]) +} + +# Read-only access grants resource "databricks_grant" "volume_r" { - for_each = { - for bucket in var.volume_buckets : "${bucket.bucket_name}_${bucket.volume_r_grant_principals}" => bucket - if length(bucket.volume_r_grant_principals) > 0 - } - volume = each.value.bucket_name - principal = each.value.volume_r_grant_principals[count.index] + for_each = { for idx, grant in local.volume_r_grants : "${grant.bucket_name}-${grant.principal}" => grant } + + volume = each.value.bucket_name + principal = each.value.principal privileges = ["READ_VOLUME"] } +# Read/write access grants resource "databricks_grant" "volume_rw" { - for_each = { - for bucket in var.volume_buckets : "${bucket.bucket_name}_${bucket.volume_rw_grant_principals}" => bucket - if length(bucket.volume_rw_grant_principals) > 0 - } - volume = each.value.bucket_name - principal = each.value.volume_rw_grant_principals[count.index] + for_each = { for idx, grant in local.volume_rw_grants : "${grant.bucket_name}-${grant.principal}" => grant } + + volume = each.value.bucket_name + principal = each.value.principal privileges = ["READ_VOLUME", "WRITE_VOLUME"] -} +} \ No newline at end of file From 81528221e223f503193006fb8047efba76fd9ffb Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 17:10:45 -0700 Subject: [PATCH 16/28] bucket name --- databricks-s3-volume-existing-catalog/iam.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/iam.tf b/databricks-s3-volume-existing-catalog/iam.tf index a1e0b2ae..255f8b5d 100644 --- a/databricks-s3-volume-existing-catalog/iam.tf +++ b/databricks-s3-volume-existing-catalog/iam.tf @@ -53,7 +53,7 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { "s3:PutLifecycleConfiguration" ] resources = [ - for bucket in var.volume_buckets : "arn:aws:s3:::${bucket}" + for bucket in var.volume_buckets : "arn:aws:s3:::${bucket.bucket_name}" ] } statement { @@ -65,7 +65,7 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" { "s3:DeleteObject", ] resources = [ - for bucket in var.volume_buckets : "arn:aws:s3:::${bucket}/*" + for bucket in var.volume_buckets : "arn:aws:s3:::${bucket.bucket_name}/*" ] } statement { From 30574e956bc4dcafe61b30ca0524968642ef0def Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 17:20:47 -0700 Subject: [PATCH 17/28] for_each --- databricks-s3-volume-existing-catalog/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index 19129191..abf99802 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -31,7 +31,7 @@ resource "time_sleep" "wait_30_seconds" { } resource "databricks_external_location" "volume" { - for_each = var.volume_buckets + for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } depends_on = [time_sleep.wait_30_seconds] name = "${each.value.bucket_name}-external-location" @@ -42,7 +42,7 @@ resource "databricks_external_location" "volume" { # New volume resource "databricks_volume" "volume" { - for_each = var.volume_buckets + for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } depends_on = [databricks_external_location.volume] name = each.value.bucket_name catalog_name = var.catalog_name From 58e067071cc22e408496e7c5a20e281c7a2aecc5 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Thu, 26 Sep 2024 18:07:11 -0700 Subject: [PATCH 18/28] name of storage cred --- databricks-s3-volume-existing-catalog/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index abf99802..79415e3b 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -16,7 +16,7 @@ resource "databricks_storage_credential" "volume" { resource.aws_iam_role_policy_attachment.volume_dbx_unity_aws_access ] - name = "${var.catalog_name}-volumes-role" + name = "${var.catalog_name}-volumes-storage-credential" aws_iam_role { role_arn = aws_iam_role.volume_dbx_unity_aws_role.arn } From 69c69e84be98513c96b44d8a3bcc815442d1d48a Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 11:45:11 -0700 Subject: [PATCH 19/28] Depends_on --- databricks-s3-volume-existing-catalog/grants.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index 915173a2..cad2d5bb 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -25,6 +25,8 @@ resource "databricks_grant" "volume_r" { volume = each.value.bucket_name principal = each.value.principal privileges = ["READ_VOLUME"] + + depends_on = [databricks_volume.volume] } # Read/write access grants @@ -34,4 +36,6 @@ resource "databricks_grant" "volume_rw" { volume = each.value.bucket_name principal = each.value.principal privileges = ["READ_VOLUME", "WRITE_VOLUME"] + + depends_on = [databricks_volume.volume] } \ No newline at end of file From fc6a32151b15a0b41abddfdf7440ac24d5ba5a5f Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 11:52:52 -0700 Subject: [PATCH 20/28] remove schema --- databricks-s3-volume-existing-catalog/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index 79415e3b..f888dbca 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -48,7 +48,7 @@ resource "databricks_volume" "volume" { catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${each.value.bucket_name}/${var.schema_name}" + storage_location = "s3://${each.value.bucket_name}" owner = var.catalog_owner comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" } From 3a6177618336b8348e49e77634b7bf965b788fe3 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 12:15:36 -0700 Subject: [PATCH 21/28] add option for bucket prefix --- databricks-s3-volume-existing-catalog/main.tf | 2 +- databricks-s3-volume-existing-catalog/variables.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index f888dbca..d8fb3a50 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -48,7 +48,7 @@ resource "databricks_volume" "volume" { catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${each.value.bucket_name}" + storage_location = "s3://${each.value.bucket_name}${each.value.bucket_prefix}" owner = var.catalog_owner comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" } diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index 5933f695..e0b4d72b 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -17,6 +17,7 @@ variable "volume_buckets" { description = "List of external buckets and their corresponding groups that should have r/rw access to it" type = list(object({ bucket_name : string + bucket_prefix: optional(string, "") volume_r_grant_principals: optional(list(string), []) volume_rw_grant_principals: optional(list(string), []) })) From f62877f4851efad18ce4300137501998e51bbc9f Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 12:18:28 -0700 Subject: [PATCH 22/28] include slash --- databricks-s3-volume-existing-catalog/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index d8fb3a50..a074144e 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -48,7 +48,7 @@ resource "databricks_volume" "volume" { catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${each.value.bucket_name}${each.value.bucket_prefix}" + storage_location = "s3://${each.value.bucket_name}${each.value.bucket_prefix != "" ? "/${each.value.bucket_prefix}" : ""}" owner = var.catalog_owner comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" } From 354f17e669583b0720c99f787803d21f1512fa6e Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 12:23:18 -0700 Subject: [PATCH 23/28] remove slashes --- databricks-s3-volume-existing-catalog/grants.tf | 4 ++-- databricks-s3-volume-existing-catalog/main.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index cad2d5bb..6e77246f 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -22,7 +22,7 @@ locals { resource "databricks_grant" "volume_r" { for_each = { for idx, grant in local.volume_r_grants : "${grant.bucket_name}-${grant.principal}" => grant } - volume = each.value.bucket_name + volume = replace(each.value.bucket_name, "-", "_") principal = each.value.principal privileges = ["READ_VOLUME"] @@ -33,7 +33,7 @@ resource "databricks_grant" "volume_r" { resource "databricks_grant" "volume_rw" { for_each = { for idx, grant in local.volume_rw_grants : "${grant.bucket_name}-${grant.principal}" => grant } - volume = each.value.bucket_name + volume = replace(each.value.bucket_name, "-", "_") principal = each.value.principal privileges = ["READ_VOLUME", "WRITE_VOLUME"] diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index a074144e..6502b4fe 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -44,7 +44,7 @@ resource "databricks_external_location" "volume" { resource "databricks_volume" "volume" { for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } depends_on = [databricks_external_location.volume] - name = each.value.bucket_name + name = replace(each.value.bucket_name, "-", "_") catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" From bccc94cc27e038a200bddecdad1530c76342ba0b Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 15:11:58 -0700 Subject: [PATCH 24/28] fix --- databricks-s3-volume-existing-catalog/grants.tf | 6 +++--- databricks-s3-volume-existing-catalog/main.tf | 4 ++-- databricks-s3-volume-existing-catalog/variables.tf | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index 6e77246f..d9a2f42a 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -22,7 +22,7 @@ locals { resource "databricks_grant" "volume_r" { for_each = { for idx, grant in local.volume_r_grants : "${grant.bucket_name}-${grant.principal}" => grant } - volume = replace(each.value.bucket_name, "-", "_") + volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal privileges = ["READ_VOLUME"] @@ -33,9 +33,9 @@ resource "databricks_grant" "volume_r" { resource "databricks_grant" "volume_rw" { for_each = { for idx, grant in local.volume_rw_grants : "${grant.bucket_name}-${grant.principal}" => grant } - volume = replace(each.value.bucket_name, "-", "_") + volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal privileges = ["READ_VOLUME", "WRITE_VOLUME"] depends_on = [databricks_volume.volume] -} \ No newline at end of file +} diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index 6502b4fe..1ece8122 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -34,7 +34,7 @@ resource "databricks_external_location" "volume" { for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } depends_on = [time_sleep.wait_30_seconds] - name = "${each.value.bucket_name}-external-location" + name = "${each.value.volume_name}-external-location" url = "s3://${each.value.bucket_name}" credential_name = databricks_storage_credential.volume.name comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" @@ -44,7 +44,7 @@ resource "databricks_external_location" "volume" { resource "databricks_volume" "volume" { for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } depends_on = [databricks_external_location.volume] - name = replace(each.value.bucket_name, "-", "_") + name = each.value.volume_name catalog_name = var.catalog_name schema_name = var.schema_name volume_type = "EXTERNAL" diff --git a/databricks-s3-volume-existing-catalog/variables.tf b/databricks-s3-volume-existing-catalog/variables.tf index e0b4d72b..5d266f55 100644 --- a/databricks-s3-volume-existing-catalog/variables.tf +++ b/databricks-s3-volume-existing-catalog/variables.tf @@ -16,6 +16,7 @@ variable "schema_name" { variable "volume_buckets" { description = "List of external buckets and their corresponding groups that should have r/rw access to it" type = list(object({ + volume_name : string bucket_name : string bucket_prefix: optional(string, "") volume_r_grant_principals: optional(list(string), []) From 502739b3b5c515852e21e915a4b3941c9b38f50b Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 15:27:50 -0700 Subject: [PATCH 25/28] fix key --- databricks-s3-volume-existing-catalog/grants.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index d9a2f42a..3ed13016 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -22,7 +22,7 @@ locals { resource "databricks_grant" "volume_r" { for_each = { for idx, grant in local.volume_r_grants : "${grant.bucket_name}-${grant.principal}" => grant } - volume = databricks_volume.volume[each.value.volume_name].id + volume = databricks_volume.volume[each.value.bucket_name].id principal = each.value.principal privileges = ["READ_VOLUME"] @@ -33,7 +33,7 @@ resource "databricks_grant" "volume_r" { resource "databricks_grant" "volume_rw" { for_each = { for idx, grant in local.volume_rw_grants : "${grant.bucket_name}-${grant.principal}" => grant } - volume = databricks_volume.volume[each.value.volume_name].id + volume = databricks_volume.volume[each.value.bucket_name].id principal = each.value.principal privileges = ["READ_VOLUME", "WRITE_VOLUME"] From aebd314cebefcbabffdf6351e5185afef0e9ceea Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 15:29:14 -0700 Subject: [PATCH 26/28] change key --- databricks-s3-volume-existing-catalog/grants.tf | 8 ++++---- databricks-s3-volume-existing-catalog/main.tf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index 3ed13016..6d1cf6d7 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -20,9 +20,9 @@ locals { # Read-only access grants resource "databricks_grant" "volume_r" { - for_each = { for idx, grant in local.volume_r_grants : "${grant.bucket_name}-${grant.principal}" => grant } + for_each = { for idx, grant in local.volume_r_grants : "${grant.volume_name}-${grant.principal}" => grant } - volume = databricks_volume.volume[each.value.bucket_name].id + volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal privileges = ["READ_VOLUME"] @@ -31,9 +31,9 @@ resource "databricks_grant" "volume_r" { # Read/write access grants resource "databricks_grant" "volume_rw" { - for_each = { for idx, grant in local.volume_rw_grants : "${grant.bucket_name}-${grant.principal}" => grant } + for_each = { for idx, grant in local.volume_rw_grants : "${grant.volume_name}-${grant.principal}" => grant } - volume = databricks_volume.volume[each.value.bucket_name].id + volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal privileges = ["READ_VOLUME", "WRITE_VOLUME"] diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index 1ece8122..ebf0b1d9 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -31,7 +31,7 @@ resource "time_sleep" "wait_30_seconds" { } resource "databricks_external_location" "volume" { - for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } + for_each = { for bucket in var.volume_buckets : bucket.volume_name => bucket } depends_on = [time_sleep.wait_30_seconds] name = "${each.value.volume_name}-external-location" @@ -42,7 +42,7 @@ resource "databricks_external_location" "volume" { # New volume resource "databricks_volume" "volume" { - for_each = { for bucket in var.volume_buckets : bucket.bucket_name => bucket } + for_each = { for bucket in var.volume_buckets : bucket.volume_name => bucket } depends_on = [databricks_external_location.volume] name = each.value.volume_name catalog_name = var.catalog_name From 4088521e063ee08b45e94db96e3f40cc65c14cb0 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Fri, 27 Sep 2024 15:37:55 -0700 Subject: [PATCH 27/28] fix --- databricks-s3-volume-existing-catalog/grants.tf | 4 ++-- databricks-s3-volume-existing-catalog/main.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index 6d1cf6d7..608c3a81 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -20,7 +20,7 @@ locals { # Read-only access grants resource "databricks_grant" "volume_r" { - for_each = { for idx, grant in local.volume_r_grants : "${grant.volume_name}-${grant.principal}" => grant } + for_each = { for grant in local.volume_rw_grants : grant.volume_name => grant } volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal @@ -31,7 +31,7 @@ resource "databricks_grant" "volume_r" { # Read/write access grants resource "databricks_grant" "volume_rw" { - for_each = { for idx, grant in local.volume_rw_grants : "${grant.volume_name}-${grant.principal}" => grant } + for_each = { for grant in local.volume_rw_grants : grant.volume_name => grant } volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal diff --git a/databricks-s3-volume-existing-catalog/main.tf b/databricks-s3-volume-existing-catalog/main.tf index ebf0b1d9..00f5588e 100644 --- a/databricks-s3-volume-existing-catalog/main.tf +++ b/databricks-s3-volume-existing-catalog/main.tf @@ -50,5 +50,5 @@ resource "databricks_volume" "volume" { volume_type = "EXTERNAL" storage_location = "s3://${each.value.bucket_name}${each.value.bucket_prefix != "" ? "/${each.value.bucket_prefix}" : ""}" owner = var.catalog_owner - comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" + comment = "Managed by Terraform - access for the volume named ${each.value.bucket_name} in ${var.catalog_name}" } From d685b715777e44c1ad8f3d08a7cf5ff5af6b9e01 Mon Sep 17 00:00:00 2001 From: Allison Doami Date: Tue, 1 Oct 2024 09:33:35 -0700 Subject: [PATCH 28/28] Read grants --- databricks-s3-volume-existing-catalog/grants.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks-s3-volume-existing-catalog/grants.tf b/databricks-s3-volume-existing-catalog/grants.tf index 608c3a81..f3d23fed 100644 --- a/databricks-s3-volume-existing-catalog/grants.tf +++ b/databricks-s3-volume-existing-catalog/grants.tf @@ -20,7 +20,7 @@ locals { # Read-only access grants resource "databricks_grant" "volume_r" { - for_each = { for grant in local.volume_rw_grants : grant.volume_name => grant } + for_each = { for grant in local.volume_r_grants : grant.volume_name => grant } volume = databricks_volume.volume[each.value.volume_name].id principal = each.value.principal