Skip to content

Commit

Permalink
Fix variabler to support existing projects.
Browse files Browse the repository at this point in the history
  • Loading branch information
@lcaggio
lcaggio committed Feb 19, 2023
1 parent ad6667a commit c8f2551
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 115 deletions.
34 changes: 17 additions & 17 deletions blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,37 +15,37 @@
# tfdoc:file:description drop off project and resources.

locals {
drop_orch_service_accounts = [
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
]
}

module "drop-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
group_iam = {
group_iam_drp = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/pubsub.editor",
"roles/storage.admin",
]
}
iam = {
iam_drp = {
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = concat(
local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
)
"roles/pubsub.subscriber" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
}
}

module "drop-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
# group_iam = local.group_iam_drp
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down
43 changes: 23 additions & 20 deletions blueprints/data-solutions/data-platform-foundations/02-load.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,22 @@
# tfdoc:file:description Load project and VPC.

locals {
group_iam_load = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam_load = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
load_service_accounts = [
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email
Expand All @@ -35,27 +51,14 @@ locals {

module "load-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
# group_iam = local.group_iam_load
iam = var.project_config.billing_account_id != null ? local.iam_load : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down Expand Up @@ -108,11 +111,11 @@ module "load-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-lod"
region = var.region
}
]
Expand All @@ -132,7 +135,7 @@ module "load-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
region = var.region
router_network = module.load-vpc.0.name
}
63 changes: 33 additions & 30 deletions blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,7 @@
# tfdoc:file:description Orchestration project and VPC.

locals {
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)

# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}

module "orch-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
group_iam = {
group_iam_orch = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
Expand All @@ -54,7 +30,7 @@ module "orch-project" {
"roles/serviceusage.serviceUsageConsumer",
]
}
iam = {
iam_orch = {
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
Expand Down Expand Up @@ -85,7 +61,34 @@ module "orch-project" {
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
oslogin = false
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)

# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}

module "orch-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
# group_iam = local.group_iam_orch
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
oslogin = false
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
Expand Down Expand Up @@ -133,11 +136,11 @@ module "orch-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-orch"
region = var.region
secondary_ip_ranges = {
pods = "10.10.8.0/22"
Expand All @@ -161,7 +164,7 @@ module "orch-nat" {
count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
region = var.region
router_network = module.orch-vpc.0.name
}
Expand Down
51 changes: 27 additions & 24 deletions blueprints/data-solutions/data-platform-foundations/04-transformation.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,13 @@
# tfdoc:file:description Trasformation project and VPC.

locals {
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.transf-vpc.0.subnet_self_links)[0]
)
transf_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.transf-vpc.0.self_link
)
}

module "transf-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
group_iam = {
group_iam_trf = {
(local.groups.data-engineers) = [
"roles/bigquery.jobUser",
"roles/dataflow.admin",
]
}
iam = {
iam_trf = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
]
Expand All @@ -55,6 +36,28 @@ module "transf-project" {
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.transf-vpc.0.subnet_self_links)[0]
)
transf_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.transf-vpc.0.self_link
)
}

module "transf-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
# group_iam = local.group_iam_trf
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down Expand Up @@ -132,11 +135,11 @@ module "transf-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-trf"
region = var.region
}
]
Expand All @@ -156,7 +159,7 @@ module "transf-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
region = var.region
router_network = module.transf-vpc.0.name
}
29 changes: 16 additions & 13 deletions blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,15 @@ locals {

module "dwh-lnd-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}"
group_iam = local.dwh_group_iam
iam = local.dwh_lnd_iam
services = local.dwh_services
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
Expand All @@ -98,14 +99,15 @@ module "dwh-lnd-project" {

module "dwh-cur-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}"
group_iam = local.dwh_group_iam
iam = local.dwh_iam
services = local.dwh_services
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
Expand All @@ -114,14 +116,15 @@ module "dwh-cur-project" {

module "dwh-conf-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}"
group_iam = local.dwh_group_iam
iam = local.dwh_iam
services = local.dwh_services
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
Expand All @@ -141,7 +144,7 @@ module "dwh-lnd-bq-0" {
module "dwh-cur-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dwh-cur-project.project_id
id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0"
id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
Expand Down
Loading

0 comments on commit c8f2551

Please sign in to comment.