From 10da9a7d09344e1548b7472fbb08952d7db3ce62 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 10:50:22 -0800 Subject: [PATCH 01/11] try to make terraform use existing network --- contrib/terraform/openstack/modules/network/main.tf | 10 ++++------ contrib/terraform/openstack/modules/network/outputs.tf | 2 +- inventory/kubejetstream/cluster.tfvars | 6 ++++++ inventory/kubejetstream/terraform_apply.sh | 2 +- inventory/kubejetstream/terraform_destroy.sh | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/contrib/terraform/openstack/modules/network/main.tf b/contrib/terraform/openstack/modules/network/main.tf index 6448cffcdaa..2ec9067da2e 100644 --- a/contrib/terraform/openstack/modules/network/main.tf +++ b/contrib/terraform/openstack/modules/network/main.tf @@ -10,18 +10,16 @@ data "openstack_networking_router_v2" "k8s" { count = var.use_neutron == 1 && var.router_id != null ? 1 : 0 } -resource "openstack_networking_network_v2" "k8s" { +data "openstack_networking_network_v2" "k8s" { name = var.network_name - count = var.use_neutron - dns_domain = var.network_dns_domain != null ? var.network_dns_domain : null - admin_state_up = "true" - port_security_enabled = var.port_security_enabled + count = 1 + } resource "openstack_networking_subnet_v2" "k8s" { name = "${var.cluster_name}-internal-network" count = var.use_neutron - network_id = openstack_networking_network_v2.k8s[count.index].id + network_id = data.openstack_networking_network_v2.k8s[count.index].id subnetpool_id = var.subnetpool_id ip_version = 4 dns_nameservers = var.dns_nameservers diff --git a/contrib/terraform/openstack/modules/network/outputs.tf b/contrib/terraform/openstack/modules/network/outputs.tf index 0e8a5004f33..6b7a6a58395 100644 --- a/contrib/terraform/openstack/modules/network/outputs.tf +++ b/contrib/terraform/openstack/modules/network/outputs.tf @@ -3,7 +3,7 @@ output "router_id" { } output "network_id" { - value = element(concat(openstack_networking_network_v2.k8s.*.id, [""]),0) + value = element(concat(data.openstack_networking_network_v2.k8s.*.id, [""]),0) } output "router_internal_port_id" { diff --git a/inventory/kubejetstream/cluster.tfvars b/inventory/kubejetstream/cluster.tfvars index 7783d979178..0d0aad62b83 100644 --- a/inventory/kubejetstream/cluster.tfvars +++ b/inventory/kubejetstream/cluster.tfvars @@ -77,6 +77,12 @@ use_access_ip = 0 # openstack router list, find the ID (first column) of the `auto_allocated_router` router_id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +# networking +network_name = "auto_allocated_network" + +# Use a existing network with the name of network_name. Set to false to create a network with name of network_name. +use_existing_network = true + # Default subnetpool ID for JetStream2; Let neutron (openstack) do the CIDR # book-keeping for you subnetpool_id = "be988956-1bfb-4828-b511-a58229fbd4ac" diff --git a/inventory/kubejetstream/terraform_apply.sh b/inventory/kubejetstream/terraform_apply.sh index 2b0d1663fce..82cf3b68bf1 100644 --- a/inventory/kubejetstream/terraform_apply.sh +++ b/inventory/kubejetstream/terraform_apply.sh @@ -1 +1 @@ -terraform apply -auto-approve -var-file=cluster.tfvars -var="cluster_name=$CLUSTER" -var="network_name=${CLUSTER}-network" ../../contrib/terraform/openstack +terraform apply -var-file=cluster.tfvars -var="cluster_name=$CLUSTER" ../../contrib/terraform/openstack diff --git a/inventory/kubejetstream/terraform_destroy.sh b/inventory/kubejetstream/terraform_destroy.sh index 15f0037b4e1..baf3c7ddbe1 100644 --- a/inventory/kubejetstream/terraform_destroy.sh +++ b/inventory/kubejetstream/terraform_destroy.sh @@ -1 +1 @@ -terraform destroy -var-file=cluster.tfvars -var="cluster_name=$CLUSTER" -var="network_name=${CLUSTER}-network" ../../contrib/terraform/openstack +terraform destroy -var-file=cluster.tfvars -var="cluster_name=$CLUSTER" ../../contrib/terraform/openstack From ee21e660d0bed7ab2a2945383111c4c6c63b5f93 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 10:50:42 -0800 Subject: [PATCH 02/11] test network auto for master node --- contrib/terraform/openstack/modules/compute/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 6c2b8ff186f..67f2b340e6a 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -228,7 +228,7 @@ locals { "image_id" = node.image_id != null ? node.image_id : local.image_to_use_master, "volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb, "volume_type" = node.volume_type != null ? node.volume_type : var.master_volume_type, - "network_id" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id) + "network_mode" = "auto" } } } From 5e0a2cd0875b16448792858173bbec20c151dc4d Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Fri, 24 Feb 2023 09:55:05 -0800 Subject: [PATCH 03/11] install only nginx --- k8s_install_nginx.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 k8s_install_nginx.sh diff --git a/k8s_install_nginx.sh b/k8s_install_nginx.sh new file mode 100644 index 00000000000..279128c06f8 --- /dev/null +++ b/k8s_install_nginx.sh @@ -0,0 +1 @@ +ansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml -b -v --limit "${CLUSTER}*" --tags apps,ingress-nginx,ingress-controller From 67459f2391872d68720bdc0dec7d547bb4c9898e Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 15:20:18 -0800 Subject: [PATCH 04/11] Revert "test network auto for master node" This reverts commit ee21e660d0bed7ab2a2945383111c4c6c63b5f93. --- contrib/terraform/openstack/modules/compute/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 67f2b340e6a..6c2b8ff186f 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -228,7 +228,7 @@ locals { "image_id" = node.image_id != null ? node.image_id : local.image_to_use_master, "volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb, "volume_type" = node.volume_type != null ? node.volume_type : var.master_volume_type, - "network_mode" = "auto" + "network_id" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id) } } } From 4c5ecaab379e22424d0b8c0662ca3078dc43d21d Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 16:44:18 -0800 Subject: [PATCH 05/11] use auto allocated network, bypass port creation. Just for master --- contrib/terraform/openstack/modules/compute/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 6c2b8ff186f..0333c88e3e0 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -336,7 +336,7 @@ resource "openstack_compute_instance_v2" "k8s_master" { } network { - port = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) + name = "auto_allocated_network" } dynamic "scheduler_hints" { @@ -940,10 +940,11 @@ resource "openstack_networking_floatingip_associate_v2" "bastion" { } -resource "openstack_networking_floatingip_associate_v2" "k8s_master" { +resource "openstack_compute_floatingip_associate_v2" "k8s_master" { count = var.number_of_k8s_masters floating_ip = var.k8s_master_fips[count.index] - port_id = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) + instance_id = element(openstack_compute_instance_v2.k8s_master.*.id, count.index) + wait_until_associated = true } resource "openstack_networking_floatingip_associate_v2" "k8s_masters" { From 6893da99fcd1068cff4e6e537d32c1379727ff83 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 16:46:44 -0800 Subject: [PATCH 06/11] for k8s_node: use auto allocated network, bypass port creation --- contrib/terraform/openstack/modules/compute/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 0333c88e3e0..4a558dcc338 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -723,7 +723,7 @@ resource "openstack_compute_instance_v2" "k8s_node" { } network { - port = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index) + name = "auto_allocated_network" } @@ -959,10 +959,11 @@ resource "openstack_networking_floatingip_associate_v2" "k8s_master_no_etcd" { port_id = element(openstack_networking_port_v2.k8s_master_no_etcd_port.*.id, count.index) } -resource "openstack_networking_floatingip_associate_v2" "k8s_node" { +resource "openstack_compute_floatingip_associate_v2" "k8s_node" { count = var.node_root_volume_size_in_gb == 0 ? var.number_of_k8s_nodes : 0 floating_ip = var.k8s_node_fips[count.index] - port_id = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index) + instance_id = element(openstack_compute_instance_v2.k8s_node.*.id, count.index) + wait_until_associated = true } resource "openstack_networking_floatingip_associate_v2" "k8s_nodes" { From 4f9a3eeb8a18d7afba9bd5c87de31ec2b7f58868 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 17:07:06 -0800 Subject: [PATCH 07/11] associate security groups to instance --- contrib/terraform/openstack/modules/compute/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 4a558dcc338..67c35f5fd9a 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -317,6 +317,7 @@ resource "openstack_compute_instance_v2" "k8s_master" { flavor_id = var.flavor_k8s_master key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered + security_groups = var.port_security_enabled ? local.master_sec_groups : null lifecycle { ignore_changes = [ image_id ] From 2de9c3992740164878e24f44e33b77602bd70f55 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 17:08:13 -0800 Subject: [PATCH 08/11] Revert "associate security groups to instance" This reverts commit 43d8d9fdd385124c1fb195f4e2866d46bcea45b8. --- contrib/terraform/openstack/modules/compute/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 67c35f5fd9a..4a558dcc338 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -317,7 +317,6 @@ resource "openstack_compute_instance_v2" "k8s_master" { flavor_id = var.flavor_k8s_master key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered - security_groups = var.port_security_enabled ? local.master_sec_groups : null lifecycle { ignore_changes = [ image_id ] From 16767b6dc6252ace555016cf5b63c7e1febac7df Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 17:08:19 -0800 Subject: [PATCH 09/11] Revert "for k8s_node: use auto allocated network, bypass port creation" This reverts commit ecfd3828314bc4c96b3b857e75f99e90f2bf58f1. --- contrib/terraform/openstack/modules/compute/main.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 4a558dcc338..0333c88e3e0 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -723,7 +723,7 @@ resource "openstack_compute_instance_v2" "k8s_node" { } network { - name = "auto_allocated_network" + port = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index) } @@ -959,11 +959,10 @@ resource "openstack_networking_floatingip_associate_v2" "k8s_master_no_etcd" { port_id = element(openstack_networking_port_v2.k8s_master_no_etcd_port.*.id, count.index) } -resource "openstack_compute_floatingip_associate_v2" "k8s_node" { +resource "openstack_networking_floatingip_associate_v2" "k8s_node" { count = var.node_root_volume_size_in_gb == 0 ? var.number_of_k8s_nodes : 0 floating_ip = var.k8s_node_fips[count.index] - instance_id = element(openstack_compute_instance_v2.k8s_node.*.id, count.index) - wait_until_associated = true + port_id = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index) } resource "openstack_networking_floatingip_associate_v2" "k8s_nodes" { From 43ff2688f648581ca7c0aef0c83f4f3e68d15ba9 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 17:08:27 -0800 Subject: [PATCH 10/11] Revert "use auto allocated network, bypass port creation." This reverts commit 16f011dec5a10b982c912447425f6e30c1f81be5. --- contrib/terraform/openstack/modules/compute/main.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 0333c88e3e0..6c2b8ff186f 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -336,7 +336,7 @@ resource "openstack_compute_instance_v2" "k8s_master" { } network { - name = "auto_allocated_network" + port = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) } dynamic "scheduler_hints" { @@ -940,11 +940,10 @@ resource "openstack_networking_floatingip_associate_v2" "bastion" { } -resource "openstack_compute_floatingip_associate_v2" "k8s_master" { +resource "openstack_networking_floatingip_associate_v2" "k8s_master" { count = var.number_of_k8s_masters floating_ip = var.k8s_master_fips[count.index] - instance_id = element(openstack_compute_instance_v2.k8s_master.*.id, count.index) - wait_until_associated = true + port_id = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) } resource "openstack_networking_floatingip_associate_v2" "k8s_masters" { From 309685301908520e6767910952ffaff3d18ab4a2 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Wed, 1 Mar 2023 17:48:54 -0800 Subject: [PATCH 11/11] now dns unnecessary --- inventory/kubejetstream/cluster.tfvars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/kubejetstream/cluster.tfvars b/inventory/kubejetstream/cluster.tfvars index 0d0aad62b83..ea11188df23 100644 --- a/inventory/kubejetstream/cluster.tfvars +++ b/inventory/kubejetstream/cluster.tfvars @@ -70,8 +70,8 @@ k8s_allowed_remote_ips = ["0.0.0.0/0"] # have Kubernetes traffic use the internal IP use_access_ip = 0 -# Uncomment below and edit to set dns-domain network property (the initial tg- is not necessary anymore) -# network_dns_domain = "xxxxxxxxx.projects.jetstream-cloud.org." +# This is now un-necessary because we use the auto_allocated_network which is preconfigured +# network_dns_domain = "xxx000000.projects.jetstream-cloud.org." # Reuse the auto allocated router, we do not want to waste floating IPs by having un-necessary routers # openstack router list, find the ID (first column) of the `auto_allocated_router`