Changing CT elements causes ip link addressing churn

[chrismarget-j]

Reproduce:

• use the following terraform config
• change bfd_enabled from false to true
• note that the generic system interface addressing has changed from 5.5.5.5/30 to 5.5.5.5/31

Possible cause:

• ip_links and bgp_peering_generic_systems elements are sets
• any change to a set member causes that member to be discarded/replaced
• individual policy elements (primitives) within the CT get replaced, wind up with new IDs
• Apstra replaces the link
• In replacing the link, Apstra attempts renumbering

terraform {
  required_providers {
    apstra = {
      source  = "Juniper/apstra"
      version = "0.76.1"
    }
  }
}

provider "apstra" {
  url                     = "xxx"
  blueprint_mutex_enabled = false
}

resource "apstra_datacenter_blueprint" "rtsto" {
  name        = "rtsto"
  template_id = "L2_Virtual_EVPN"
}

resource "apstra_datacenter_routing_zone" "rtsto" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  name         = "rtsto"
}

resource "apstra_datacenter_connectivity_template_interface" "rtsto" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  name         = "rtsto"
  ip_links = [
    {
      name                 = "ip link"
      routing_zone_id      = apstra_datacenter_routing_zone.rtsto.id
      ipv4_addressing_type = "numbered"
      ipv6_addressing_type = "none"
      bgp_peering_generic_systems = [
        {
          name                 = "bgp"
          bfd_enabled          = false
          ipv4_addressing_type = "addressed"
          ipv6_addressing_type = "none"
          neighbor_asn_dynamic = false
          peer_from_loopback   = false
          peer_to              = "interface_or_ip_endpoint"

        }
      ]
    }
  ]
}

resource "apstra_datacenter_resource_pool_allocation" "global_spine_asn" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Private-64512-65534"]
  role         = "spine_asns"
}

resource "apstra_datacenter_resource_pool_allocation" "global_leaf_asn" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Private-64512-65534"]
  role         = "leaf_asns"
}

resource "apstra_datacenter_resource_pool_allocation" "global_generic_asn" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Private-64512-65534"]
  role         = "generic_asns"
}

resource "apstra_datacenter_resource_pool_allocation" "global_spine_loopback" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Private-10_0_0_0-8"]
  role         = "spine_loopback_ips"
}

resource "apstra_datacenter_resource_pool_allocation" "global_leaf_loopback" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Private-10_0_0_0-8"]
  role         = "leaf_loopback_ips"
}

resource "apstra_datacenter_resource_pool_allocation" "global_spine_leaf" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Private-10_0_0_0-8"]
  role         = "spine_leaf_link_ips"
}

resource "apstra_datacenter_resource_pool_allocation" "rz_leaf_loopback" {
  blueprint_id    = apstra_datacenter_blueprint.rtsto.id
  pool_ids        = ["Private-10_0_0_0-8"]
  role            = "leaf_loopback_ips"
  routing_zone_id = apstra_datacenter_routing_zone.rtsto.id
}

# resource "apstra_datacenter_resource_pool_allocation" "rz_leaf_generic" {
#   blueprint_id    = apstra_datacenter_blueprint.rtsto.id
#   pool_ids        = ["Private-10_0_0_0-8"]
#   role            = "to_generic_link_ips"
#   routing_zone_id = apstra_datacenter_routing_zone.rtsto.id
# }


resource "apstra_datacenter_resource_pool_allocation" "vni" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  pool_ids     = ["Default-10000-20000"]
  role         = "evpn_l3_vnis"
}

data "apstra_datacenter_systems" "spines" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  filters = [
    {
      role        = "spine"
      system_type = "switch"
    }
  ]
}

data "apstra_datacenter_systems" "leafs" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  filters = [
    {
      role        = "leaf"
      system_type = "switch"
    }
  ]
}

data "apstra_datacenter_system" "spines" {
  for_each     = data.apstra_datacenter_systems.spines.ids
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  id           = each.key
}

data "apstra_datacenter_system" "leafs" {
  for_each     = data.apstra_datacenter_systems.leafs.ids
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  id           = each.key
}

resource "apstra_datacenter_device_allocation" "spines" {
  for_each                 = data.apstra_datacenter_system.spines
  blueprint_id             = apstra_datacenter_blueprint.rtsto.id
  node_name                = each.value.attributes.label
  initial_interface_map_id = "Juniper_vQFX__AOS-7x10-Spine"
}

resource "apstra_datacenter_device_allocation" "leafs" {
  for_each                 = data.apstra_datacenter_system.leafs
  blueprint_id             = apstra_datacenter_blueprint.rtsto.id
  node_name                = each.value.attributes.label
  initial_interface_map_id = "Juniper_vQFX__AOS-7x10-Leaf"
}

resource "apstra_datacenter_generic_system" "rtsto" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  name         = "rtsto"
  asn          = 100
  links = [
    {
      tags                          = ["rtsto"]
      target_switch_id              = sort(tolist(data.apstra_datacenter_systems.leafs.ids))[0]
      target_switch_if_name         = "xe-0/0/2"
      target_switch_if_transform_id = 1
    }
  ]
}

data "apstra_datacenter_interfaces_by_link_tag" "rtsto" {
  blueprint_id = apstra_datacenter_blueprint.rtsto.id
  tags         = ["rtsto"]
}

resource "apstra_datacenter_connectivity_template_assignments" "rtsto" {
  blueprint_id             = apstra_datacenter_blueprint.rtsto.id
  application_point_ids    = data.apstra_datacenter_interfaces_by_link_tag.rtsto.ids
  connectivity_template_id = apstra_datacenter_connectivity_template_interface.rtsto.id
  fetch_ip_link_ids        = true
}

resource "apstra_datacenter_ip_link_addressing" "rtsto" {
  blueprint_id        = apstra_datacenter_blueprint.rtsto.id
  link_id             = apstra_datacenter_connectivity_template_assignments.rtsto.ip_link_ids[one(data.apstra_datacenter_interfaces_by_link_tag.rtsto.ids)][0]
  switch_ipv4_address_type = "numbered"        # none | numbered
  switch_ipv4_address = "5.5.5.5/30"
  generic_ipv4_address_type = "numbered"        # none | numbered
  generic_ipv4_address = "5.5.5.6/30"
}