feat(ippool): add serverIP, start, and end fields if missing

If IPPool object is created without explicitly specified the serverIP, the start and end IP of the pool, the IPPool mutator will try to assign IPs to them. Also, if the serverIP or router is within the pool range, i.e., in between the start and end IPs, those IPs will then be marked as "RESERVED" (by the controller) and will not be allocatable. Signed-off-by: Zespre Chang <[email protected]>
harvester · Feb 26, 2024 · 8a91b61 · 8a91b61
1 parent e401ece
commit 8a91b61
Show file tree

Hide file tree

Showing 8 changed files with 513 additions and 18 deletions.
diff --git a/cmd/webhook/run.go b/cmd/webhook/run.go
@@ -75,6 +75,12 @@ func run(ctx context.Context, cfg *rest.Config, options *config.Options) error {
 		return err
 	}
 
+	if err := webhookServer.RegisterMutators(
+		ippool.NewMutator(),
+	); err != nil {
+		return err
+	}
+
 	if err := webhookServer.Start(); err != nil {
 		return err
 	}

diff --git a/pkg/agent/ippool/ippool.go b/pkg/agent/ippool/ippool.go
@@ -17,7 +17,7 @@ func (c *Controller) Update(ipPool *networkv1.IPPool) error {
 		return nil
 	}
 	allocated := ipPool.Status.IPv4.Allocated
-	filterExcluded(allocated)
+	filterExcludedAndReserved(allocated)
 	return c.updatePoolCacheAndLeaseStore(allocated, ipPool.Spec.IPv4Config)
 }
 
@@ -62,9 +62,9 @@ func (c *Controller) updatePoolCacheAndLeaseStore(latest map[string]string, ipv4
 	return nil
 }
 
-func filterExcluded(allocated map[string]string) {
+func filterExcludedAndReserved(allocated map[string]string) {
 	for ip, mac := range allocated {
-		if mac == util.ExcludedMark {
+		if mac == util.ExcludedMark || mac == util.ReservedMark {
 			delete(allocated, ip)
 		}
 	}

diff --git a/pkg/controller/ippool/controller.go b/pkg/controller/ippool/controller.go
@@ -217,6 +217,12 @@ func (h *Handler) OnChange(key string, ipPool *networkv1.IPPool) (*networkv1.IPP
 	if allocated == nil {
 		allocated = make(map[string]string)
 	}
+	if util.IsIPInBetweenOf(ipPool.Spec.IPv4Config.ServerIP, ipPool.Spec.IPv4Config.Pool.Start, ipPool.Spec.IPv4Config.Pool.End) {
+		allocated[ipPool.Spec.IPv4Config.ServerIP] = util.ReservedMark
+	}
+	if util.IsIPInBetweenOf(ipPool.Spec.IPv4Config.Router, ipPool.Spec.IPv4Config.Pool.Start, ipPool.Spec.IPv4Config.Pool.End) {
+		allocated[ipPool.Spec.IPv4Config.Router] = util.ReservedMark
+	}
 	for _, eIP := range ipPool.Spec.IPv4Config.Pool.Exclude {
 		allocated[eIP] = util.ExcludedMark
 	}
@@ -364,6 +370,18 @@ func (h *Handler) BuildCache(ipPool *networkv1.IPPool, status networkv1.IPPoolSt
 		return status, err
 	}
 
+	// Revoke server IP address in IPAM
+	if err := h.ipAllocator.RevokeIP(ipPool.Spec.NetworkName, ipPool.Spec.IPv4Config.ServerIP); err != nil {
+		return status, err
+	}
+	logrus.Debugf("(ippool.BuildCache) server ip %s was revoked in ipam %s", ipPool.Spec.IPv4Config.ServerIP, ipPool.Spec.NetworkName)
+
+	// Revoke router IP address in IPAM
+	if err := h.ipAllocator.RevokeIP(ipPool.Spec.NetworkName, ipPool.Spec.IPv4Config.Router); err != nil {
+		return status, err
+	}
+	logrus.Debugf("(ippool.BuildCache) router ip %s was revoked in ipam %s", ipPool.Spec.IPv4Config.Router, ipPool.Spec.NetworkName)
+
 	// Revoke excluded IP addresses in IPAM
 	for _, eIP := range ipPool.Spec.IPv4Config.Pool.Exclude {
 		if err := h.ipAllocator.RevokeIP(ipPool.Spec.NetworkName, eIP); err != nil {

diff --git a/pkg/controller/ippool/controller_test.go b/pkg/controller/ippool/controller_test.go
@@ -31,10 +31,13 @@ const (
 	testPodName            = testNADNamespace + "-" + testNADName + "-agent"
 	testUID                = "3a955369-9eaa-43db-94f3-9153289d7dc2"
 	testClusterNetwork     = "provider"
-	testServerIP           = "192.168.0.2"
+	testServerIP1          = "192.168.0.2"
+	testServerIP2          = "192.168.0.110"
 	testNetworkName        = testNADNamespace + "/" + testNADName
 	testNetworkNameLong    = testNADNamespace + "/" + testNADNameLong
 	testCIDR               = "192.168.0.0/24"
+	testRouter1            = "192.168.0.1"
+	testRouter2            = "192.168.0.120"
 	testStartIP            = "192.168.0.101"
 	testEndIP              = "192.168.0.200"
 	testServiceAccountName = "vdca"
@@ -125,14 +128,14 @@ func TestHandler_OnChange(t *testing.T) {
 			IPSubnet(testNetworkName, testCIDR, testStartIP, testEndIP).
 			Build()
 		givenIPPool := newTestIPPoolBuilder().
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			PoolRange(testStartIP, testEndIP).
 			NetworkName(testNetworkName).
 			CacheReadyCondition(corev1.ConditionTrue, "", "").Build()
 
 		expectedIPPool := newTestIPPoolBuilder().
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			PoolRange(testStartIP, testEndIP).
 			NetworkName(testNetworkName).
@@ -268,7 +271,7 @@ func TestHandler_OnChange(t *testing.T) {
 func TestHandler_DeployAgent(t *testing.T) {
 	t.Run("ippool created", func(t *testing.T) {
 		givenIPPool := newTestIPPoolBuilder().
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			NetworkName(testNetworkName).Build()
 		givenNAD := newTestNetworkAttachmentDefinitionBuilder().
@@ -278,7 +281,7 @@ func TestHandler_DeployAgent(t *testing.T) {
 			AgentPodRef(testPodNamespace, testPodName, testImage, "").Build()
 		expectedPod, _ := prepareAgentPod(
 			NewIPPoolBuilder(testIPPoolNamespace, testIPPoolName).
-				ServerIP(testServerIP).
+				ServerIP(testServerIP1).
 				CIDR(testCIDR).
 				NetworkName(testNetworkName).Build(),
 			false,
@@ -367,15 +370,15 @@ func TestHandler_DeployAgent(t *testing.T) {
 
 	t.Run("agent pod already exists", func(t *testing.T) {
 		givenIPPool := newTestIPPoolBuilder().
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			NetworkName(testNetworkName).
 			AgentPodRef(testPodNamespace, testPodName, testImage, "").Build()
 		givenNAD := newTestNetworkAttachmentDefinitionBuilder().
 			Label(clusterNetworkLabelKey, testClusterNetwork).Build()
 		givenPod, _ := prepareAgentPod(
 			NewIPPoolBuilder(testIPPoolNamespace, testIPPoolName).
-				ServerIP(testServerIP).
+				ServerIP(testServerIP1).
 				CIDR(testCIDR).
 				NetworkName(testNetworkName).Build(),
 			false,
@@ -392,7 +395,7 @@ func TestHandler_DeployAgent(t *testing.T) {
 			AgentPodRef(testPodNamespace, testPodName, testImage, "").Build()
 		expectedPod, _ := prepareAgentPod(
 			NewIPPoolBuilder(testIPPoolNamespace, testIPPoolName).
-				ServerIP(testServerIP).
+				ServerIP(testServerIP1).
 				CIDR(testCIDR).
 				NetworkName(testNetworkName).Build(),
 			false,
@@ -442,7 +445,7 @@ func TestHandler_DeployAgent(t *testing.T) {
 
 	t.Run("very long name ippool created", func(t *testing.T) {
 		givenIPPool := NewIPPoolBuilder(testIPPoolNamespace, testIPPoolNameLong).
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			NetworkName(testNetworkNameLong).Build()
 		givenNAD := NewNetworkAttachmentDefinitionBuilder(testNADNamespace, testNADNameLong).
@@ -452,7 +455,7 @@ func TestHandler_DeployAgent(t *testing.T) {
 			AgentPodRef(testPodNamespace, testPodNameLong, testImage, "").Build()
 		expectedPod, _ := prepareAgentPod(
 			NewIPPoolBuilder(testIPPoolNamespace, testIPPoolNameLong).
-				ServerIP(testServerIP).
+				ServerIP(testServerIP1).
 				CIDR(testCIDR).
 				NetworkName(testNetworkNameLong).Build(),
 			false,
@@ -500,7 +503,7 @@ func TestHandler_DeployAgent(t *testing.T) {
 
 	t.Run("agent pod upgrade (from main to dev)", func(t *testing.T) {
 		givenIPPool := newTestIPPoolBuilder().
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			NetworkName(testNetworkName).
 			AgentPodRef(testPodNamespace, testPodName, testImage, "").Build()
@@ -615,15 +618,15 @@ func TestHandler_DeployAgent(t *testing.T) {
 
 	t.Run("existing agent pod uid mismatch", func(t *testing.T) {
 		givenIPPool := newTestIPPoolBuilder().
-			ServerIP(testServerIP).
+			ServerIP(testServerIP1).
 			CIDR(testCIDR).
 			NetworkName(testNetworkName).
 			AgentPodRef(testPodNamespace, testPodName, testImage, testUID).Build()
 		givenNAD := newTestNetworkAttachmentDefinitionBuilder().
 			Label(clusterNetworkLabelKey, testClusterNetwork).Build()
 		givenPod, _ := prepareAgentPod(
 			NewIPPoolBuilder(testIPPoolNamespace, testIPPoolName).
-				ServerIP(testServerIP).
+				ServerIP(testServerIP1).
 				CIDR(testCIDR).
 				NetworkName(testNetworkName).Build(),
 			false,

diff --git a/pkg/util/common.go b/pkg/util/common.go
@@ -8,6 +8,7 @@ import (
 
 const (
 	ExcludedMark = "EXCLUDED"
+	ReservedMark = "RESERVED"
 
 	AgentSuffixName = "agent"
 )

diff --git a/pkg/util/network.go b/pkg/util/network.go
@@ -18,7 +18,7 @@ type PoolInfo struct {
 	RouterIPAddr    netip.Addr
 }
 
-func loadCIDR(cidr string) (ipNet *net.IPNet, networkIPAddr netip.Addr, broadcastIPAddr netip.Addr, err error) {
+func LoadCIDR(cidr string) (ipNet *net.IPNet, networkIPAddr netip.Addr, broadcastIPAddr netip.Addr, err error) {
 	_, ipNet, err = net.ParseCIDR(cidr)
 	if err != nil {
 		return
@@ -45,7 +45,7 @@ func loadCIDR(cidr string) (ipNet *net.IPNet, networkIPAddr netip.Addr, broadcas
 }
 
 func LoadPool(ipPool *networkv1.IPPool) (pi PoolInfo, err error) {
-	pi.IPNet, pi.NetworkIPAddr, pi.BroadcastIPAddr, err = loadCIDR(ipPool.Spec.IPv4Config.CIDR)
+	pi.IPNet, pi.NetworkIPAddr, pi.BroadcastIPAddr, err = LoadCIDR(ipPool.Spec.IPv4Config.CIDR)
 	if err != nil {
 		return
 	}
@@ -91,3 +91,29 @@ func LoadAllocated(allocated map[string]string) (ipAddrList []netip.Addr) {
 	}
 	return
 }
+
+func IsIPAddrInList(ipAddr netip.Addr, ipAddrList []netip.Addr) bool {
+	for i := range ipAddrList {
+		if ipAddr == ipAddrList[i] {
+			return true
+		}
+	}
+	return false
+}
+
+func IsIPInBetweenOf(ip, ip1, ip2 string) bool {
+	ipAddr, err := netip.ParseAddr(ip)
+	if err != nil {
+		return false
+	}
+	ip1Addr, err := netip.ParseAddr(ip1)
+	if err != nil {
+		return false
+	}
+	ip2Addr, err := netip.ParseAddr(ip2)
+	if err != nil {
+		return false
+	}
+
+	return ipAddr.Compare(ip1Addr) >= 0 && ipAddr.Compare(ip2Addr) <= 0
+}