diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index c675b83d0..7fed28462 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -15,6 +15,7 @@ import ( v1 "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -27,12 +28,14 @@ import ( "github.com/k8snetworkplumbingwg/whereabouts/e2e/retrievers" testenv "github.com/k8snetworkplumbingwg/whereabouts/e2e/testenvironment" "github.com/k8snetworkplumbingwg/whereabouts/pkg/api/whereabouts.cni.cncf.io/v1alpha1" + "github.com/k8snetworkplumbingwg/whereabouts/pkg/iphelpers" wbstorage "github.com/k8snetworkplumbingwg/whereabouts/pkg/storage/kubernetes" "github.com/k8snetworkplumbingwg/whereabouts/pkg/types" ) const ( createPodTimeout = 10 * time.Second + ipPoolNamespace = "kube-system" ) func TestWhereaboutsE2E(t *testing.T) { @@ -86,10 +89,15 @@ var _ = Describe("Whereabouts functionality", func() { }) Context("Single pod tests", func() { - BeforeEach(func() { - const singlePodName = "whereabouts-basic-test" - var err error + const singlePodName = "whereabouts-basic-test" + var err error + AfterEach(func() { + By("deleting pod with whereabouts net-attach-def") + _ = clientInfo.DeletePod(pod) + }) + + It("allocates a single pod with a single interface", func() { By("creating a pod with whereabouts net-attach-def") pod, err = clientInfo.ProvisionPod( singlePodName, @@ -98,19 +106,55 @@ var _ = Describe("Whereabouts functionality", func() { entities.PodNetworkSelectionElements(testNetworkName), ) Expect(err).NotTo(HaveOccurred()) - }) - AfterEach(func() { - By("deleting pod with whereabouts net-attach-def") - Expect(clientInfo.DeletePod(pod)).To(Succeed()) - }) - - It("allocates a single pod within the correct IP range", func() { By("checking pod IP is within whereabouts IPAM range") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1") Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).NotTo(BeEmpty()) Expect(inRange(ipv4TestRange, secondaryIfaceIPs[0])).To(Succeed()) + + By("verifying allocation") + verifyAllocations(clientInfo, ipv4TestRange, secondaryIfaceIPs[0], testNamespace, pod.Name, "net1") + + By("deleting pod") + err = clientInfo.DeletePod(pod) + Expect(err).NotTo(HaveOccurred()) + + By("checking that the IP allocation is removed") + verifyNoAllocationsForPodRef(clientInfo, ipv4TestRange, testNamespace, pod.Name, secondaryIfaceIPs) + }) + It("allocates a single pod with multiple interfaces", func() { + By("creating a pod with whereabouts net-attach-def") + pod, err = clientInfo.ProvisionPod( + singlePodName, + testNamespace, + podTierLabel(singlePodName), + entities.PodNetworkSelectionElements(testNetworkName, testNetworkName, testNetworkName), + ) + Expect(err).NotTo(HaveOccurred()) + + By("checking pod IP is within whereabouts IPAM range") + var secondaryIPs []string + + for _, ifName := range []string{"net1", "net2", "net3"} { + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) + Expect(err).NotTo(HaveOccurred()) + Expect(secondaryIfaceIPs).NotTo(BeEmpty()) + for _, ip := range secondaryIfaceIPs { + Expect(inRange(ipv4TestRange, ip)).To(Succeed()) + + By("verifying allocation") + verifyAllocations(clientInfo, ipv4TestRange, ip, testNamespace, pod.Name, ifName) + } + secondaryIPs = append(secondaryIPs, secondaryIfaceIPs...) + } + + By("deleting pod") + err = clientInfo.DeletePod(pod) + Expect(err).NotTo(HaveOccurred()) + + By("checking that the IP allocation is removed") + verifyNoAllocationsForPodRef(clientInfo, ipv4TestRange, testNamespace, pod.Name, secondaryIPs) }) }) @@ -160,7 +204,7 @@ var _ = Describe("Whereabouts functionality", func() { It("allocates a single pod within the correct IP ranges", func() { By("checking pod IP is within whereabouts IPAM ranges") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1") Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).To(HaveLen(2)) Expect(inRange(dualStackIPv4Range, secondaryIfaceIPs[0])).To(Succeed()) @@ -202,7 +246,7 @@ var _ = Describe("Whereabouts functionality", func() { It("allocates a single pod within the correct IP ranges", func() { By("checking pod IP is within whereabouts IPAM ranges") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1") Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).To(HaveLen(3)) Expect(inRange(ipv4TestRange, secondaryIfaceIPs[0])).To(Succeed()) @@ -225,7 +269,6 @@ var _ = Describe("Whereabouts functionality", func() { By("creating a replicaset with whereabouts net-attach-def") var err error - const ipPoolNamespace = "kube-system" k8sIPAM, err = wbstorage.NewKubernetesIPAMWithNamespace("", "", types.IPAMConfig{ Kubernetes: types.KubernetesConfig{ KubeConfigPath: testConfig.KubeconfigPath, @@ -296,7 +339,6 @@ var _ = Describe("Whereabouts functionality", func() { Context("stateful set tests", func() { const ( initialReplicaNumber = 20 - ipPoolNamespace = "kube-system" namespace = "default" serviceName = "web" selector = "app=" + serviceName @@ -484,8 +526,8 @@ var _ = Describe("Whereabouts functionality", func() { Expect(err).NotTo(HaveOccurred()) Expect(ipPool.Spec.Allocations).NotTo(BeEmpty()) - Expect(allocationForPodRef(podRef, *ipPool).ContainerID).NotTo(Equal(containerID)) - Expect(allocationForPodRef(podRef, *ipPool).PodRef).To(Equal(podRef)) + Expect(allocationForPodRef(podRef, *ipPool)[0].ContainerID).NotTo(Equal(containerID)) + Expect(allocationForPodRef(podRef, *ipPool)[0].PodRef).To(Equal(podRef)) }) }) }) @@ -551,13 +593,14 @@ var _ = Describe("Whereabouts functionality", func() { }) It("allocates the correct IP address to the second pod", func() { + ifName := "net1" By("checking pod IP is within whereabouts IPAM range") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).NotTo(BeEmpty()) By("checking pod 2 IP is within whereabouts IPAM range") - secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2) + secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs2).NotTo(BeEmpty()) @@ -656,19 +699,20 @@ var _ = Describe("Whereabouts functionality", func() { }) It("allocates the same IP to the Pods as they are in different address collision domains", func() { + ifName := "net1" By("checking pod IP is within whereabouts IPAM range") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).NotTo(BeEmpty()) By("checking pod 2 IP is within whereabouts IPAM range and has the same IP as pod 1") - secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2) + secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs2).NotTo(BeEmpty()) Expect(secondaryIfaceIPs[0]).To(Equal(secondaryIfaceIPs2[0])) By("checking pod 3 IP is within whereabouts IPAM range and has a different IP from pod 2") - secondaryIfaceIPs3, err := retrievers.SecondaryIfaceIPValue(pod3) + secondaryIfaceIPs3, err := retrievers.SecondaryIfaceIPValue(pod3, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs3).NotTo(BeEmpty()) Expect(secondaryIfaceIPs2[0]).NotTo(Equal(secondaryIfaceIPs3[0])) @@ -677,13 +721,55 @@ var _ = Describe("Whereabouts functionality", func() { }) }) -func allocationForPodRef(podRef string, ipPool v1alpha1.IPPool) *v1alpha1.IPAllocation { +func verifyNoAllocationsForPodRef(clientInfo *wbtestclient.ClientInfo, ipv4TestRange, testNamespace, podName string, secondaryIfaceIPs []string) { + Eventually(func() bool { + ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: ipv4TestRange, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + allocation := allocationForPodRef(getPodRef(testNamespace, podName), *ipPool) + return len(allocation) == 0 + }, 3*time.Second, 500*time.Millisecond).Should(BeTrue()) + + for _, ip := range secondaryIfaceIPs { + Eventually(func() bool { + _, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return true + } + return false + }, 3*time.Second, 500*time.Millisecond).Should(BeTrue()) + } +} + +func verifyAllocations(clientInfo *wbtestclient.ClientInfo, ipv4TestRange, ip, testNamespace, podName, ifName string) { + ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: ipv4TestRange, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + firstIP, _, err := net.ParseCIDR(ipv4TestRange) + Expect(err).NotTo(HaveOccurred()) + offset, err := iphelpers.IPGetOffset(net.ParseIP(ip), firstIP) + Expect(err).NotTo(HaveOccurred()) + + allocation, ok := ipPool.Spec.Allocations[fmt.Sprintf("%d", offset)] + Expect(ok).To(BeTrue()) + Expect(allocation.PodRef).To(Equal(getPodRef(testNamespace, podName))) + Expect(allocation.IfName).To(Equal(ifName)) + + overlapping, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(overlapping.Spec.IfName).To(Equal(ifName)) + Expect(overlapping.Spec.PodRef).To(Equal(getPodRef(testNamespace, podName))) +} + +func allocationForPodRef(podRef string, ipPool v1alpha1.IPPool) []v1alpha1.IPAllocation { + var allocations []v1alpha1.IPAllocation for _, allocation := range ipPool.Spec.Allocations { if allocation.PodRef == podRef { - return &allocation + allocations = append(allocations, allocation) } } - return nil + return allocations } func clusterConfig() (*rest.Config, error) { @@ -760,25 +846,21 @@ func macvlanNetworkWithWhereaboutsIPAMNetwork(networkName string, namespaceName macvlanConfig := fmt.Sprintf(`{ "cniVersion": "0.3.0", "disableCheck": true, - "plugins": [ - { - "type": "macvlan", - "master": "eth0", - "mode": "bridge", - "ipam": { - "type": "whereabouts", - "leader_lease_duration": 1500, - "leader_renew_deadline": 1000, - "leader_retry_period": 500, - "range": "%s", - "ipRanges": %s, - "log_level": "debug", - "log_file": "/tmp/wb", - "network_name": "%s", - "enable_overlapping_ranges": %v - } - } - ] + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "leader_lease_duration": 1500, + "leader_renew_deadline": 1000, + "leader_retry_period": 500, + "range": "%s", + "ipRanges": %s, + "log_level": "debug", + "log_file": "/tmp/wb", + "network_name": "%s", + "enable_overlapping_ranges": %v + } }`, ipRange, createIPRanges(ipRanges), poolName, enableOverlappingRanges) return generateNetAttachDefSpec(networkName, namespaceName, macvlanConfig) } @@ -805,3 +887,7 @@ func createIPRanges(ranges []string) string { ipRanges := "[" + strings.Join(formattedRanges[:], ",") + "]" return ipRanges } + +func getPodRef(namespace, name string) string { + return fmt.Sprintf("%s/%s", namespace, name) +} diff --git a/e2e/poolconsistency/checker.go b/e2e/poolconsistency/checker.go index fbc202b3a..fca9dca1c 100644 --- a/e2e/poolconsistency/checker.go +++ b/e2e/poolconsistency/checker.go @@ -22,7 +22,7 @@ func NewPoolConsistencyCheck(ipPool storage.IPPool, podList []corev1.Pod) *Check func (pc *Checker) MissingIPs() []string { var mismatchedIPs []string for _, pod := range pc.podList { - podIPs, err := retrievers.SecondaryIfaceIPValue(&pod) + podIPs, err := retrievers.SecondaryIfaceIPValue(&pod, "net1") podIP := podIPs[len(podIPs)-1] if err != nil { return []string{} @@ -51,7 +51,7 @@ func (pc *Checker) StaleIPs() []string { reservedIP := allocation.IP.String() found := false for _, pod := range pc.podList { - podIPs, err := retrievers.SecondaryIfaceIPValue(&pod) + podIPs, err := retrievers.SecondaryIfaceIPValue(&pod, "net1") podIP := podIPs[len(podIPs)-1] if err != nil { continue diff --git a/e2e/retrievers/pod.go b/e2e/retrievers/pod.go index 6a266ca20..d8a2307a3 100644 --- a/e2e/retrievers/pod.go +++ b/e2e/retrievers/pod.go @@ -19,7 +19,7 @@ func filterNetworkStatus( return nil } -func SecondaryIfaceIPValue(pod *core.Pod) ([]string, error) { +func SecondaryIfaceIPValue(pod *core.Pod, ifName string) ([]string, error) { podNetStatus, found := pod.Annotations[nettypes.NetworkStatusAnnot] if !found { return nil, fmt.Errorf("the pod must feature the `networks-status` annotation") @@ -31,7 +31,7 @@ func SecondaryIfaceIPValue(pod *core.Pod) ([]string, error) { } secondaryInterfaceNetworkStatus := filterNetworkStatus(netStatus, func(status nettypes.NetworkStatus) bool { - return status.Interface == "net1" + return status.Interface == ifName }) if secondaryInterfaceNetworkStatus == nil { diff --git a/pkg/storage/kubernetes/ipam.go b/pkg/storage/kubernetes/ipam.go index f1e58b9f7..12e507943 100644 --- a/pkg/storage/kubernetes/ipam.go +++ b/pkg/storage/kubernetes/ipam.go @@ -271,7 +271,7 @@ func (i *KubernetesIPAM) GetOverlappingRangeStore() (storage.OverlappingRangeSto // ranges. func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP, networkName string) (bool, error) { - normalizedIP := normalizeIP(ip, networkName) + normalizedIP := NormalizeIP(ip, networkName) logging.Debugf("OverlappingRangewide allocation check; normalized IP: %q, IP: %q, networkName: %q", normalizedIP, ip, networkName) @@ -294,7 +294,7 @@ func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx cont // UpdateOverlappingRangeAllocation updates clusterwide allocation for overlapping ranges. func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, podRef, ifName, networkName string) error { - normalizedIP := normalizeIP(ip, networkName) + normalizedIP := NormalizeIP(ip, networkName) clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{ ObjectMeta: metav1.ObjectMeta{Name: normalizedIP, Namespace: c.namespace}, @@ -328,9 +328,9 @@ func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx c return nil } -// normalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us +// NormalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us // to add the network-name when it's different from the unnamed network. -func normalizeIP(ip net.IP, networkName string) string { +func NormalizeIP(ip net.IP, networkName string) string { ipStr := fmt.Sprint(ip) if ipStr[len(ipStr)-1] == ':' { ipStr += "0"