Skip to content

Commit

Permalink
Add allocation checks for single pod e2e tests
Browse files Browse the repository at this point in the history 
This allows to validate the lifecycle of allocations for
single and multiple interfaces pods.

Signed-off-by: Marcelo Guerrero <[email protected]>
  • Loading branch information
@mlguerrero12
mlguerrero12 committed Jun 21, 2024
1 parent 3a667d1 commit 45029aa
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 52 deletions.
174 changes: 130 additions & 44 deletions e2e/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

v1 "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
Expand All @@ -27,12 +28,14 @@ import (
"github.com/k8snetworkplumbingwg/whereabouts/e2e/retrievers"
testenv "github.com/k8snetworkplumbingwg/whereabouts/e2e/testenvironment"
"github.com/k8snetworkplumbingwg/whereabouts/pkg/api/whereabouts.cni.cncf.io/v1alpha1"
"github.com/k8snetworkplumbingwg/whereabouts/pkg/iphelpers"
wbstorage "github.com/k8snetworkplumbingwg/whereabouts/pkg/storage/kubernetes"
"github.com/k8snetworkplumbingwg/whereabouts/pkg/types"
)

const (
createPodTimeout = 10 * time.Second
ipPoolNamespace = "kube-system"
)

func TestWhereaboutsE2E(t *testing.T) {
Expand Down Expand Up @@ -86,10 +89,15 @@ var _ = Describe("Whereabouts functionality", func() {
})

Context("Single pod tests", func() {
BeforeEach(func() {
const singlePodName = "whereabouts-basic-test"
var err error
const singlePodName = "whereabouts-basic-test"
var err error

AfterEach(func() {
By("deleting pod with whereabouts net-attach-def")
_ = clientInfo.DeletePod(pod)
})

It("allocates a single pod with a single interface", func() {
By("creating a pod with whereabouts net-attach-def")
pod, err = clientInfo.ProvisionPod(
singlePodName,
Expand All @@ -98,19 +106,55 @@ var _ = Describe("Whereabouts functionality", func() {
entities.PodNetworkSelectionElements(testNetworkName),
)
Expect(err).NotTo(HaveOccurred())
})

AfterEach(func() {
By("deleting pod with whereabouts net-attach-def")
Expect(clientInfo.DeletePod(pod)).To(Succeed())
})

It("allocates a single pod within the correct IP range", func() {
By("checking pod IP is within whereabouts IPAM range")
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod)
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1")
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs).NotTo(BeEmpty())
Expect(inRange(ipv4TestRange, secondaryIfaceIPs[0])).To(Succeed())

By("verifying allocation")
verifyAllocations(clientInfo, ipv4TestRange, secondaryIfaceIPs[0], testNamespace, pod.Name, "net1")

By("deleting pod")
err = clientInfo.DeletePod(pod)
Expect(err).NotTo(HaveOccurred())

By("checking that the IP allocation is removed")
verifyNoAllocationsForPodRef(clientInfo, ipv4TestRange, testNamespace, pod.Name, secondaryIfaceIPs)
})
It("allocates a single pod with multiple interfaces", func() {
By("creating a pod with whereabouts net-attach-def")
pod, err = clientInfo.ProvisionPod(
singlePodName,
testNamespace,
podTierLabel(singlePodName),
entities.PodNetworkSelectionElements(testNetworkName, testNetworkName, testNetworkName),
)
Expect(err).NotTo(HaveOccurred())

By("checking pod IP is within whereabouts IPAM range")
var secondaryIPs []string

for _, ifName := range []string{"net1", "net2", "net3"} {
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName)
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs).NotTo(BeEmpty())
for _, ip := range secondaryIfaceIPs {
Expect(inRange(ipv4TestRange, ip)).To(Succeed())

By("verifying allocation")
verifyAllocations(clientInfo, ipv4TestRange, ip, testNamespace, pod.Name, ifName)
}
secondaryIPs = append(secondaryIPs, secondaryIfaceIPs...)
}

By("deleting pod")
err = clientInfo.DeletePod(pod)
Expect(err).NotTo(HaveOccurred())

By("checking that the IP allocation is removed")
verifyNoAllocationsForPodRef(clientInfo, ipv4TestRange, testNamespace, pod.Name, secondaryIPs)
})
})

Expand Down Expand Up @@ -160,7 +204,7 @@ var _ = Describe("Whereabouts functionality", func() {

It("allocates a single pod within the correct IP ranges", func() {
By("checking pod IP is within whereabouts IPAM ranges")
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod)
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1")
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs).To(HaveLen(2))
Expect(inRange(dualStackIPv4Range, secondaryIfaceIPs[0])).To(Succeed())
Expand Down Expand Up @@ -202,7 +246,7 @@ var _ = Describe("Whereabouts functionality", func() {

It("allocates a single pod within the correct IP ranges", func() {
By("checking pod IP is within whereabouts IPAM ranges")
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod)
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1")
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs).To(HaveLen(3))
Expect(inRange(ipv4TestRange, secondaryIfaceIPs[0])).To(Succeed())
Expand All @@ -225,7 +269,6 @@ var _ = Describe("Whereabouts functionality", func() {
By("creating a replicaset with whereabouts net-attach-def")
var err error

const ipPoolNamespace = "kube-system"
k8sIPAM, err = wbstorage.NewKubernetesIPAMWithNamespace("", "", types.IPAMConfig{
Kubernetes: types.KubernetesConfig{
KubeConfigPath: testConfig.KubeconfigPath,
Expand Down Expand Up @@ -296,7 +339,6 @@ var _ = Describe("Whereabouts functionality", func() {
Context("stateful set tests", func() {
const (
initialReplicaNumber = 20
ipPoolNamespace = "kube-system"
namespace = "default"
serviceName = "web"
selector = "app=" + serviceName
Expand Down Expand Up @@ -484,8 +526,8 @@ var _ = Describe("Whereabouts functionality", func() {
Expect(err).NotTo(HaveOccurred())
Expect(ipPool.Spec.Allocations).NotTo(BeEmpty())

Expect(allocationForPodRef(podRef, *ipPool).ContainerID).NotTo(Equal(containerID))
Expect(allocationForPodRef(podRef, *ipPool).PodRef).To(Equal(podRef))
Expect(allocationForPodRef(podRef, *ipPool)[0].ContainerID).NotTo(Equal(containerID))
Expect(allocationForPodRef(podRef, *ipPool)[0].PodRef).To(Equal(podRef))
})
})
})
Expand Down Expand Up @@ -551,13 +593,14 @@ var _ = Describe("Whereabouts functionality", func() {
})

It("allocates the correct IP address to the second pod", func() {
ifName := "net1"
By("checking pod IP is within whereabouts IPAM range")
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod)
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName)
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs).NotTo(BeEmpty())

By("checking pod 2 IP is within whereabouts IPAM range")
secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2)
secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2, ifName)
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs2).NotTo(BeEmpty())

Expand Down Expand Up @@ -656,19 +699,20 @@ var _ = Describe("Whereabouts functionality", func() {
})

It("allocates the same IP to the Pods as they are in different address collision domains", func() {
ifName := "net1"
By("checking pod IP is within whereabouts IPAM range")
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod)
secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName)
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs).NotTo(BeEmpty())

By("checking pod 2 IP is within whereabouts IPAM range and has the same IP as pod 1")
secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2)
secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2, ifName)
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs2).NotTo(BeEmpty())
Expect(secondaryIfaceIPs[0]).To(Equal(secondaryIfaceIPs2[0]))

By("checking pod 3 IP is within whereabouts IPAM range and has a different IP from pod 2")
secondaryIfaceIPs3, err := retrievers.SecondaryIfaceIPValue(pod3)
secondaryIfaceIPs3, err := retrievers.SecondaryIfaceIPValue(pod3, ifName)
Expect(err).NotTo(HaveOccurred())
Expect(secondaryIfaceIPs3).NotTo(BeEmpty())
Expect(secondaryIfaceIPs2[0]).NotTo(Equal(secondaryIfaceIPs3[0]))
Expand All @@ -677,13 +721,55 @@ var _ = Describe("Whereabouts functionality", func() {
})
})

func allocationForPodRef(podRef string, ipPool v1alpha1.IPPool) *v1alpha1.IPAllocation {
func verifyNoAllocationsForPodRef(clientInfo *wbtestclient.ClientInfo, ipv4TestRange, testNamespace, podName string, secondaryIfaceIPs []string) {
Eventually(func() bool {
ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: ipv4TestRange, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())

allocation := allocationForPodRef(getPodRef(testNamespace, podName), *ipPool)
return len(allocation) == 0
}, 3*time.Second, 500*time.Millisecond).Should(BeTrue())

for _, ip := range secondaryIfaceIPs {
Eventually(func() bool {
_, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{})
if err != nil && errors.IsNotFound(err) {
return true
}
return false
}, 3*time.Second, 500*time.Millisecond).Should(BeTrue())
}
}

func verifyAllocations(clientInfo *wbtestclient.ClientInfo, ipv4TestRange, ip, testNamespace, podName, ifName string) {
ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: ipv4TestRange, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())

firstIP, _, err := net.ParseCIDR(ipv4TestRange)
Expect(err).NotTo(HaveOccurred())
offset, err := iphelpers.IPGetOffset(net.ParseIP(ip), firstIP)
Expect(err).NotTo(HaveOccurred())

allocation, ok := ipPool.Spec.Allocations[fmt.Sprintf("%d", offset)]
Expect(ok).To(BeTrue())
Expect(allocation.PodRef).To(Equal(getPodRef(testNamespace, podName)))
Expect(allocation.IfName).To(Equal(ifName))

overlapping, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())

Expect(overlapping.Spec.IfName).To(Equal(ifName))
Expect(overlapping.Spec.PodRef).To(Equal(getPodRef(testNamespace, podName)))
}

func allocationForPodRef(podRef string, ipPool v1alpha1.IPPool) []v1alpha1.IPAllocation {
var allocations []v1alpha1.IPAllocation
for _, allocation := range ipPool.Spec.Allocations {
if allocation.PodRef == podRef {
return &allocation
allocations = append(allocations, allocation)
}
}
return nil
return allocations
}

func clusterConfig() (*rest.Config, error) {
Expand Down Expand Up @@ -760,25 +846,21 @@ func macvlanNetworkWithWhereaboutsIPAMNetwork(networkName string, namespaceName
macvlanConfig := fmt.Sprintf(`{
"cniVersion": "0.3.0",
"disableCheck": true,
"plugins": [
{
"type": "macvlan",
"master": "eth0",
"mode": "bridge",
"ipam": {
"type": "whereabouts",
"leader_lease_duration": 1500,
"leader_renew_deadline": 1000,
"leader_retry_period": 500,
"range": "%s",
"ipRanges": %s,
"log_level": "debug",
"log_file": "/tmp/wb",
"network_name": "%s",
"enable_overlapping_ranges": %v
}
}
]
"type": "macvlan",
"master": "eth0",
"mode": "bridge",
"ipam": {
"type": "whereabouts",
"leader_lease_duration": 1500,
"leader_renew_deadline": 1000,
"leader_retry_period": 500,
"range": "%s",
"ipRanges": %s,
"log_level": "debug",
"log_file": "/tmp/wb",
"network_name": "%s",
"enable_overlapping_ranges": %v
}
}`, ipRange, createIPRanges(ipRanges), poolName, enableOverlappingRanges)
return generateNetAttachDefSpec(networkName, namespaceName, macvlanConfig)
}
Expand All @@ -805,3 +887,7 @@ func createIPRanges(ranges []string) string {
ipRanges := "[" + strings.Join(formattedRanges[:], ",") + "]"
return ipRanges
}

func getPodRef(namespace, name string) string {
return fmt.Sprintf("%s/%s", namespace, name)
}
4 changes: 2 additions & 2 deletions e2e/poolconsistency/checker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func NewPoolConsistencyCheck(ipPool storage.IPPool, podList []corev1.Pod) *Check
func (pc *Checker) MissingIPs() []string {
var mismatchedIPs []string
for _, pod := range pc.podList {
podIPs, err := retrievers.SecondaryIfaceIPValue(&pod)
podIPs, err := retrievers.SecondaryIfaceIPValue(&pod, "net1")
podIP := podIPs[len(podIPs)-1]
if err != nil {
return []string{}
Expand Down Expand Up @@ -51,7 +51,7 @@ func (pc *Checker) StaleIPs() []string {
reservedIP := allocation.IP.String()
found := false
for _, pod := range pc.podList {
podIPs, err := retrievers.SecondaryIfaceIPValue(&pod)
podIPs, err := retrievers.SecondaryIfaceIPValue(&pod, "net1")
podIP := podIPs[len(podIPs)-1]
if err != nil {
continue
Expand Down
4 changes: 2 additions & 2 deletions e2e/retrievers/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func filterNetworkStatus(
return nil
}

func SecondaryIfaceIPValue(pod *core.Pod) ([]string, error) {
func SecondaryIfaceIPValue(pod *core.Pod, ifName string) ([]string, error) {
podNetStatus, found := pod.Annotations[nettypes.NetworkStatusAnnot]
if !found {
return nil, fmt.Errorf("the pod must feature the `networks-status` annotation")
Expand All @@ -31,7 +31,7 @@ func SecondaryIfaceIPValue(pod *core.Pod) ([]string, error) {
}

secondaryInterfaceNetworkStatus := filterNetworkStatus(netStatus, func(status nettypes.NetworkStatus) bool {
return status.Interface == "net1"
return status.Interface == ifName
})

if secondaryInterfaceNetworkStatus == nil {
Expand Down
8 changes: 4 additions & 4 deletions pkg/storage/kubernetes/ipam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func (i *KubernetesIPAM) GetOverlappingRangeStore() (storage.OverlappingRangeSto
// ranges.
func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP,
networkName string) (bool, error) {
normalizedIP := normalizeIP(ip, networkName)
normalizedIP := NormalizeIP(ip, networkName)

logging.Debugf("OverlappingRangewide allocation check; normalized IP: %q, IP: %q, networkName: %q",
normalizedIP, ip, networkName)
Expand All @@ -294,7 +294,7 @@ func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx cont
// UpdateOverlappingRangeAllocation updates clusterwide allocation for overlapping ranges.
func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP,
podRef, ifName, networkName string) error {
normalizedIP := normalizeIP(ip, networkName)
normalizedIP := NormalizeIP(ip, networkName)

clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{
ObjectMeta: metav1.ObjectMeta{Name: normalizedIP, Namespace: c.namespace},
Expand Down Expand Up @@ -328,9 +328,9 @@ func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx c
return nil
}

// normalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us
// NormalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us
// to add the network-name when it's different from the unnamed network.
func normalizeIP(ip net.IP, networkName string) string {
func NormalizeIP(ip net.IP, networkName string) string {
ipStr := fmt.Sprint(ip)
if ipStr[len(ipStr)-1] == ':' {
ipStr += "0"
Expand Down

0 comments on commit 45029aa

Please sign in to comment.