From 5eb19086fd2d4ca6a8d768b8e5f5c3f1f6fa8d25 Mon Sep 17 00:00:00 2001 From: Charles CAPORALI Date: Thu, 14 Nov 2024 16:31:02 +0100 Subject: [PATCH 1/3] Adding the support of Linux bridge interface for VM setup Use the --bridge option to create a NAD that will be attached to the VMs --- cmd/k8s-netperf/k8s-netperf.go | 13 +++++ pkg/config/config.go | 1 + pkg/k8s/kubernetes.go | 34 ++++++++++++- pkg/k8s/kubevirt.go | 93 +++++++++++++++++++++++++++++++--- 4 files changed, 131 insertions(+), 10 deletions(-) diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 684ec27..b454e90 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -44,6 +44,7 @@ var ( full bool vm bool debug bool + bridge string promURL string id string searchURL string @@ -182,6 +183,14 @@ var rootCmd = &cobra.Command{ log.Error(err) } s.KClient = kclient + s.DClient = dynClient + if len(bridge) > 0 { + err := k8s.DeployNADBridge(s.DClient, bridge) + if err != nil { + log.Error(err) + } + s.Bridge = true + } } // Build the SUT (Deployments) @@ -415,6 +424,9 @@ func executeWorkload(nc config.Config, if err != nil { log.Fatal(err) } + //when using a bridge, ip is static + } else if s.Bridge { + serverIP = "10.10.10.14" } else { if hostNet { serverIP = s.ServerHost.Items[0].Status.PodIP @@ -513,6 +525,7 @@ func main() { rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)") rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log") rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.") + rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.") rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL") rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID") rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port") diff --git a/pkg/config/config.go b/pkg/config/config.go index 44a0cb6..b3099ea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,6 +39,7 @@ type PerfScenarios struct { VM bool VMHost string Udn bool + Bridge bool ServerNodeInfo metrics.NodeInfo ClientNodeInfo metrics.NodeInfo Client apiv1.PodList diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index 36e953a..047943a 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -171,6 +171,36 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error { return nil } +// Create a NetworkAttachcmentDefinition object for a bridge connection +func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error { + nadBridge := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "br-netperf", + "namespace": "netperf", + "annotations": map[string]interface{}{ + "k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName, + }, + }, + "spec": map[string]interface{}{ + "config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`, + }, + }, + } + gvr := schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + } + _, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + // BuildSUT Build the k8s env to run network performance tests func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { var netperfDataPorts []int32 @@ -527,7 +557,7 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) { // launchServerVM will create the ServerVM with the specific node and pod affinity. func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff) + _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge) if err != nil { return err } @@ -552,7 +582,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA // launchClientVM will create the ClientVM with the specific node and pod affinity. func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff) + host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge) if err != nil { return err } diff --git a/pkg/k8s/kubevirt.go b/pkg/k8s/kubevirt.go index 6834ee4..f96addf 100644 --- a/pkg/k8s/kubevirt.go +++ b/pkg/k8s/kubevirt.go @@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC // CreateVMClient takes in the affinity rules and deploys the VMI func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset, - dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) (string, error) { + dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool) (string, error) { label := map[string]string{ "app": name, "role": name, @@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli if err != nil { return "", err } + netData := "{}" data := fmt.Sprintf(`#cloud-config users: - name: fedora @@ -184,7 +185,7 @@ chpasswd: { expire: False } runcmd: - export HOME=/home/fedora - dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -196,7 +197,43 @@ runcmd: - curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf - chmod 0777 /usr/bin/super-netperf `, ssh) - _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridge { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = `version: 2 +ethernets: + eth1: + addresses: [ 10.10.10.12/24 ]` + } + _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) if err != nil { return "", err } @@ -213,11 +250,12 @@ runcmd: // CreateVMServer will take the pod and node affinity and deploy the VMI func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, bridge bool) (*v1.VirtualMachineInstance, error) { label := map[string]string{ "app": name, "role": role, } + netData := "{}" dirname, err := os.UserHomeDir() if err != nil { return nil, err @@ -239,7 +277,7 @@ chpasswd: { expire: False } runcmd: - dnf install -y --nodocs uperf iperf3 git ethtool - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -252,12 +290,48 @@ runcmd: - iperf3 -s -p %d & - netserver & `, string(ssh), UperfServerCtlPort, IperfServerCtlPort) - return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridge { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = `version: 2 +ethernets: + eth1: + addresses: [ 10.10.10.14/24 ]` + } + return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) } // CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity. func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) { delSeconds := int64(0) mutliQ := true vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{ @@ -300,8 +374,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin }, }, }, + Interfaces: interfaces, }, }, + Networks: networks, Volumes: []v1.Volume{ v1.Volume{ Name: "disk0", @@ -315,7 +391,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin Name: "cloudinit", VolumeSource: v1.VolumeSource{ CloudInitNoCloud: &v1.CloudInitNoCloudSource{ - UserDataBase64: b64data, + UserDataBase64: b64data, + NetworkDataBase64: netDatab64, }, }, }, From 6e35be6f2483bc914798cf104663c1d72e9fc44a Mon Sep 17 00:00:00 2001 From: Charles CAPORALI Date: Thu, 14 Nov 2024 16:51:53 +0100 Subject: [PATCH 2/3] Updating the README for the lxn-bridge option --- README.md | 29 +++++++++++++++++++++++++++++ cmd/k8s-netperf/k8s-netperf.go | 1 - 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fa1d79..90caffe 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,35 @@ If the two above are in place, users can orhestrate k8s-netperf to launch VMs by `k8s-netperf --vm` +### Using a linux bridge interface +When using `--bridge`, a NetworkAttachmentDefinition defining a bridge interface is attached to the VMs and is used for the test. It requires the name of the bridge as it is defined in the NetworkNodeConfigurationPolicy, NMstate operator is required. For example: +```yaml +apiVersion: nmstate.io/v1alpha1 +kind: NodeNetworkConfigurationPolicy +metadata: + name: br0-eth1 +spec: + desiredState: + interfaces: + - name: br0 + description: Linux bridge with eno2 as a port + type: linux-bridge + state: up + ipv4: + dhcp: true + enabled: true + bridge: + options: + stp: + enabled: false + port: + - name: eno2 +``` +Then you can launch a test using the bridge interface: +``` +./bin/amd64/k8s-netperf --vm --bridge br0 +``` + ### Config file #### Config File v2 The v2 config file will be executed in the order the tests are presented in the config file. diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index b454e90..fd25445 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -183,7 +183,6 @@ var rootCmd = &cobra.Command{ log.Error(err) } s.KClient = kclient - s.DClient = dynClient if len(bridge) > 0 { err := k8s.DeployNADBridge(s.DClient, bridge) if err != nil { From 38345a348fa46cae692a1c68291c88366063f7c1 Mon Sep 17 00:00:00 2001 From: Charles CAPORALI Date: Fri, 6 Dec 2024 16:10:05 +0100 Subject: [PATCH 3/3] Adding support of UDN l2 primary using VMs Needed to be run on 4.18 and above, uses the l2bridge binding method --- pkg/k8s/kubernetes.go | 16 +++++++++++---- pkg/k8s/kubevirt.go | 47 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index 047943a..24d7acb 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "strings" + "time" "github.com/cloud-bulldozer/k8s-netperf/pkg/config" log "github.com/cloud-bulldozer/k8s-netperf/pkg/logging" @@ -151,8 +152,9 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error { "spec": map[string]interface{}{ "topology": "Layer2", "layer2": map[string]interface{}{ - "role": "Primary", - "subnets": []string{"10.0.0.0/24", "2001:db8::/60"}, + "role": "Primary", + "subnets": []string{"10.0.0.0/24"}, + "ipamLifecycle": "Persistent", }, }, }, @@ -557,7 +559,7 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) { // launchServerVM will create the ServerVM with the specific node and pod affinity. func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge) + _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge, perf.Udn) if err != nil { return err } @@ -565,6 +567,9 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA if err != nil { return err } + log.Infof("⏰ Waiting 120s for the Server VMI to be ready...") + //wait for the VMI to be ready + time.Sleep(120 * time.Second) if strings.Contains(name, "host") { perf.ServerHost, err = GetPods(perf.ClientSet, fmt.Sprintf("app=%s", serverRole)) if err != nil { @@ -582,7 +587,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA // launchClientVM will create the ClientVM with the specific node and pod affinity. func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge) + host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge, perf.Udn) if err != nil { return err } @@ -591,6 +596,9 @@ func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA if err != nil { return err } + log.Infof("⏰ Waiting 200s for the client VMI to be ready...") + //wait for the VMI to be ready + time.Sleep(200 * time.Second) if strings.Contains(name, "host") { perf.ClientHost, err = GetPods(perf.ClientSet, fmt.Sprintf("app=%s", name)) if err != nil { diff --git a/pkg/k8s/kubevirt.go b/pkg/k8s/kubevirt.go index f96addf..e8613e2 100644 --- a/pkg/k8s/kubevirt.go +++ b/pkg/k8s/kubevirt.go @@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC // CreateVMClient takes in the affinity rules and deploys the VMI func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset, - dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool) (string, error) { + dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool, udn bool) (string, error) { label := map[string]string{ "app": name, "role": name, @@ -232,6 +232,27 @@ runcmd: ethernets: eth1: addresses: [ 10.10.10.12/24 ]` + } else if udn { + interfaces = []v1.Interface{ + { + Name: "primary-l2-net", + Binding: &v1.PluginBinding{ + Name: "l2bridge", + }, + }, + } + networks = []v1.Network{ + { + Name: "primary-l2-net", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + netData = `version: 2 +ethernets: + eth0: + dhcp4: true` } _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) if err != nil { @@ -250,7 +271,7 @@ ethernets: // CreateVMServer will take the pod and node affinity and deploy the VMI func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity, bridge bool) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, bridge bool, udn bool) (*v1.VirtualMachineInstance, error) { label := map[string]string{ "app": name, "role": role, @@ -275,6 +296,7 @@ ssh_deletekeys: false password: fedora chpasswd: { expire: False } runcmd: + - export HOME=/home/fedora - dnf install -y --nodocs uperf iperf3 git ethtool - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - git clone https://github.com/HewlettPackard/netperf.git @@ -325,6 +347,27 @@ runcmd: ethernets: eth1: addresses: [ 10.10.10.14/24 ]` + } else if udn { + interfaces = []v1.Interface{ + { + Name: "primary-l2-net", + Binding: &v1.PluginBinding{ + Name: "l2bridge", + }, + }, + } + networks = []v1.Network{ + { + Name: "primary-l2-net", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + netData = `version: 2 +ethernets: + eth0: + dhcp4: true` } return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) }