Adding the support of Linux bridge interface for VM setup

Use the --bridge option to create a NAD that will be attached to the VMs
cloud-bulldozer · Nov 14, 2024 · 349b8a8 · 349b8a8
1 parent 3771765
commit 349b8a8
Show file tree

Hide file tree

Showing 4 changed files with 133 additions and 10 deletions.
diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go
@@ -43,6 +43,7 @@ var (
 	full        bool
 	vm          bool
 	debug       bool
+	bridge      string
 	promURL     string
 	id          string
 	searchURL   string
@@ -164,6 +165,13 @@ var rootCmd = &cobra.Command{
 			}
 			s.KClient = kclient
 			s.DClient = dynClient
+			if len(bridge) > 0 {
+				err := k8s.DeployNADBridge(s.DClient, bridge)
+				if err != nil {
+					log.Error(err)
+				}
+				s.Bridge = true
+			}
 		}
 
 		// Build the SUT (Deployments)
@@ -384,6 +392,9 @@ func executeWorkload(nc config.Config,
 		} else {
 			serverIP = s.NetperfService.Spec.ClusterIP
 		}
+		//when using a bridge, ip is static
+	} else if s.Bridge {
+		serverIP = "10.10.10.14"
 	} else {
 		if hostNet {
 			serverIP = s.ServerHost.Items[0].Status.PodIP
@@ -481,6 +492,7 @@ func main() {
 	rootCmd.Flags().BoolVar(&acrossAZ, "across", false, "Place the client and server across availability zones")
 	rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)")
 	rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log")
+	rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.")
 	rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL")
 	rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID")
 	rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port")

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -38,6 +38,7 @@ type PerfScenarios struct {
 	Configs        []Config
 	VM             bool
 	VMHost         string
+	Bridge         bool
 	ServerNodeInfo metrics.NodeInfo
 	ClientNodeInfo metrics.NodeInfo
 	Client         apiv1.PodList

diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go
@@ -12,9 +12,12 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/pointer"
 )
@@ -124,6 +127,36 @@ func BuildInfra(client *kubernetes.Clientset) error {
 	return nil
 }
 
+// Create a NetworkAttachcmentDefinition object for a bridge connection
+func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error {
+	nadBridge := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "k8s.cni.cncf.io/v1",
+			"kind":       "NetworkAttachmentDefinition",
+			"metadata": map[string]interface{}{
+				"name":      "br-netperf",
+				"namespace": "netperf",
+				"annotations": map[string]interface{}{
+					"k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName,
+				},
+			},
+			"spec": map[string]interface{}{
+				"config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`,
+			},
+		},
+	}
+	gvr := schema.GroupVersionResource{
+		Group:    "k8s.cni.cncf.io",
+		Version:  "v1",
+		Resource: "network-attachment-definitions",
+	}
+	_, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // BuildSUT Build the k8s env to run network performance tests
 func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
 	var netperfDataPorts []int32
@@ -451,7 +484,7 @@ func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
 
 // launchServerVM will create the ServerVM with the specific node and pod affinity.
 func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
-	_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff)
+	_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge)
 	if err != nil {
 		return err
 	}
@@ -476,7 +509,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA
 
 // launchClientVM will create the ClientVM with the specific node and pod affinity.
 func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
-	host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff)
+	host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/k8s/kubevirt.go b/pkg/k8s/kubevirt.go
@@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC
 
 // CreateVMClient takes in the affinity rules and deploys the VMI
 func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset,
-	dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) (string, error) {
+	dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool) (string, error) {
 	label := map[string]string{
 		"app":  name,
 		"role": name,
@@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli
 	if err != nil {
 		return "", err
 	}
+	netData := "{}"
 	data := fmt.Sprintf(`#cloud-config
 users:
   - name: fedora
@@ -184,7 +185,7 @@ chpasswd: { expire: False }
 runcmd:
   - export HOME=/home/fedora
   - dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=*
-  - git clone https://github.com/HewlettPackard/netperf
+  - git clone https://github.com/HewlettPackard/netperf.git
   - cd netperf
   - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4
   - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff
@@ -196,7 +197,43 @@ runcmd:
   - curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf
   - chmod 0777 /usr/bin/super-netperf
 `, ssh)
-	_, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff)
+	interfaces := []v1.Interface{
+		{
+			Name: "default",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		},
+	}
+	networks := []v1.Network{
+		{
+			Name: "default",
+			NetworkSource: v1.NetworkSource{
+				Pod: &v1.PodNetwork{},
+			},
+		},
+	}
+	if bridge {
+		interfaces = append(interfaces, v1.Interface{
+			Name: "br-netperf",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		})
+		networks = append(networks, v1.Network{
+			Name: "br-netperf",
+			NetworkSource: v1.NetworkSource{
+				Multus: &v1.MultusNetwork{
+					NetworkName: "netperf/br-netperf",
+				},
+			},
+		})
+		netData = `version: 2
+ethernets:
+  eth1:
+    addresses: [ 10.10.10.12/24 ]`
+	}
+	_, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData)))
 	if err != nil {
 		return "", err
 	}
@@ -213,11 +250,12 @@ runcmd:
 
 // CreateVMServer will take the pod and node affinity and deploy the VMI
 func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity,
-	nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) {
+	nodeAff corev1.NodeAffinity, bridge bool) (*v1.VirtualMachineInstance, error) {
 	label := map[string]string{
 		"app":  name,
 		"role": role,
 	}
+	netData := "{}"
 	dirname, err := os.UserHomeDir()
 	if err != nil {
 		return nil, err
@@ -239,7 +277,7 @@ chpasswd: { expire: False }
 runcmd:
   - dnf install -y --nodocs uperf iperf3 git ethtool
   - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=*
-  - git clone https://github.com/HewlettPackard/netperf
+  - git clone https://github.com/HewlettPackard/netperf.git
   - cd netperf
   - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4
   - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff
@@ -252,12 +290,48 @@ runcmd:
   - iperf3 -s -p %d &
   - netserver &
 `, string(ssh), UperfServerCtlPort, IperfServerCtlPort)
-	return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff)
+	interfaces := []v1.Interface{
+		{
+			Name: "default",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		},
+	}
+	networks := []v1.Network{
+		{
+			Name: "default",
+			NetworkSource: v1.NetworkSource{
+				Pod: &v1.PodNetwork{},
+			},
+		},
+	}
+	if bridge {
+		interfaces = append(interfaces, v1.Interface{
+			Name: "br-netperf",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		})
+		networks = append(networks, v1.Network{
+			Name: "br-netperf",
+			NetworkSource: v1.NetworkSource{
+				Multus: &v1.MultusNetwork{
+					NetworkName: "netperf/br-netperf",
+				},
+			},
+		})
+		netData = `version: 2
+ethernets:
+  eth1:
+    addresses: [ 10.10.10.14/24 ]`
+	}
+	return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData)))
 }
 
 // CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity.
 func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity,
-	nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) {
+	nodeAff corev1.NodeAffinity, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) {
 	delSeconds := int64(0)
 	mutliQ := true
 	vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{
@@ -300,8 +374,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin
 							},
 						},
 					},
+					Interfaces: interfaces,
 				},
 			},
+			Networks: networks,
 			Volumes: []v1.Volume{
 				v1.Volume{
 					Name: "disk0",
@@ -315,7 +391,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin
 					Name: "cloudinit",
 					VolumeSource: v1.VolumeSource{
 						CloudInitNoCloud: &v1.CloudInitNoCloudSource{
-							UserDataBase64: b64data,
+							UserDataBase64:    b64data,
+							NetworkDataBase64: netDatab64,
 						},
 					},
 				},