[WIP] Adding support of UDN l2 primary using VMs

README.md

@@ -110,6 +110,35 @@ If the two above are in place, users can orhestrate k8s-netperf to launch VMs by
 
 `k8s-netperf --vm`
 
+### Using a linux bridge interface
+When using `--bridge`, a NetworkAttachmentDefinition defining a bridge interface is attached to the VMs and is used for the test. It requires the name of the bridge as it is defined in the NetworkNodeConfigurationPolicy, NMstate operator is required. For example:
+```yaml
+apiVersion: nmstate.io/v1alpha1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+  name: br0-eth1
+spec:
+  desiredState:
+    interfaces:
+      - name: br0
+        description: Linux bridge with eno2 as a port
+        type: linux-bridge
+        state: up
+        ipv4:
+          dhcp: true
+          enabled: true
+        bridge:
+          options:
+            stp:
+              enabled: false
+          port:
+            - name: eno2
+```
+Then you can launch a test using the bridge interface:
+```
+./bin/amd64/k8s-netperf --vm --bridge br0
+```
+
 ### Config file
 #### Config File v2
 The v2 config file will be executed in the order the tests are presented in the config file.

cmd/k8s-netperf/k8s-netperf.go

@@ -44,6 +44,7 @@ var (
 	full        bool
 	vm          bool
 	debug       bool
+	bridge      string
 	promURL     string
 	id          string
 	searchURL   string
@@ -182,6 +183,13 @@ var rootCmd = &cobra.Command{
 				log.Error(err)
 			}
 			s.KClient = kclient
+			if len(bridge) > 0 {
+				err := k8s.DeployNADBridge(s.DClient, bridge)
+				if err != nil {
+					log.Error(err)
+				}
+				s.Bridge = true
+			}
 		}
 
 		// Build the SUT (Deployments)
@@ -415,6 +423,9 @@ func executeWorkload(nc config.Config,
 		if err != nil {
 			log.Fatal(err)
 		}
+		//when using a bridge, ip is static
+	} else if s.Bridge {
+		serverIP = "10.10.10.14"
 	} else {
 		if hostNet {
 			serverIP = s.ServerHost.Items[0].Status.PodIP
@@ -513,6 +524,7 @@ func main() {
 	rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)")
 	rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log")
 	rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.")
+	rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.")
 	rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL")
 	rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID")
 	rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port")

pkg/config/config.go

@@ -39,6 +39,7 @@ type PerfScenarios struct {
 	VM             bool
 	VMHost         string
 	Udn            bool
+	Bridge         bool
 	ServerNodeInfo metrics.NodeInfo
 	ClientNodeInfo metrics.NodeInfo
 	Client         apiv1.PodList

pkg/k8s/kubernetes.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"time"
 
 	"github.com/cloud-bulldozer/k8s-netperf/pkg/config"
 	log "github.com/cloud-bulldozer/k8s-netperf/pkg/logging"
@@ -151,8 +152,9 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error {
 			"spec": map[string]interface{}{
 				"topology": "Layer2",
 				"layer2": map[string]interface{}{
-					"role":    "Primary",
-					"subnets": []string{"10.0.0.0/24", "2001:db8::/60"},
+					"role":          "Primary",
+					"subnets":       []string{"10.0.0.0/24"},
+					"ipamLifecycle": "Persistent",
 				},
 			},
 		},
@@ -171,6 +173,36 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error {
 	return nil
 }
 
+// Create a NetworkAttachcmentDefinition object for a bridge connection
+func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error {
+	nadBridge := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "k8s.cni.cncf.io/v1",
+			"kind":       "NetworkAttachmentDefinition",
+			"metadata": map[string]interface{}{
+				"name":      "br-netperf",
+				"namespace": "netperf",
+				"annotations": map[string]interface{}{
+					"k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName,
+				},
+			},
+			"spec": map[string]interface{}{
+				"config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`,
+			},
+		},
+	}
+	gvr := schema.GroupVersionResource{
+		Group:    "k8s.cni.cncf.io",
+		Version:  "v1",
+		Resource: "network-attachment-definitions",
+	}
+	_, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // BuildSUT Build the k8s env to run network performance tests
 func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
 	var netperfDataPorts []int32
@@ -527,14 +559,17 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) {
 
 // launchServerVM will create the ServerVM with the specific node and pod affinity.
 func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
-	_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff)
+	_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge, perf.Udn)
 	if err != nil {
 		return err
 	}
 	err = WaitForVMI(perf.KClient, serverRole)
 	if err != nil {
 		return err
 	}
+	log.Infof("⏰ Waiting 120s for the Server VMI to be ready...")
+	//wait for the VMI to be ready
+	time.Sleep(120 * time.Second)
 	if strings.Contains(name, "host") {
 		perf.ServerHost, err = GetPods(perf.ClientSet, fmt.Sprintf("app=%s", serverRole))
 		if err != nil {
@@ -552,7 +587,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA
 
 // launchClientVM will create the ClientVM with the specific node and pod affinity.
 func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
-	host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff)
+	host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge, perf.Udn)
 	if err != nil {
 		return err
 	}
@@ -561,6 +596,9 @@ func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA
 	if err != nil {
 		return err
 	}
+	log.Infof("⏰ Waiting 200s for the client VMI to be ready...")
+	//wait for the VMI to be ready
+	time.Sleep(200 * time.Second)
 	if strings.Contains(name, "host") {
 		perf.ClientHost, err = GetPods(perf.ClientSet, fmt.Sprintf("app=%s", name))
 		if err != nil {

pkg/k8s/kubevirt.go

@@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC
 
 // CreateVMClient takes in the affinity rules and deploys the VMI
 func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset,
-	dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) (string, error) {
+	dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool, udn bool) (string, error) {
 	label := map[string]string{
 		"app":  name,
 		"role": name,
@@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli
 	if err != nil {
 		return "", err
 	}
+	netData := "{}"
 	data := fmt.Sprintf(`#cloud-config
 users:
   - name: fedora
@@ -184,7 +185,7 @@ chpasswd: { expire: False }
 runcmd:
   - export HOME=/home/fedora
   - dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=*
-  - git clone https://github.com/HewlettPackard/netperf
+  - git clone https://github.com/HewlettPackard/netperf.git
   - cd netperf
   - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4
   - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff
@@ -196,7 +197,64 @@ runcmd:
   - curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf
   - chmod 0777 /usr/bin/super-netperf
 `, ssh)
-	_, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff)
+	interfaces := []v1.Interface{
+		{
+			Name: "default",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		},
+	}
+	networks := []v1.Network{
+		{
+			Name: "default",
+			NetworkSource: v1.NetworkSource{
+				Pod: &v1.PodNetwork{},
+			},
+		},
+	}
+	if bridge {
+		interfaces = append(interfaces, v1.Interface{
+			Name: "br-netperf",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		})
+		networks = append(networks, v1.Network{
+			Name: "br-netperf",
+			NetworkSource: v1.NetworkSource{
+				Multus: &v1.MultusNetwork{
+					NetworkName: "netperf/br-netperf",
+				},
+			},
+		})
+		netData = `version: 2
+ethernets:
+  eth1:
+    addresses: [ 10.10.10.12/24 ]`
+	} else if udn {
+		interfaces = []v1.Interface{
+			{
+				Name: "primary-l2-net",
+				Binding: &v1.PluginBinding{
+					Name: "l2bridge",
+				},
+			},
+		}
+		networks = []v1.Network{
+			{
+				Name: "primary-l2-net",
+				NetworkSource: v1.NetworkSource{
+					Pod: &v1.PodNetwork{},
+				},
+			},
+		}
+		netData = `version: 2
+ethernets:
+  eth0:
+    dhcp4: true`
+	}
+	_, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData)))
 	if err != nil {
 		return "", err
 	}
@@ -213,11 +271,12 @@ runcmd:
 
 // CreateVMServer will take the pod and node affinity and deploy the VMI
 func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity,
-	nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) {
+	nodeAff corev1.NodeAffinity, bridge bool, udn bool) (*v1.VirtualMachineInstance, error) {
 	label := map[string]string{
 		"app":  name,
 		"role": role,
 	}
+	netData := "{}"
 	dirname, err := os.UserHomeDir()
 	if err != nil {
 		return nil, err
@@ -237,9 +296,10 @@ ssh_deletekeys: false
 password: fedora
 chpasswd: { expire: False }
 runcmd:
+  - export HOME=/home/fedora
   - dnf install -y --nodocs uperf iperf3 git ethtool
   - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=*
-  - git clone https://github.com/HewlettPackard/netperf
+  - git clone https://github.com/HewlettPackard/netperf.git
   - cd netperf
   - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4
   - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff
@@ -252,12 +312,69 @@ runcmd:
   - iperf3 -s -p %d &
   - netserver &
 `, string(ssh), UperfServerCtlPort, IperfServerCtlPort)
-	return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff)
+	interfaces := []v1.Interface{
+		{
+			Name: "default",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		},
+	}
+	networks := []v1.Network{
+		{
+			Name: "default",
+			NetworkSource: v1.NetworkSource{
+				Pod: &v1.PodNetwork{},
+			},
+		},
+	}
+	if bridge {
+		interfaces = append(interfaces, v1.Interface{
+			Name: "br-netperf",
+			InterfaceBindingMethod: v1.InterfaceBindingMethod{
+				Bridge: &v1.InterfaceBridge{},
+			},
+		})
+		networks = append(networks, v1.Network{
+			Name: "br-netperf",
+			NetworkSource: v1.NetworkSource{
+				Multus: &v1.MultusNetwork{
+					NetworkName: "netperf/br-netperf",
+				},
+			},
+		})
+		netData = `version: 2
+ethernets:
+  eth1:
+    addresses: [ 10.10.10.14/24 ]`
+	} else if udn {
+		interfaces = []v1.Interface{
+			{
+				Name: "primary-l2-net",
+				Binding: &v1.PluginBinding{
+					Name: "l2bridge",
+				},
+			},
+		}
+		networks = []v1.Network{
+			{
+				Name: "primary-l2-net",
+				NetworkSource: v1.NetworkSource{
+					Pod: &v1.PodNetwork{},
+				},
+			},
+		}
+		netData = `version: 2
+ethernets:
+  eth0:
+    dhcp4: true`
+	}
+	return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData)))
 }
 
 // CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity.
 func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity,
-	nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) {
+	nodeAff corev1.NodeAffinity, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) {
 	delSeconds := int64(0)
 	mutliQ := true
 	vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{
@@ -300,8 +417,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin
 							},
 						},
 					},
+					Interfaces: interfaces,
 				},
 			},
+			Networks: networks,
 			Volumes: []v1.Volume{
 				v1.Volume{
 					Name: "disk0",
@@ -315,7 +434,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin
 					Name: "cloudinit",
 					VolumeSource: v1.VolumeSource{
 						CloudInitNoCloud: &v1.CloudInitNoCloudSource{
-							UserDataBase64: b64data,
+							UserDataBase64:    b64data,
+							NetworkDataBase64: netDatab64,
 						},
 					},
 				},