diff --git a/README.md b/README.md index 6fa1d79..90caffe 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,35 @@ If the two above are in place, users can orhestrate k8s-netperf to launch VMs by `k8s-netperf --vm` +### Using a linux bridge interface +When using `--bridge`, a NetworkAttachmentDefinition defining a bridge interface is attached to the VMs and is used for the test. It requires the name of the bridge as it is defined in the NetworkNodeConfigurationPolicy, NMstate operator is required. For example: +```yaml +apiVersion: nmstate.io/v1alpha1 +kind: NodeNetworkConfigurationPolicy +metadata: + name: br0-eth1 +spec: + desiredState: + interfaces: + - name: br0 + description: Linux bridge with eno2 as a port + type: linux-bridge + state: up + ipv4: + dhcp: true + enabled: true + bridge: + options: + stp: + enabled: false + port: + - name: eno2 +``` +Then you can launch a test using the bridge interface: +``` +./bin/amd64/k8s-netperf --vm --bridge br0 +``` + ### Config file #### Config File v2 The v2 config file will be executed in the order the tests are presented in the config file. diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 684ec27..fd25445 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -44,6 +44,7 @@ var ( full bool vm bool debug bool + bridge string promURL string id string searchURL string @@ -182,6 +183,13 @@ var rootCmd = &cobra.Command{ log.Error(err) } s.KClient = kclient + if len(bridge) > 0 { + err := k8s.DeployNADBridge(s.DClient, bridge) + if err != nil { + log.Error(err) + } + s.Bridge = true + } } // Build the SUT (Deployments) @@ -415,6 +423,9 @@ func executeWorkload(nc config.Config, if err != nil { log.Fatal(err) } + //when using a bridge, ip is static + } else if s.Bridge { + serverIP = "10.10.10.14" } else { if hostNet { serverIP = s.ServerHost.Items[0].Status.PodIP @@ -513,6 +524,7 @@ func main() { rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)") rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log") rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.") + rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.") rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL") rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID") rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port") diff --git a/pkg/config/config.go b/pkg/config/config.go index 44a0cb6..b3099ea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,6 +39,7 @@ type PerfScenarios struct { VM bool VMHost string Udn bool + Bridge bool ServerNodeInfo metrics.NodeInfo ClientNodeInfo metrics.NodeInfo Client apiv1.PodList diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index 36e953a..047943a 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -171,6 +171,36 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error { return nil } +// Create a NetworkAttachcmentDefinition object for a bridge connection +func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error { + nadBridge := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "br-netperf", + "namespace": "netperf", + "annotations": map[string]interface{}{ + "k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName, + }, + }, + "spec": map[string]interface{}{ + "config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`, + }, + }, + } + gvr := schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + } + _, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + // BuildSUT Build the k8s env to run network performance tests func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { var netperfDataPorts []int32 @@ -527,7 +557,7 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) { // launchServerVM will create the ServerVM with the specific node and pod affinity. func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff) + _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge) if err != nil { return err } @@ -552,7 +582,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA // launchClientVM will create the ClientVM with the specific node and pod affinity. func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff) + host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge) if err != nil { return err } diff --git a/pkg/k8s/kubevirt.go b/pkg/k8s/kubevirt.go index 6834ee4..f96addf 100644 --- a/pkg/k8s/kubevirt.go +++ b/pkg/k8s/kubevirt.go @@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC // CreateVMClient takes in the affinity rules and deploys the VMI func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset, - dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) (string, error) { + dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool) (string, error) { label := map[string]string{ "app": name, "role": name, @@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli if err != nil { return "", err } + netData := "{}" data := fmt.Sprintf(`#cloud-config users: - name: fedora @@ -184,7 +185,7 @@ chpasswd: { expire: False } runcmd: - export HOME=/home/fedora - dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -196,7 +197,43 @@ runcmd: - curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf - chmod 0777 /usr/bin/super-netperf `, ssh) - _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridge { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = `version: 2 +ethernets: + eth1: + addresses: [ 10.10.10.12/24 ]` + } + _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) if err != nil { return "", err } @@ -213,11 +250,12 @@ runcmd: // CreateVMServer will take the pod and node affinity and deploy the VMI func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, bridge bool) (*v1.VirtualMachineInstance, error) { label := map[string]string{ "app": name, "role": role, } + netData := "{}" dirname, err := os.UserHomeDir() if err != nil { return nil, err @@ -239,7 +277,7 @@ chpasswd: { expire: False } runcmd: - dnf install -y --nodocs uperf iperf3 git ethtool - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -252,12 +290,48 @@ runcmd: - iperf3 -s -p %d & - netserver & `, string(ssh), UperfServerCtlPort, IperfServerCtlPort) - return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridge { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = `version: 2 +ethernets: + eth1: + addresses: [ 10.10.10.14/24 ]` + } + return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) } // CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity. func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) { delSeconds := int64(0) mutliQ := true vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{ @@ -300,8 +374,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin }, }, }, + Interfaces: interfaces, }, }, + Networks: networks, Volumes: []v1.Volume{ v1.Volume{ Name: "disk0", @@ -315,7 +391,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin Name: "cloudinit", VolumeSource: v1.VolumeSource{ CloudInitNoCloud: &v1.CloudInitNoCloudSource{ - UserDataBase64: b64data, + UserDataBase64: b64data, + NetworkDataBase64: netDatab64, }, }, },