Merge pull request #163 from redhat-partner-solutions/performance

Performance Testing Implementation

Author: dkosteck

.github/workflows/e2e.yaml

@@ -67,10 +67,10 @@ jobs:
           cd sriov/tests
           cp /config/*.yaml ./
           if [[ "${mode}" == "full" ]]; then
-            pytest -v --html=report.html --self-contained-html SR_IOV_* common
+            pytest -v --html=report.html --self-contained-html SR_IOV_* common --ignore-glob="*test_SR_IOV_Performance.py"
             echo "generated=true" >> $GITHUB_ENV
           elif [[ ${#tests[@]} -ne 0 ]]; then
-            test_string="pytest -v --html=report.html --self-contained-html"
+            test_string="pytest -v --html=report.html --self-contained-html --ignore-glob='*test_SR_IOV_Performance.py'"
             for testname in $(echo "${tests[@]}" | tr ' ' '\n' | sort -u); do
               test_string="${test_string} ${testname}"
               echo "Testing ${testname}"

README.md

@@ -108,7 +108,15 @@ container_volumes:                # the volume mapping to use with the container
                                   # example: "-v /sys:/sys -v /dev:/dev -v /lib/modules:/lib/modules"
 vlan:                             # vlan tag used by the vlan tests, default is 10
 mtu:                              # MTU size; if unspecified, the script will derive it
-bonding_switch_delay              # Expected bonding switch over/back delay in second, default is 1
+bonding_switch_delay:             # Expected bonding switch over/back delay in second, default is 1
+# Below required for SR_IOV_Performance
+testpmd_img:                      # testpmd container image
+testpmd_port:                     # testpmd REST port
+trafficgen_img:                   # trafficgen container image
+trafficgen_port:                  # trafficgen REST port
+trafficgen_timeout:               # trafficgen command timeout (in minutes)
+trafficgen_rx_bps_limit:          # trafficgen baseline comparison (bps)
+log_performance:                  # boolean, use false to omit performance test details in logs/result files (only pass or fail)
 ```
 
 A current version of Python is recommended to run the tests. As of writing the minimum version to avoid warnings would be 3.7. However, the tests have been successfully run up to version 3.11, the latest active release as of writing. The same is true of pip, which should be a current version (23.0 as of writing, but this should be upgraded in the following steps).

sriov/common/configtestdata.py

@@ -55,3 +55,7 @@ def __init__(self, settings: Config) -> None:
         )
         self.ping = {}  # track ping test
         self.mtu = {}  # track mtu change
+
+        # track testpmd and trafficgen container IDs from SR_IOV_Performance for cleanup
+        self.testpmd_id = ""
+        self.trafficgen_id = ""

sriov/common/utils.py

@@ -56,9 +56,10 @@ def bind_driver(ssh_obj: ShellHandler, pci: str, driver: str, timeout: int = 5)
     return True
 
 
-def bind_driver_with_dpdk(settings: object, ssh_obj: ShellHandler, pci: str,
-                          driver: str, timeout: int = 5) -> bool:
-    """ Bind the PCI address to the driver using dpdk-devbind.py
+def bind_driver_with_dpdk(
+    settings: object, ssh_obj: ShellHandler, pci: str, driver: str, timeout: int = 5
+) -> bool:
+    """Bind the PCI address to the driver using dpdk-devbind.py
         in the dpdk container
 
     Args:
@@ -78,12 +79,10 @@ def bind_driver_with_dpdk(settings: object, ssh_obj: ShellHandler, pci: str,
     dpdk_devbind_cmd = (
         f"{settings.config['container_manager']} run -it --rm --privileged "
         f"{settings.config['container_volumes']} "
-        f"{settings.config['dpdk_img']} dpdk-devbind.py -b {driver} {pci}\n")
+        f"{settings.config['dpdk_img']} dpdk-devbind.py -b {driver} {pci}\n"
+    )
 
-    steps = [
-        ("modprobe {}".format(driver), None),
-        (dpdk_devbind_cmd, "Error")
-    ]
+    steps = [("modprobe {}".format(driver), None), (dpdk_devbind_cmd, "Error")]
 
     for step, errorOnStr in steps:
         ssh_obj.log_str(step)
@@ -802,7 +801,11 @@ def set_pipefail(ssh_obj: ShellHandler) -> bool:
 
 
 def execute_and_assert(
-    ssh_obj: ShellHandler, cmds: list, exit_code: int, timeout: int = 0
+    ssh_obj: ShellHandler,
+    cmds: list,
+    exit_code: int,
+    timeout: int = 0,
+    cmd_timeout: int = 5,
 ) -> Tuple[list, list]:
     """Execute the list of commands, assert exit code, and return stdouts and stderrs
 
@@ -811,6 +814,7 @@ def execute_and_assert(
         cmds (list):     list of str commands to run
         exit_code (int): the code to assert
         timeout (int):   optional timeout between cmds (default 0)
+        cmd_timeout (int): optional timeout to wait for commands to complete (default 5)
 
     Returns:
         outs (list): list of lists of str stdout lines
@@ -820,7 +824,7 @@ def execute_and_assert(
     errs = []
     for cmd in cmds:
         ssh_obj.log_str(cmd)
-        code, out, err = ssh_obj.execute(cmd)
+        code, out, err = ssh_obj.execute(cmd, cmd_timeout)
         outs.append(out)
         errs.append(err)
         assert code == exit_code, "\nstdout:" + str(outs) + "\nstderr:" + str(errs)
@@ -851,7 +855,7 @@ def execute_until_timeout(
             return True
         count -= 1
         time.sleep(1)
-    print("\nstdout:" + str(out) + "\nstderr:" + str(err))
+    print("\nstdout:" + str(out) + "\nstderr:" + str(err) + "\ncode:" + str(code))
     return False
 
 
@@ -897,14 +901,13 @@ def get_isolated_cpus(ssh_obj: ShellHandler) -> list:
 
     Args:
         ssh_obj (ShellHandler): ssh connection obj
-        type (str): type of hugepage, 1G or 2M
 
     Returns:
         list: The list of isolated CPUs
     """
     cmd = ["cat /sys/devices/system/cpu/isolated"]
     outs, errs = execute_and_assert(ssh_obj, cmd, 0)
-    isolated = outs[0][0]
+    isolated = outs[0][0].strip()
     isolated_cores = isolated.split(",")
     isolated_list = []
     for core in isolated_cores:
@@ -918,6 +921,29 @@ def get_isolated_cpus(ssh_obj: ShellHandler) -> list:
     return isolated_list
 
 
+def get_isolated_cpus_numa(ssh_obj: ShellHandler, numa: int) -> list:
+    """Return a list of the isolated CPUs belonging to a NUMA node
+
+    Args:
+        ssh_obj (ShellHandler): ssh connection obj
+        numa (int): the numa node
+
+    Returns:
+        list: The list of isolated CPUs belonging to numa
+    """
+    isolated_list = get_isolated_cpus(ssh_obj)
+
+    cmd = [f"lscpu | grep 'NUMA node{numa}'"]
+    outs, errs = execute_and_assert(ssh_obj, cmd, 0)
+    isolated_numa = outs[0][0]
+    isolated_numa_cores = isolated_numa.split(":")[1].strip().split(",")
+    isolated_numa_list = []
+    for core in isolated_numa_cores:
+        isolated_numa_list.append(int(core))
+
+    return list(set(isolated_list) & set(isolated_numa_list))
+
+
 def page_in_kb(type: str) -> str:
     """convert "1G" or "2M" to page size in KB
 

sriov/tests/SR_IOV_Performance/README.md

@@ -0,0 +1,73 @@
+# Test Case Name: SR-IOV.Performance
+
+### Objective(s): A RFC2544 performance test to evaluate a system's (relative) performance when running DPDK workloads. This relies on a trafficgen client and containers built from [netgauge](https://github.com/redhat-eets/netgauge)
+
+### Test procedure
+
+* Reset the VFs
+```
+echo 0 > /sys/class/net/$PF/device/sriov_numvfs
+```
+
+* Ensure the reset succeeds (or check no VF exists under the ```$PF``` by ensuring ```sriov_numvfs``` is 0
+
+* Ensure the trafficgen server ports are on the same numa node, repeat with the two dut server ports
+```
+cat /sys/bus/pci/devices/<pci_address>/numa_node
+```
+
+* Ensure the trafficgen server has 2 1GB hugepages, repeat with the dut server
+
+* On the trafficgen, get 7 isolated CPUs from the numa node associated with the trafficgen ports, repeat on the dut server with 3 isolated CPUs
+
+* On the dut server, create 1 VF on each PF, setting spoof checking off and trust mode on, and bind to vfio-pci
+```
+echo 1 > /sys/class/net/$PF/device/sriov_numvfs
+ip link set $PF vf 0 spoof off
+ip link set $PF vf 0 trust on
+echo $VF_PCI > /sys/bus/pci/devices/$VF_PCI/driver/unbind
+echo vfio-pci > /sys/bus/pci/devices/$VF_PCI/driver_override
+echo $VF_PCI > /sys/bus/pci/drivers/vfio-pci/bind
+```
+
+* On the dut server, start the prebuilt testpmd container
+```
+podman run -d --rm --privileged -p $PORT:$PORT -v /dev/hugepages:/dev/hugepages -v /sys/bus/pci/devices:/sys/bus/pci/devices -v /lib/firmware:/lib/firmware --cpuset-cpus $CPUs $testpmd_container --pci $VF1 --pci $VF2 --http-port $PORT --auto
+```
+
+* On the dut server, ensure that testpmd has started using the REST API
+```
+curl localhost:$PORT/testpmd/status
+```
+
+* On the trafficgen, bind the 2 trafficgen PF ports to vfio-pci
+```
+echo $PF_PCI > /sys/bus/pci/devices/$PF_PCI/driver/unbind
+echo vfio-pci > /sys/bus/pci/devices/$PF_PCI/driver_override
+echo $PF_PCI > /sys/bus/pci/drivers/vfio-pci/bind
+```
+
+* On the trafficgen, start the trafficgen container
+```
+podman run -d --rm --privileged -p $PORT:$PORT -v /dev:/dev -v /sys:/sys -v /lib/modules:/lib/modules --cpuset-cpus $CPUs -e pci_list=$PF1,$PF2 --ip=$IP $trafficgen_container
+```
+
+* On the trafficgen, start the client (once to stabilize, once to collect results)
+```
+python3 /tmp/client.py status --server-addr $IP --server-port $PORT
+python3 /tmp/client.py start --server-addr $IP --server-port $PORT --timeout 60
+python3 /tmp/client.py stop --server-addr $IP --server-port $PORT
+python3 /tmp/client.py auto --server-addr $IP --server-port $PORT
+```
+
+* Compare the results bps to the baseline value
+
+### Clean up
+* Kill containers on dut and trafficgen
+
+* Reset PF driver on trafficgen
+
+* Remove VFs on dut
+```
+echo 0 > /sys/class/net/$PF/device/sriov_numvfs
+```