diff --git a/lisa/sut_orchestrator/libvirt/context.py b/lisa/sut_orchestrator/libvirt/context.py index 230e70212c..62f4fbe801 100644 --- a/lisa/sut_orchestrator/libvirt/context.py +++ b/lisa/sut_orchestrator/libvirt/context.py @@ -81,6 +81,9 @@ class NodeContext: default_factory=list, ) + # Add host detail under node context for device-passthrough testcases + host_node: Any = None + def get_environment_context(environment: Environment) -> EnvironmentContext: return environment.get_context(EnvironmentContext) diff --git a/lisa/sut_orchestrator/libvirt/libvirt_device_pool.py b/lisa/sut_orchestrator/libvirt/libvirt_device_pool.py index 7e4c871c71..37c8bfeb5c 100644 --- a/lisa/sut_orchestrator/libvirt/libvirt_device_pool.py +++ b/lisa/sut_orchestrator/libvirt/libvirt_device_pool.py @@ -186,24 +186,74 @@ def create_device_pool( vendor_id=vendor_id, device_id=device_id, ) - primary_nic_iommu = self.get_primary_nic_id() - for item in device_list: - device = DeviceAddressSchema() - domain, bus, slot, fn = self._parse_pci_address_str(addr=item.slot) - device.domain = domain - device.bus = bus - device.slot = slot - device.function = fn + bdf_list = [i.slot for i in device_list] + self._create_pool(pool_type, bdf_list) + + def create_device_pool_from_pci_addresses( + self, + pool_type: HostDevicePoolType, + pci_addr_list: List[str], + ) -> None: + self.available_host_devices[pool_type] = {} + for bdf in pci_addr_list: + domain, bus, slot, fn = self._parse_pci_address_str(bdf) + device = self._get_pci_address_instance(domain, bus, slot, fn) iommu_group = self._get_device_iommu_group(device) - is_vfio_pci = self._is_driver_vfio_pci(device) - if not is_vfio_pci and iommu_group not in primary_nic_iommu: + # Get all the devices of that iommu group + iommu_path = f"/sys/kernel/iommu_groups/{iommu_group}/devices" + bdf_list = [i.strip() for i in self.host_node.tools[Ls].list(iommu_path)] + bdf_list.append(bdf.strip()) # append the given device in list + + self._create_pool(pool_type, bdf_list) + + def _create_pool( + self, + pool_type: HostDevicePoolType, + bdf_list: List[str], + ) -> None: + iommu_grp_of_used_devices = [] + primary_nic_iommu = self.get_primary_nic_id() + for bdf in bdf_list: + domain, bus, slot, fn = self._parse_pci_address_str(bdf) + dev = self._get_pci_address_instance(domain, bus, slot, fn) + is_vfio_pci = self._is_driver_vfio_pci(dev) + iommu_group = self._get_device_iommu_group(dev) + + if iommu_group in iommu_grp_of_used_devices: + # No need to add this device in pool as one of the devices for this + # iommu group is in use + continue + + if is_vfio_pci: + # Do not consider any device for pool if any device of same iommu group + # is already assigned + pool = self.available_host_devices.get(pool_type, {}) + pool.pop(iommu_group, []) + self.available_host_devices[pool_type] = pool + iommu_grp_of_used_devices.append(iommu_group) + elif ( + iommu_group not in primary_nic_iommu and + iommu_group not in iommu_grp_of_used_devices + ): pool = self.available_host_devices.get(pool_type, {}) devices = pool.get(iommu_group, []) - devices.append(device) + if dev not in devices: + devices.append(dev) pool[iommu_group] = devices self.available_host_devices[pool_type] = pool + def _get_pci_address_instance( + self, domain: str, bus: str, slot: str, fn: str, + ) -> DeviceAddressSchema: + device = DeviceAddressSchema() + device.domain = domain + device.bus = bus + device.slot = slot + device.function = fn + + return device + def _add_device_passthrough_xml( self, devices: ET.Element, diff --git a/lisa/sut_orchestrator/libvirt/platform.py b/lisa/sut_orchestrator/libvirt/platform.py index 2946385686..abd5543a39 100644 --- a/lisa/sut_orchestrator/libvirt/platform.py +++ b/lisa/sut_orchestrator/libvirt/platform.py @@ -813,6 +813,10 @@ def _fill_nodes_metadata(self, environment: Environment, log: Logger) -> None: ) node_context = get_node_context(node) + if self.host_node.is_remote: + node_context.host_node = remote_node + else: + node_context.host_node = self.host_node if node_context.init_system == InitSystem.CLOUD_INIT: # Ensure cloud-init completes its setup. node.execute( diff --git a/lisa/sut_orchestrator/libvirt/schema.py b/lisa/sut_orchestrator/libvirt/schema.py index 13271d135a..655ab302bc 100644 --- a/lisa/sut_orchestrator/libvirt/schema.py +++ b/lisa/sut_orchestrator/libvirt/schema.py @@ -48,6 +48,16 @@ class DeviceAddressSchema: slot: str = "" function: str = "" + def __eq__(self, other): + if isinstance(other, DeviceAddressSchema): + return ( + self.domain == other.domain and + self.bus == other.bus and + self.slot == other.slot and + self.function == other.function + ) + return False + # QEMU orchestrator's global configuration options. @dataclass_json() diff --git a/lisa/sut_orchestrator/util/device_pool.py b/lisa/sut_orchestrator/util/device_pool.py index 7f91d3e787..61245642e1 100644 --- a/lisa/sut_orchestrator/util/device_pool.py +++ b/lisa/sut_orchestrator/util/device_pool.py @@ -1,6 +1,11 @@ from typing import Any, List, Optional -from lisa.sut_orchestrator.util.schema import HostDevicePoolSchema, HostDevicePoolType +from lisa.sut_orchestrator.util.schema import ( + HostDevicePoolSchema, + HostDevicePoolType, + PciAddressIdentifier, + VendorDeviceIdIdentifier, +) from lisa.util import LisaException @@ -16,6 +21,13 @@ def create_device_pool( ) -> None: raise NotImplementedError() + def create_device_pool_from_pci_addresses( + self, + pool_type: HostDevicePoolType, + pci_addr_list: List[str], + ) -> None: + raise NotImplementedError() + def get_primary_nic_id(self) -> List[str]: raise NotImplementedError() @@ -44,22 +56,31 @@ def configure_device_passthrough_pool( f"Pool type '{pool_type}' is not supported by platform" ) for config in device_configs: - vendor_device_list = config.devices - if len(vendor_device_list) > 1: - raise LisaException( - "Device Pool does not support more than one " - "vendor/device id list for given pool type" - ) + device_list = config.devices + if all(isinstance(d, VendorDeviceIdIdentifier) for d in device_list): + if len(device_list) > 1: + raise LisaException( + "Device Pool does not support more than one " + "vendor/device id list for given pool type" + ) - vendor_device_id = vendor_device_list[0] - assert vendor_device_id.vendor_id.strip() - vendor_id = vendor_device_id.vendor_id.strip() + vendor_device_id = device_list[0] + assert vendor_device_id.vendor_id.strip() + vendor_id = vendor_device_id.vendor_id.strip() - assert vendor_device_id.device_id.strip() - device_id = vendor_device_id.device_id.strip() + assert vendor_device_id.device_id.strip() + device_id = vendor_device_id.device_id.strip() - self.create_device_pool( - pool_type=config.type, - vendor_id=vendor_id, - device_id=device_id, - ) + self.create_device_pool( + pool_type=config.type, + vendor_id=vendor_id, + device_id=device_id, + ) + elif all(isinstance(d, PciAddressIdentifier) for d in device_list): + # Create pool from the list of PCI addresses + self.create_device_pool_from_pci_addresses( + pool_type=config.type, + pci_addr_list=device_list, + ) + else: + raise LisaException("Unknown device identifier") diff --git a/lisa/sut_orchestrator/util/schema.py b/lisa/sut_orchestrator/util/schema.py index 369d5eb9fd..139cc13644 100644 --- a/lisa/sut_orchestrator/util/schema.py +++ b/lisa/sut_orchestrator/util/schema.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field from enum import Enum -from typing import List +from typing import List, Union from dataclasses_json import dataclass_json @@ -12,17 +12,26 @@ class HostDevicePoolType(Enum): @dataclass_json() @dataclass -class DeviceIdentifier: +class VendorDeviceIdIdentifier: vendor_id: str = "" device_id: str = "" +@dataclass_json() +@dataclass +class PciAddressIdentifier: + # ex. 0000:3b:00.0 - ::. + pci_bdf: str = "" + + # Configuration options for device-passthrough for the VM. @dataclass_json() @dataclass class HostDevicePoolSchema: type: HostDevicePoolType = HostDevicePoolType.PCI_NIC - devices: List[DeviceIdentifier] = field(default_factory=list) + devices: Union[ + List[VendorDeviceIdIdentifier], List[PciAddressIdentifier] + ] = field(default_factory=list) @dataclass_json() diff --git a/lisa/tools/iperf3.py b/lisa/tools/iperf3.py index 32f993c4d3..bad5f02d42 100644 --- a/lisa/tools/iperf3.py +++ b/lisa/tools/iperf3.py @@ -119,6 +119,7 @@ def run_as_server_async( use_json_format: bool = False, one_connection_only: bool = False, daemon: bool = True, + interface_ip: str = "", ) -> Process: # -s: run iperf3 as server mode # -D: run iperf3 as a daemon @@ -135,6 +136,8 @@ def run_as_server_async( cmd += f" -f {report_unit} " if port: cmd += f" -p {port} " + if interface_ip: + cmd += f" -B {interface_ip}" process = self.node.execute_async( f"{self.command} {cmd}", shell=True, sudo=True ) diff --git a/microsoft/testsuites/performance/common.py b/microsoft/testsuites/performance/common.py index 975ccadac5..f0b06c9186 100644 --- a/microsoft/testsuites/performance/common.py +++ b/microsoft/testsuites/performance/common.py @@ -446,17 +446,35 @@ def perf_iperf( connections: List[int], buffer_length_list: List[int], udp_mode: bool = False, + server: Optional[RemoteNode] = None, + client: Optional[RemoteNode] = None, + run_server_on_internal_address: bool = False, ) -> None: environment = test_result.environment assert environment, "fail to get environment from testresult" - client = cast(RemoteNode, environment.nodes[0]) - server = cast(RemoteNode, environment.nodes[1]) + if server is not None or client is not None: + assert server is not None, "server need to be specified, if client is set" + assert client is not None, "client need to be specified, if server is set" + else: + environment = test_result.environment + assert environment, "fail to get environment from testresult" + # set server and client from environment, if not set explicitly + client = cast(RemoteNode, environment.nodes[0]) + server = cast(RemoteNode, environment.nodes[1]) + + # Ensure that both server and client are non-None before accessing tools + assert client is not None, "client is None, cannot access tools" + assert server is not None, "server is None, cannot access tools" + client_iperf3, server_iperf3 = run_in_parallel( [lambda: client.tools[Iperf3], lambda: server.tools[Iperf3]] ) test_case_name = inspect.stack()[1][3] iperf3_messages_list: List[Any] = [] + server_interface_ip = "" + if run_server_on_internal_address: + server_interface_ip = server.internal_address if udp_mode: for node in [client, server]: ssh = node.tools[Ssh] @@ -481,7 +499,13 @@ def perf_iperf( current_server_iperf_instances += 1 server_iperf3_process_list.append( server_iperf3.run_as_server_async( - current_server_port, "g", 10, True, True, False + port=current_server_port, + report_unit="g", + report_periodic=10, + use_json_format=True, + one_connection_only=True, + daemon=False, + interface_ip=server_interface_ip, ) ) current_server_port += 1 diff --git a/microsoft/testsuites/performance/networkperf.py b/microsoft/testsuites/performance/networkperf.py index adb014b1ec..3edd82f33f 100644 --- a/microsoft/testsuites/performance/networkperf.py +++ b/microsoft/testsuites/performance/networkperf.py @@ -1,10 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import re from functools import partial -from typing import Any +from typing import Any, Tuple, cast from lisa import ( Logger, + RemoteNode, TestCaseMetadata, TestSuite, TestSuiteMetadata, @@ -16,8 +18,10 @@ from lisa.environment import Environment, Node from lisa.features import Sriov, Synthetic from lisa.operating_system import BSD, Windows +from lisa.sut_orchestrator import CLOUD_HYPERVISOR +from lisa.sut_orchestrator.libvirt.context import get_node_context from lisa.testsuite import TestResult -from lisa.tools import Sysctl +from lisa.tools import Lspci, Sysctl from lisa.tools.iperf3 import ( IPERF_TCP_BUFFER_LENGTHS, IPERF_TCP_CONCURRENCY, @@ -25,6 +29,7 @@ IPERF_UDP_CONCURRENCY, ) from lisa.tools.sockperf import SOCKPERF_TCP, SOCKPERF_UDP +from lisa.util import SkippedException, constants, find_group_in_lines from lisa.util.parallel import run_in_parallel from microsoft.testsuites.performance.common import ( cleanup_process, @@ -450,3 +455,190 @@ def do_sysctl_cleanup(node: Node) -> None: run_in_parallel( [partial(do_sysctl_cleanup, x) for x in environment.nodes.list()] ) + + @TestCaseMetadata( + description=""" + This test case uses iperf3 to test passthrough tcp network throughput. + """, + priority=3, + timeout=TIMEOUT, + requirement=simple_requirement( + min_count=1, + supported_platform_type=[CLOUD_HYPERVISOR], + ), + ) + def perf_tcp_iperf_passthrough( + self, + node: Node, + result: TestResult, + ) -> None: + # Run iperf server on VM and client on host + server, client = self._get_server_and_client_nodes(node) + perf_iperf( + test_result=result, + connections=IPERF_TCP_CONCURRENCY, + buffer_length_list=IPERF_TCP_BUFFER_LENGTHS, + server=server, + client=client, + run_server_on_internal_address=True, + ) + + @TestCaseMetadata( + description=""" + This test case uses iperf3 to test passthrough udp network throughput. + """, + priority=3, + timeout=TIMEOUT, + requirement=simple_requirement( + min_count=1, + supported_platform_type=[CLOUD_HYPERVISOR], + ), + ) + def perf_udp_iperf_passthrough( + self, + node: Node, + result: TestResult, + ) -> None: + # Run iperf server on VM and client on host + server, client = self._get_server_and_client_nodes(node) + perf_iperf( + test_result=result, + connections=IPERF_TCP_CONCURRENCY, + buffer_length_list=IPERF_TCP_BUFFER_LENGTHS, + server=server, + client=client, + udp_mode=True, + run_server_on_internal_address=True, + ) + + @TestCaseMetadata( + description=""" + This test case uses sar to test passthrough network PPS (Packets Per Second) + when running netperf with single port. Test will consider VM as + server node and host as client node. + """, + priority=3, + timeout=PPS_TIMEOUT, + requirement=simple_requirement( + min_count=1, + supported_platform_type=[CLOUD_HYPERVISOR], + ), + ) + def perf_tcp_single_pps_passthrough( + self, + result: TestResult, + node: Node, + ) -> None: + server, client = self._get_server_and_client_nodes(node) + + perf_tcp_pps( + test_result=result, + test_type="singlepps", + server=server, + client=client, + ) + + @TestCaseMetadata( + description=""" + This test case uses sar to test passthrough network PPS (Packets Per Second) + when running netperf with multiple ports. Test will consider VM as + server node and host as client node. + """, + priority=3, + timeout=PPS_TIMEOUT, + requirement=simple_requirement( + min_count=1, + supported_platform_type=[CLOUD_HYPERVISOR], + ), + ) + def perf_tcp_max_pps_passthrough( + self, + result: TestResult, + node: Node, + ) -> None: + server, client = self._get_server_and_client_nodes(node) + + perf_tcp_pps( + test_result=result, + test_type="maxpps", + server=server, + client=client, + ) + + def _get_server_and_client_nodes( + self, + node: Node, + ) -> Tuple[RemoteNode, RemoteNode]: + ctx = get_node_context(node) + if not ctx.passthrough_devices: + raise SkippedException("No passthrough devices found for node") + + # Configure the nw interface on guest + node.execute( + cmd="dhclient", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message="dhclient command failed", + ) + + lspci = node.tools[Lspci] + pci_devices = lspci.get_devices_by_type( + constants.DEVICE_TYPE_SRIOV, force_run=True + ) + device_addr = None + + # Get the first non-virtio device + for device in pci_devices: + kernel_driver = lspci.get_used_module(device.slot) + if kernel_driver != "virtio-pci": + device_addr = device.slot + break + print(f"passthrough device: {device_addr}") + + # Get the interface name + err_msg: str = "Can't find interface from PCI address" + device_path = node.execute( + cmd=( + "find /sys/class/net/*/device/subsystem/devices" + f" -name '*{device_addr}*'" + ), + sudo=True, + shell=True, + expected_exit_code=0, + expected_exit_code_failure_message=err_msg, + ).stdout + + pattern = re.compile(r"/sys/class/net/(?P\w+)/device") + interface_name_raw = find_group_in_lines( + pattern=pattern, + lines=device_path, + ) + interface_name = interface_name_raw.get("INTERFACE_NAME", "") + assert interface_name, "Can not find interface name" + print(f"interface_name: {interface_name}") + + # Get the interface ip + err_msg = f"Failed to get interface details for: {interface_name}" + interface_details = node.execute( + cmd=f"ip addr show {interface_name}", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=err_msg, + ).stdout + print(f"interface_details: {interface_details}") + ip_regex = re.compile(r"\binet (?P\d+\.\d+\.\d+\.\d+)/\d+\b") + interface_ip = find_group_in_lines( + lines=interface_details, + pattern=ip_regex, + single_line=False, + ) + passthrough_nic_ip = interface_ip.get("INTERFACE_IP", "") + print(f"passthrough_nic_ip: {passthrough_nic_ip}") + assert passthrough_nic_ip, "Can not find interface IP" + + server_node = cast(RemoteNode, node) + server_node.internal_address = passthrough_nic_ip + + host_node = cast(RemoteNode, ctx.host_node) + + return server_node, host_node