From 23f57d89a3aaff385ef6b39c51dbede1d097af71 Mon Sep 17 00:00:00 2001 From: Oleg Strizhechenko Date: Fri, 7 Dec 2018 19:49:22 +0500 Subject: [PATCH] Support of PCI-slot-based queue naming (#214) * Fixed(rss-docs): __eval return bindings to CPUs, not sockets. * Added(rss-ladder): support for setups with pci-slot-id instead device name * Added(rss-tests): data example for mlx5 setup with pci-slot-based queue naming * Incremented version Example: mlx5_comp0@pci:0000:01:00.0 mlx5_comp1@pci:0000:01:00.0 mlx5_comp2@pci:0000:01:00.0 mlx5_comp3@pci:0000:01:00.0 --- netutils_linux_tuning/rss_ladder.py | 52 +++++++++++++++---- setup.py | 2 +- tests/rss-ladder-test | 2 +- tests/rss-ladder.tests/mlx5.Q6700/expected | 8 +++ tests/rss-ladder.tests/mlx5.Q6700/interrupts | 42 +++++++++++++++ .../rss-ladder.tests/mlx5.Q6700/lscpu_output | 8 +++ tests/rss-ladder.tests/mlx5.Q6700/params | 1 + 7 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 tests/rss-ladder.tests/mlx5.Q6700/expected create mode 100644 tests/rss-ladder.tests/mlx5.Q6700/interrupts create mode 100644 tests/rss-ladder.tests/mlx5.Q6700/lscpu_output create mode 100644 tests/rss-ladder.tests/mlx5.Q6700/params diff --git a/netutils_linux_tuning/rss_ladder.py b/netutils_linux_tuning/rss_ladder.py index e44c653..27a8f7c 100644 --- a/netutils_linux_tuning/rss_ladder.py +++ b/netutils_linux_tuning/rss_ladder.py @@ -51,8 +51,9 @@ def parse(self): def eval(self): """ Top of all the logic, decide what to do and then apply new settings """ interrupts = open(self.interrupts_file).readlines() - for postfix in sorted(self.queue_postfixes_detect(interrupts)): - self.apply(self.__eval(postfix, interrupts)) + extract_func = self.queue_suffix_extract if 'pci' in self.options.dev else self.queue_postfix_extract + for queue_pattern in sorted(self.queue_pattern_detect(interrupts, extract_func)): + self.apply(self.__eval(queue_pattern, interrupts)) def apply(self, decision): """ @@ -76,14 +77,25 @@ def apply(self, decision): with open(filename, 'w') as irq_file: irq_file.write(str(socket_cpu)) - def __eval(self, postfix, interrupts): + def queue_name_regex(self, queue_pattern): """ - :param postfix: '-TxRx-' - :return: list of tuples(irq, queue_name, socket) + :param queue_pattern: -TxRx- or mlx5_comp + :return: regex to much entire queue name + """ + if 'pci' in self.options.dev: + # mlx5_comp0@pci:0000:01:00.0 + return r'{1}[0-9]+@{0}'.format(self.options.dev, queue_pattern) + # eth0-TxRx-[^ \n]+ + return r'{0}{1}[^ \n]+'.format(self.options.dev, queue_pattern) + + def __eval(self, queue_pattern, interrupts): + """ + :param queue_pattern: '-TxRx-' + :return: list of tuples(irq, queue_name, cpu) """ print_('- distribute interrupts of {0} ({1}) on socket {2}'.format( - self.options.dev, postfix, self.options.socket)) - queue_regex = r'{0}{1}[^ \n]+'.format(self.options.dev, postfix) + self.options.dev, queue_pattern, self.options.socket)) + queue_regex = self.queue_name_regex(queue_pattern) rss_cpus = self.rss_cpus_detect() for _ in xrange(self.options.offset): rss_cpus.pop() @@ -106,6 +118,7 @@ def parse_cpus(self, lscpu_output): def queue_postfix_extract(self, line): """ + used for device based queue-naming :param line: '31312 0 0 0 blabla eth0-TxRx-0' :return: '-TxRx-' """ @@ -114,12 +127,29 @@ def queue_postfix_extract(self, line): if queue_name: return re.sub(r'({0}|[0-9])'.format(self.options.dev), '', queue_name[0]) - def queue_postfixes_detect(self, interrupts): + def queue_suffix_extract(self, line): """ - self.dev: eth0 - :return: '-TxRx-' + used for pci-bus-id based queue-naming + :param line: '33: 122736116 0 0 5465612 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.0' + :return: mlx5_comp + """ + queue_regex = r'[^ ]*{0}'.format(self.options.dev) + queue_name = re.findall(queue_regex, line) + if not queue_name: + return + if '@' in queue_name[0]: + queue_name = queue_name[0].split('@') # ['mlx5_comp3', 'pci:0000:01:00.0'] + return re.sub(r'({0}|[0-9]+$)'.format(self.options.dev), '', queue_name[0]) + + @staticmethod + def queue_pattern_detect(interrupts, extract_func): + """ + self.dev: eth0 or pci:0000:01:00.0 + :param interrupts: lines of /proc/interrupts + :param extract_func: function to extract queue pattern from lines + :return: set(['-TxRx-']) or set(['mlx5_comp']) """ - return set([line for line in [self.queue_postfix_extract(line) for line in interrupts] if line]) + return set([line for line in [extract_func(line) for line in interrupts] if line]) def rss_cpus_detect(self): """ diff --git a/setup.py b/setup.py index b3a2a69..6de94db 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def read(*paths): setuptools.setup( name='netutils-linux', - version='2.7.8', + version='2.7.9', author='Oleg Strizhechenko', author_email='oleg.strizhechenko@gmail.com', license='MIT', diff --git a/tests/rss-ladder-test b/tests/rss-ladder-test index f44829a..b8cdbe1 100755 --- a/tests/rss-ladder-test +++ b/tests/rss-ladder-test @@ -29,7 +29,7 @@ run_test() { } retval=0 -for test in ixgbe.E5645 igb.E5606; do +for test in ixgbe.E5645 igb.E5606 mlx5.Q6700; do run_test $test || retval=1 done rm -f interrupts lscpu_output output diff --git a/tests/rss-ladder.tests/mlx5.Q6700/expected b/tests/rss-ladder.tests/mlx5.Q6700/expected new file mode 100644 index 0000000..fb9b2ee --- /dev/null +++ b/tests/rss-ladder.tests/mlx5.Q6700/expected @@ -0,0 +1,8 @@ +- distribute interrupts of pci:0000:01:00.0 (mlx5_async_eq) on socket 0 +- distribute interrupts of pci:0000:01:00.0 (mlx5_cmd_eq) on socket 0 +- distribute interrupts of pci:0000:01:00.0 (mlx5_comp) on socket 0 + - pci:0000:01:00.0: queue mlx5_comp0@pci:0000:01:00.0 (irq 30) bound to CPU0 + - pci:0000:01:00.0: queue mlx5_comp1@pci:0000:01:00.0 (irq 31) bound to CPU1 + - pci:0000:01:00.0: queue mlx5_comp2@pci:0000:01:00.0 (irq 32) bound to CPU2 + - pci:0000:01:00.0: queue mlx5_comp3@pci:0000:01:00.0 (irq 33) bound to CPU3 +- distribute interrupts of pci:0000:01:00.0 (mlx5_pages_eq) on socket 0 diff --git a/tests/rss-ladder.tests/mlx5.Q6700/interrupts b/tests/rss-ladder.tests/mlx5.Q6700/interrupts new file mode 100644 index 0000000..a6d64b6 --- /dev/null +++ b/tests/rss-ladder.tests/mlx5.Q6700/interrupts @@ -0,0 +1,42 @@ + CPU0 CPU1 CPU2 CPU3 + 0: 166 0 0 0 IO-APIC-edge timer + 1: 2 0 0 0 IO-APIC-edge i8042 + 8: 1 0 0 0 IO-APIC-edge rtc0 + 9: 0 0 0 0 IO-APIC-fasteoi acpi + 12: 4 0 0 0 IO-APIC-edge i8042 + 14: 195 0 0 0 IO-APIC-edge ata_piix + 15: 0 0 0 0 IO-APIC-edge ata_piix + 16: 0 0 0 0 IO-APIC-fasteoi uhci_hcd:usb5 + 17: 1087624 0 0 0 IO-APIC-fasteoi eth1 + 18: 302 0 0 0 IO-APIC-fasteoi uhci_hcd:usb4, radeon + 19: 135245 0 0 0 IO-APIC-fasteoi uhci_hcd:usb3, ata_piix + 23: 0 0 0 0 IO-APIC-fasteoi ehci_hcd:usb1, uhci_hcd:usb2 + 27: 1 0 0 0 PCI-MSI-edge mlx5_pages_eq@pci:0000:01:00.0 + 28: 221496 0 0 0 PCI-MSI-edge mlx5_cmd_eq@pci:0000:01:00.0 + 29: 0 0 0 0 PCI-MSI-edge mlx5_async_eq@pci:0000:01:00.0 + 30: 127355089 0 0 0 PCI-MSI-edge mlx5_comp0@pci:0000:01:00.0 + 31: 120112828 5482507 0 0 PCI-MSI-edge mlx5_comp1@pci:0000:01:00.0 + 32: 121978940 0 5524729 0 PCI-MSI-edge mlx5_comp2@pci:0000:01:00.0 + 33: 122736116 0 0 5465612 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.0 + 34: 1 0 0 0 PCI-MSI-edge mlx5_pages_eq@pci:0000:01:00.1 + 35: 208713 0 0 0 PCI-MSI-edge mlx5_cmd_eq@pci:0000:01:00.1 + 36: 0 0 0 0 PCI-MSI-edge mlx5_async_eq@pci:0000:01:00.1 + 37: 1 0 0 0 PCI-MSI-edge mlx5_comp0@pci:0000:01:00.1 + 38: 1 0 0 0 PCI-MSI-edge mlx5_comp1@pci:0000:01:00.1 + 39: 1 0 0 0 PCI-MSI-edge mlx5_comp2@pci:0000:01:00.1 + 40: 1 0 0 0 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.1 + 41: 15573731 0 0 0 PCI-MSI-edge eth0 +NMI: 13767 6976 2066 6238 Non-maskable interrupts +LOC: 6714889 6309050 2414331 5652397 Local timer interrupts +SPU: 0 0 0 0 Spurious interrupts +PMI: 13767 6976 2066 6238 Performance monitoring interrupts +IWI: 0 0 0 0 IRQ work interrupts +RES: 216058 412432 563703 302262 Rescheduling interrupts +CAL: 121 78869 169 207 Function call interrupts +TLB: 30449 33578 52341 37224 TLB shootdowns +TRM: 0 0 0 0 Thermal event interrupts +THR: 0 0 0 0 Threshold APIC interrupts +MCE: 0 0 0 0 Machine check exceptions +MCP: 40 40 40 40 Machine check polls +ERR: 0 +MIS: 0 diff --git a/tests/rss-ladder.tests/mlx5.Q6700/lscpu_output b/tests/rss-ladder.tests/mlx5.Q6700/lscpu_output new file mode 100644 index 0000000..70ec714 --- /dev/null +++ b/tests/rss-ladder.tests/mlx5.Q6700/lscpu_output @@ -0,0 +1,8 @@ +# The following is the parsable format, which can be fed to other +# programs. Each different item in every column has an unique ID +# starting from zero. +# CPU,Core,Socket,Node,,L1d,L1i,L2 +0,0,0,0,,0,0,0 +1,1,0,0,,1,1,1 +2,2,0,0,,2,2,0 +3,3,0,0,,3,3,1 diff --git a/tests/rss-ladder.tests/mlx5.Q6700/params b/tests/rss-ladder.tests/mlx5.Q6700/params new file mode 100644 index 0000000..0e186f4 --- /dev/null +++ b/tests/rss-ladder.tests/mlx5.Q6700/params @@ -0,0 +1 @@ +pci:0000:01:00.0