Mellanox pci rss support

netutils_linux_tuning/rss_ladder.py

@@ -51,8 +51,9 @@ def parse(self):
  def eval(self):
  """ Top of all the logic, decide what to do and then apply new settings """
  interrupts = open(self.interrupts_file).readlines()
- for postfix in sorted(self.queue_postfixes_detect(interrupts)):
- self.apply(self.__eval(postfix, interrupts))
+ extract_func = self.queue_suffix_extract if 'pci' in self.options.dev else self.queue_postfix_extract
+ for queue_pattern in sorted(self.queue_pattern_detect(interrupts, extract_func)):
+ self.apply(self.__eval(queue_pattern, interrupts))
 
  def apply(self, decision):
  """
@@ -76,14 +77,25 @@ def apply(self, decision):
  with open(filename, 'w') as irq_file:
  irq_file.write(str(socket_cpu))
 
- def __eval(self, postfix, interrupts):
+ def queue_name_regex(self, queue_pattern):
  """
- :param postfix: '-TxRx-'
- :return: list of tuples(irq, queue_name, socket)
+ :param queue_pattern: -TxRx- or mlx5_comp
+ :return: regex to much entire queue name
+ """
+ if 'pci' in self.options.dev:
+ # mlx5_comp0@pci:0000:01:00.0
+ return r'{1}[0-9]+@{0}'.format(self.options.dev, queue_pattern)
+ # eth0-TxRx-[^ \n]+
+ return r'{0}{1}[^ \n]+'.format(self.options.dev, queue_pattern)
+
+ def __eval(self, queue_pattern, interrupts):
+ """
+ :param queue_pattern: '-TxRx-'
+ :return: list of tuples(irq, queue_name, cpu)
  """
  print_('- distribute interrupts of {0} ({1}) on socket {2}'.format(
- self.options.dev, postfix, self.options.socket))
- queue_regex = r'{0}{1}[^ \n]+'.format(self.options.dev, postfix)
+ self.options.dev, queue_pattern, self.options.socket))
+ queue_regex = self.queue_name_regex(queue_pattern)
  rss_cpus = self.rss_cpus_detect()
  for _ in xrange(self.options.offset):
  rss_cpus.pop()
@@ -106,6 +118,7 @@ def parse_cpus(self, lscpu_output):
 
  def queue_postfix_extract(self, line):
  """
+ used for device based queue-naming
  :param line: '31312 0 0 0 blabla eth0-TxRx-0'
  :return: '-TxRx-'
  """
@@ -114,12 +127,29 @@ def queue_postfix_extract(self, line):
  if queue_name:
  return re.sub(r'({0}|[0-9])'.format(self.options.dev), '', queue_name[0])
 
- def queue_postfixes_detect(self, interrupts):
+ def queue_suffix_extract(self, line):
  """
- self.dev: eth0
- :return: '-TxRx-'
+ used for pci-bus-id based queue-naming
+ :param line: '33: 122736116 0 0 5465612 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.0'
+ :return: mlx5_comp
+ """
+ queue_regex = r'[^ ]*{0}'.format(self.options.dev)
+ queue_name = re.findall(queue_regex, line)
+ if not queue_name:
+ return
+ if '@' in queue_name[0]:
+ queue_name = queue_name[0].split('@') # ['mlx5_comp3', 'pci:0000:01:00.0']
+ return re.sub(r'({0}|[0-9]+$)'.format(self.options.dev), '', queue_name[0])
+
+ @staticmethod
+ def queue_pattern_detect(interrupts, extract_func):
+ """
+ self.dev: eth0 or pci:0000:01:00.0
+ :param interrupts: lines of /proc/interrupts
+ :param extract_func: function to extract queue pattern from lines
+ :return: set(['-TxRx-']) or set(['mlx5_comp'])
  """
- return set([line for line in [self.queue_postfix_extract(line) for line in interrupts] if line])
+ return set([line for line in [extract_func(line) for line in interrupts] if line])
 
  def rss_cpus_detect(self):
  """

setup.py

@@ -16,7 +16,7 @@ def read(*paths):
 
 setuptools.setup(
  name='netutils-linux',
- version='2.7.8',
+ version='2.7.9',
  author='Oleg Strizhechenko',
  author_email='[email protected]',
  license='MIT',

tests/rss-ladder-test

@@ -29,7 +29,7 @@ run_test() {
 }
 
 retval=0
-for test in ixgbe.E5645 igb.E5606; do
+for test in ixgbe.E5645 igb.E5606 mlx5.Q6700; do
  run_test $test || retval=1
 done
 rm -f interrupts lscpu_output output

tests/rss-ladder.tests/mlx5.Q6700/expected

@@ -0,0 +1,8 @@
+- distribute interrupts of pci:0000:01:00.0 (mlx5_async_eq) on socket 0
+- distribute interrupts of pci:0000:01:00.0 (mlx5_cmd_eq) on socket 0
+- distribute interrupts of pci:0000:01:00.0 (mlx5_comp) on socket 0
+ - pci:0000:01:00.0: queue mlx5_comp0@pci:0000:01:00.0 (irq 30) bound to CPU0
+ - pci:0000:01:00.0: queue mlx5_comp1@pci:0000:01:00.0 (irq 31) bound to CPU1
+ - pci:0000:01:00.0: queue mlx5_comp2@pci:0000:01:00.0 (irq 32) bound to CPU2
+ - pci:0000:01:00.0: queue mlx5_comp3@pci:0000:01:00.0 (irq 33) bound to CPU3
+- distribute interrupts of pci:0000:01:00.0 (mlx5_pages_eq) on socket 0

tests/rss-ladder.tests/mlx5.Q6700/interrupts

@@ -0,0 +1,42 @@
+ CPU0 CPU1 CPU2 CPU3 
+ 0: 166 0 0 0 IO-APIC-edge timer
+ 1: 2 0 0 0 IO-APIC-edge i8042
+ 8: 1 0 0 0 IO-APIC-edge rtc0
+ 9: 0 0 0 0 IO-APIC-fasteoi acpi
+ 12: 4 0 0 0 IO-APIC-edge i8042
+ 14: 195 0 0 0 IO-APIC-edge ata_piix
+ 15: 0 0 0 0 IO-APIC-edge ata_piix
+ 16: 0 0 0 0 IO-APIC-fasteoi uhci_hcd:usb5
+ 17: 1087624 0 0 0 IO-APIC-fasteoi eth1
+ 18: 302 0 0 0 IO-APIC-fasteoi uhci_hcd:usb4, radeon
+ 19: 135245 0 0 0 IO-APIC-fasteoi uhci_hcd:usb3, ata_piix
+ 23: 0 0 0 0 IO-APIC-fasteoi ehci_hcd:usb1, uhci_hcd:usb2
+ 27: 1 0 0 0 PCI-MSI-edge mlx5_pages_eq@pci:0000:01:00.0
+ 28: 221496 0 0 0 PCI-MSI-edge mlx5_cmd_eq@pci:0000:01:00.0
+ 29: 0 0 0 0 PCI-MSI-edge mlx5_async_eq@pci:0000:01:00.0
+ 30: 127355089 0 0 0 PCI-MSI-edge mlx5_comp0@pci:0000:01:00.0
+ 31: 120112828 5482507 0 0 PCI-MSI-edge mlx5_comp1@pci:0000:01:00.0
+ 32: 121978940 0 5524729 0 PCI-MSI-edge mlx5_comp2@pci:0000:01:00.0
+ 33: 122736116 0 0 5465612 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.0
+ 34: 1 0 0 0 PCI-MSI-edge mlx5_pages_eq@pci:0000:01:00.1
+ 35: 208713 0 0 0 PCI-MSI-edge mlx5_cmd_eq@pci:0000:01:00.1
+ 36: 0 0 0 0 PCI-MSI-edge mlx5_async_eq@pci:0000:01:00.1
+ 37: 1 0 0 0 PCI-MSI-edge mlx5_comp0@pci:0000:01:00.1
+ 38: 1 0 0 0 PCI-MSI-edge mlx5_comp1@pci:0000:01:00.1
+ 39: 1 0 0 0 PCI-MSI-edge mlx5_comp2@pci:0000:01:00.1
+ 40: 1 0 0 0 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.1
+ 41: 15573731 0 0 0 PCI-MSI-edge eth0
+NMI: 13767 6976 2066 6238 Non-maskable interrupts
+LOC: 6714889 6309050 2414331 5652397 Local timer interrupts
+SPU: 0 0 0 0 Spurious interrupts
+PMI: 13767 6976 2066 6238 Performance monitoring interrupts
+IWI: 0 0 0 0 IRQ work interrupts
+RES: 216058 412432 563703 302262 Rescheduling interrupts
+CAL: 121 78869 169 207 Function call interrupts
+TLB: 30449 33578 52341 37224 TLB shootdowns
+TRM: 0 0 0 0 Thermal event interrupts
+THR: 0 0 0 0 Threshold APIC interrupts
+MCE: 0 0 0 0 Machine check exceptions
+MCP: 40 40 40 40 Machine check polls
+ERR: 0
+MIS: 0

tests/rss-ladder.tests/mlx5.Q6700/lscpu_output

@@ -0,0 +1,8 @@
+# The following is the parsable format, which can be fed to other
+# programs. Each different item in every column has an unique ID
+# starting from zero.
+# CPU,Core,Socket,Node,,L1d,L1i,L2
+0,0,0,0,,0,0,0
+1,1,0,0,,1,1,1
+2,2,0,0,,2,2,0
+3,3,0,0,,3,3,1

tests/rss-ladder.tests/mlx5.Q6700/params

@@ -0,0 +1 @@
+pci:0000:01:00.0