Rss ladder py (#114)

* Refactoring: rss-ladder is rewriten in python
* Added: numa.py support run based on saved lscpu output.
* Added: argparse dependency.
* Increment version.

Makefile

@@ -1,5 +1,5 @@
 test:
- ./tests/rss-ladder
+ ./tests/rss-ladder-test
  ./tests/server-info-show
  ./tests/link_rate_units.sh
  pytest netutils_linux_*/

netutils_linux_hardware/assessor_math.py

@@ -19,7 +19,7 @@ def extract(dictionary, key_sequence):
 
 def any2int(value):
  if isinstance(value, bytes):
- value = str(value, 'utf-8')
+ value = str(value)
  if isinstance(value, int):
  return value
  elif value is None:

netutils_linux_monitoring/numa.py

@@ -28,11 +28,11 @@ class Numa(object):
  numa_layout = None
  socket_layout = None
 
- def __init__(self, devices=None, fake=False):
+ def __init__(self, devices=None, fake=False, lscpu_output=None):
  if fake:
  self.numa_layout = self.socket_layout = self.__FAKE_LAYOUT
  else:
- self.detect_layouts()
+ self.detect_layouts(lscpu_output=lscpu_output)
  if len(set(self.numa_layout.values())) >= 2:
  self.layout = self.numa_layout
  self.layout_kind = 'NUMA'
@@ -68,14 +68,31 @@ def detect_layouts_fallback(self):
  cpu_count = int(stdout.strip())
  self.socket_layout = self.numa_layout = dict(enumerate([0] * cpu_count))
 
- def detect_layouts(self):
-  """ Determine NUMA and sockets layout """
+ @staticmethod
+ def __detect_layout_lscpu():
  process = Popen(['lscpu', '-p'], stdout=PIPE, stderr=PIPE)
  stdout, _ = process.communicate()
- if process.returncode != 0:
+ return stdout, process.returncode
+
+ def detect_layout_lscpu(self, lscpu_output=None):
+ """
+ :param lscpu_output: <str> with output of `lscpu -p` or None
+ :return: <str> output of `lscpu -p` or None
+ """
+ if lscpu_output:
+ return lscpu_output
+ stdout, return_code = self.__detect_layout_lscpu()
+ if return_code != 0:
  return self.detect_layouts_fallback()
- rows = [row for row in stdout.strip().split(b'\n') if not row.startswith(b'#')]
- layouts = [list(map(any2int, row.split(b',')[2:4])) for row in rows]
+ if isinstance(stdout, bytes):
+ stdout = str(stdout)
+ return stdout
+
+ def detect_layouts(self, lscpu_output=None):
+ """ Determine NUMA and sockets layout """
+ stdout = self.detect_layout_lscpu(lscpu_output)
+ rows = [row for row in stdout.strip().split('\n') if not row.startswith('#')]
+ layouts = [list(map(any2int, row.split(',')[2:4])) for row in rows]
  numa_layout, socket_layout = zip(*layouts)
  self.numa_layout = dict(enumerate(numa_layout))
  self.socket_layout = dict(enumerate(socket_layout))

netutils_linux_tuning/rss_ladder.py

@@ -0,0 +1,95 @@
+# coding=utf-8
+
+""" Receive Side Scaling tuning utility """
+
+import re
+from os.path import join
+from argparse import ArgumentParser
+from six import iteritems, print_
+from netutils_linux_monitoring.numa import Numa
+from netutils_linux_hardware.assessor_math import any2int
+
+
+class RSSLadder(object):
+
+ def __init__(self):
+ interrupts_file = '/proc/interrupts'
+ self.options = self.parse_options()
+ lscpu_output = None
+ if self.options.test_dir:
+ interrupts_file = join(self.options.test_dir, "interrupts")
+ lscpu_output_filename = join(self.options.test_dir, "lscpu_output")
+ lscpu_output = open(lscpu_output_filename).read()
+ # Popen.stdout in python 2.7 returns <str>
+ # Popen.stdout in python 3.6 returns <bytes>
+ # read() in both cases return <str>
+ if isinstance(lscpu_output, bytes):
+ lscpu_output = str(lscpu_output)
+ self.interrupts = open(interrupts_file).readlines()
+ self.numa = Numa(lscpu_output=lscpu_output)
+ for postfix in sorted(self.queue_postfixes_detect()):
+ self.smp_affinity_list_apply(self.smp_affinity_list_make(postfix))
+
+ @staticmethod
+ def parse_options():
+ parser = ArgumentParser()
+ parser.add_argument('-t', '--test-dir', type=str,
+ help="Use prepared test dataset in TEST_DIR directory instead of /proc/interrupts.")
+ parser.add_argument('-d', '--dry-run', help="Don't write anything to smp_affinity_list.",
+ action='store_true', default=False)
+ parser.add_argument('-o', '--offset', type=int, default=0,
+ help="If you have 2 NICs with 4 queues and 1 socket with 8 cpus, you may be want "
+ "distribution like this: eth0: [0, 1, 2, 3]; eth1: [4, 5, 6, 7]; "
+ "so run: rss-ladder-test eth0; rss-ladder-test --offset=4 eth1")
+ parser.add_argument('dev', type=str)
+ parser.add_argument('socket', nargs='?', type=int, default=0)
+ return parser.parse_args()
+
+ def queue_postfix_extract(self, line):
+ """
+ :param line: "31312 0 0 0 blabla eth0-TxRx-0"
+ :return: "-TxRx-"
+ """
+ queue_regex = r'{0}[^ \n]+'.format(self.options.dev)
+ queue_name = re.findall(queue_regex, line)
+ if queue_name:
+ return re.sub(r'({0}|[0-9])'.format(self.options.dev), '', queue_name[0])
+
+ def queue_postfixes_detect(self):
+ """
+ self.dev: eth0
+ :return: "-TxRx-"
+ """
+ return set([line for line in [self.queue_postfix_extract(line) for line in self.interrupts] if line])
+
+ def smp_affinity_list_make(self, postfix):
+ """
+ :param postfix: "-TxRx-"
+ :return: list of tuples(irq, queue_name, socket)
+ """
+ print_("- distribute interrupts of {0} ({1}) on socket {2}".format(
+ self.options.dev, postfix, self.options.socket))
+ queue_regex = r'{0}{1}[^ \n]+'.format(self.options.dev, postfix)
+ socket_cpus = [k for k, v in iteritems(self.numa.socket_layout) if v == self.options.socket] * 4
+ socket_cpus.reverse()
+ for line in self.interrupts:
+ queue_name = re.findall(queue_regex, line)
+ if queue_name:
+ yield any2int(line.split()[0]), queue_name[0], socket_cpus.pop()
+
+ def smp_affinity_list_apply(self, smp_affinity):
+ """
+ '* 4' is in case of NIC has more queues than socket has CPUs
+ :param smp_affinity: list of tuples(irq, queue_name, socket)
+ """
+ for irq, queue_name, socket_cpu in smp_affinity:
+ print_(" - {0}: irq {1} {2} -> {3}".format(self.options.dev, irq, queue_name, socket_cpu))
+ if self.options.dry_run:
+ continue
+ filename = "/proc/irq/{0}/smp_affinity_list".format(irq)
+ with open(filename, 'w') as irq_file:
+ irq_file.write(str(socket_cpu))
+
+
+if __name__ == '__main__':
+ RSSLadder()

requirements.txt

@@ -8,3 +8,4 @@ flake8
 pytest
 collective.checkdocs
 Pygments
+argparse

setup.py

@@ -14,7 +14,7 @@ def read(*paths):
 
 setuptools.setup(
  name='netutils-linux',
- version='2.0.14',
+ version='2.1.0',
  author='Oleg Strizhechenko',
  author_email='[email protected]',
  license='MIT',
@@ -24,7 +24,7 @@ def read(*paths):
  long_description=(read('README.rst')),
  packages=setuptools.find_packages(exclude=['tests*']),
  scripts=[os.path.join('utils/', script) for script in os.listdir('utils/')],
- install_requires=['pyyaml', 'ipaddress', 'six', 'colorama', 'prettytable'],
+ install_requires=['pyyaml', 'ipaddress', 'six', 'colorama', 'prettytable', 'argparse'],
  classifiers=[
  'Development Status :: 3 - Alpha',
  'Intended Audience :: Developers',

tests/rss-ladder → tests/rss-ladder-test

@@ -12,7 +12,7 @@ run_test() {
  cp $testdata/$name/interrupts ./interrupts || return 1
  cp $testdata/$name/lscpu_output ./lscpu_output || return 2
  read -a params < $testdata/$name/params
- rss-ladder "${params[@]}" > ./output || return 3
+ rss-ladder --dry-run --test-dir $testdata/$name "${params[@]}" > ./output || return 3
  cmp -s $testdata/$name/expected output || rc=$?
  if [ "$rc" = '0' ]; then
  echo OK

tests/rss-ladder.tests/igb.E5606/params

@@ -1 +1 @@
---test eth0
+eth0

tests/rss-ladder.tests/ixgbe.E5645/params

@@ -1 +1 @@
---test eth1 1
+eth1 1

utils/rss-ladder

@@ -1,120 +1,7 @@
-#!/bin/bash
+#!/usr/bin/env python
+# coding=utf-8
 
-set -euo pipefail
+from netutils_linux_tuning.rss_ladder import RSSLadder
 
-DRYRUN=0
-INTERRUPTS=/proc/interrupts
-
-# Local development/debug/testing:
-# put files:
-# - cat /proc/interrupts -> ./interrupts
-# - lscpu --extended=CPU,SOCKET -> ./lscpu_output
-# rss-ladder --test ixgbe eth1 1
-
-if [ "${1:-}" = '--test' ]; then
- INTERRUPTS=interrupts
- lscpu() {
- cat lscpu_output
- }
- shift
- DRYRUN=1
-fi
-
-
-usage() {
- echo "Usage: $0 [--dry-run] [--test] <dev> [cpu socket]"
- echo
- echo "# 90% use-case:"
- echo "Example: $0 eth1"
- echo
- echo "# You may be like to have 'dry run' before really change affinities:"
- echo "Example: $0 --dry-run eth1"
- echo
- echo "# On multiprocessor machine it's better to explicitely define which socket will be used"
- echo "Example: $0 eth1 0"
- echo "Example: $0 eth2 1"
-}
-
-__socket_cpu_list() {
- local socket="$1"
- # $1 $3 -> CPU SOCKET
- lscpu -p | grep -v '^#' | awk -F ',' '{print $1" "$3}' | egrep "[0-9]+[ ]+$socket$" | awk '{print $1}'
-}
-
-cpus_list_all() {
- lscpu -p | grep -v '^#' | awk -F ',' '{print $1}'
-}
-
-socket_cpu_list() {
- local socket="$1"
- local queue_count="$2"
- local cpus times=4
- cpus=( $(__socket_cpu_list $socket) )
- for _ in $(seq $times); do
- __socket_cpu_list $socket
- done > /tmp/cpus.$socket
- head -n $queue_count /tmp/cpus.$socket
- rm -f /tmp/cpus.$socket
-}
-
-interrtupts() {
- local dev="$1"
- local postfix="$2"
- grep "$dev$postfix" $INTERRUPTS
-}
-
-set_affinity() {
- local dev="$1"
- local cpu_bound="$2"
- local queue_count="$3"
- local irq queue cpu cpus
- cpus="$(cpus_list_all | sed 's/.*/cpu&/')"
- # shellcheck disable=SC2046
- while read _ irq $cpus _ queue cpu _; do
- irq="${irq//:}"
- echo " - $dev: irq $irq $queue -> $cpu"
- if [ "$DRYRUN" = 0 ]; then
- echo $cpu > /proc/irq/$irq/smp_affinity_list
- fi
- done
-}
-
-tune() {
- local dev="$1"
- local cpu_bound="$2"
- local postfix="$3"
- local queue_count
- echo "- distribute interrupts of $dev ($postfix) on socket $cpu_bound"
- interrtupts "$dev" "$postfix" | cat -n > /tmp/interrupts.$dev
- queue_count="$(wc -l < /tmp/interrupts.$dev)"
- socket_cpu_list "$cpu_bound" "$queue_count" | cat -n > /tmp/cpu.$cpu_bound
- join /tmp/interrupts.$dev /tmp/cpu.$cpu_bound > /tmp/irq.$dev
- set_affinity "$dev" "$cpu_bound" "$queue_count" < /tmp/irq.$dev
-}
-
-parse_params() {
- if [ "$1" == '--dry-run' ]; then
- DRYRUN=1
- shift
- fi
- if [ "$1" = '--help' ]; then
- usage
- exit 0
- fi
- dev="$1"
- cpu_bound="${2:-0}"
-}
-
-detect_postfixes() {
- grep -o "$1-.*" "$INTERRUPTS" | sed "s/$1//g" | tr -d '[0-9]' | sort -u
-}
-
-main() {
- local postfix
- parse_params "$@"
- for postfix in $(detect_postfixes $dev); do
- tune $dev $cpu_bound $postfix
- done
-}
-
-main "$@"
+if __name__ == '__main__':
+ RSSLadder()