-
Notifications
You must be signed in to change notification settings - Fork 74
/
rss_ladder.py
196 lines (172 loc) · 7.59 KB
/
rss_ladder.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
# coding=utf-8
""" Receive Side Scaling tuning utility """
import re
import sys
from os.path import join, exists
from six import print_
from six.moves import xrange
from netutils_linux_hardware.rate_math import any2int
from netutils_linux_monitoring.colors import Color
from netutils_linux_monitoring.topology import Topology
from netutils_linux_tuning.base_tune import CPUBasedTune
MAX_QUEUE_PER_DEVICE = 16
class RSSLadder(CPUBasedTune):
""" Distributor of queues' interrupts by multiple CPUs """
topology = None
color = None
interrupts_file = None
def __init__(self, argv=None):
if argv:
sys.argv = [sys.argv[0]] + argv
CPUBasedTune.__init__(self)
self.interrupts_file, lscpu_output = self.parse()
if not exists(self.interrupts_file): # unit-tests
return
self.parse_cpus(lscpu_output)
self.eval()
def parse(self):
""" Detects sources of system data """
interrupts_file = '/proc/interrupts'
lscpu_output = None
if self.options.test_dir:
interrupts_file = join(self.options.test_dir, 'interrupts')
lscpu_output_filename = join(self.options.test_dir, 'lscpu_output')
lscpu_output = open(lscpu_output_filename).read()
# Popen.stdout: python 2.7 returns <str>, 3.6 returns <bytes>
# read() in both cases return <str>
if isinstance(lscpu_output, bytes):
lscpu_output = str(lscpu_output)
return interrupts_file, lscpu_output
def eval(self):
""" Top of all the logic, decide what to do and then apply new settings """
interrupts = open(self.interrupts_file).readlines()
extract_func = self.queue_suffix_extract if 'pci' in self.options.dev else self.queue_postfix_extract
for queue_pattern in sorted(self.queue_pattern_detect(interrupts, extract_func)):
self.apply(self.__eval(queue_pattern, interrupts))
def apply(self, decision):
"""
'* 4' is in case of NIC has more queues than socket has CPUs
:param decision: list of tuples(irq, queue_name, socket)
"""
affinity = list(decision)
cpus = [socket_cpu for irq, queue, socket_cpu in affinity]
if len(set(cpus)) != len(cpus):
warning = 'WARNING: some CPUs process multiple queues, consider reduce queue count for this network device'
if self.options.color:
print_(self.color.wrap(warning, self.color.YELLOW))
else:
print_(warning)
for irq, queue_name, socket_cpu in affinity:
print_(" - {0}: queue {1} (irq {2}) bound to CPU{3}".format(
self.dev_colorize(), queue_name, irq, self.cpu_colorize(socket_cpu)))
if self.options.dry_run:
continue
filename = '/proc/irq/{0}/smp_affinity_list'.format(irq)
with open(filename, 'w') as irq_file:
irq_file.write(str(socket_cpu))
def queue_name_regex(self, queue_pattern):
"""
:param queue_pattern: -TxRx- or mlx5_comp
:return: regex to much entire queue name
"""
if 'pci' in self.options.dev:
# mlx5_comp0@pci:0000:01:00.0
return r'{1}[0-9]+@{0}'.format(self.options.dev, queue_pattern)
# eth0-TxRx-[^ \n]+
return r'{0}{1}[^ \n]+'.format(self.options.dev, queue_pattern)
def __eval(self, queue_pattern, interrupts):
"""
:param queue_pattern: '-TxRx-'
:return: list of tuples(irq, queue_name, cpu)
"""
print_('- distribute interrupts of {0} ({1}) on socket {2}'.format(
self.options.dev, queue_pattern, self.options.socket))
queue_regex = self.queue_name_regex(queue_pattern)
rss_cpus = self.rss_cpus_detect()
for _ in xrange(self.options.offset):
rss_cpus.pop()
for line in interrupts:
queue_name = re.findall(queue_regex, line)
if queue_name:
yield any2int(line.split()[0]), queue_name[0], rss_cpus.pop()
def parse_cpus(self, lscpu_output):
"""
:param lscpu_output: string, output of `lscpu -p`
"""
if self.options.cpus: # no need to detect topology if user gave us cpu list
return
self.topology = Topology(lscpu_output=lscpu_output)
self.color = Color(self.topology, self.options.color)
if not any([self.options.socket is not None, self.options.cpus]):
self.socket_detect()
self.pci.devices = self.pci.node_dev_dict([self.options.dev], False)
def queue_postfix_extract(self, line):
"""
used for device based queue-naming
:param line: '31312 0 0 0 blabla eth0-TxRx-0'
:return: '-TxRx-'
"""
queue_regex = r'{0}[^ \n]+'.format(self.options.dev)
queue_name = re.findall(queue_regex, line)
if queue_name:
return re.sub(r'({0}|[0-9])'.format(self.options.dev), '', queue_name[0])
def queue_suffix_extract(self, line):
"""
used for pci-bus-id based queue-naming
:param line: '33: 122736116 0 0 5465612 PCI-MSI-edge mlx5_comp3@pci:0000:01:00.0'
:return: mlx5_comp
"""
queue_regex = r'[^ ]*{0}'.format(self.options.dev)
queue_name = re.findall(queue_regex, line)
if not queue_name:
return
if '@' in queue_name[0]:
queue_name = queue_name[0].split('@') # ['mlx5_comp3', 'pci:0000:01:00.0']
return re.sub(r'({0}|[0-9]+$)'.format(self.options.dev), '', queue_name[0])
@staticmethod
def queue_pattern_detect(interrupts, extract_func):
"""
self.dev: eth0 or pci:0000:01:00.0
:param interrupts: lines of /proc/interrupts
:param extract_func: function to extract queue pattern from lines
:return: set(['-TxRx-']) or set(['mlx5_comp'])
"""
return set([line for line in [extract_func(line) for line in interrupts] if line])
def rss_cpus_detect(self):
"""
:return: list of cpu ids to use with current queues distribution
"""
rss_cpus = self.options.cpus if self.options.cpus else self.cpus_detect_real()
rss_cpus.reverse()
return rss_cpus * MAX_QUEUE_PER_DEVICE
def dev_colorize(self):
"""
:return: highlighted by NUMA-node name of the device
"""
if not self.pci or not self.options.color or not self.pci.devices:
return self.options.dev
dev_color = self.pci.devices.get(self.options.dev)
color = self.color.COLORS_NODE.get(dev_color)
return self.color.wrap(self.options.dev, color)
def cpu_colorize(self, cpu):
"""
:param cpu: cpu number (0)
:return: highlighted by NUMA-node cpu number.
"""
if not self.topology:
return cpu
return self.color.wrap(cpu, self.color.colorize_cpu(cpu))
def parse_options(self):
"""
:return: parsed arguments
"""
parser = CPUBasedTune.make_parser()
parser.add_argument('--no-color', help='Disable all highlights', dest='color', action='store_false',
default=True)
parser.add_argument('-o', '--offset', type=int, default=0,
help='If you have 2 NICs with 4 queues and 1 socket with 8 cpus, you may be want '
'distribution like this: eth0: [0, 1, 2, 3]; eth1: [4, 5, 6, 7]; '
'so run: rss-ladder-test eth0; rss-ladder-test --offset=4 eth1')
return parser.parse_args()
if __name__ == '__main__':
RSSLadder()