-
Notifications
You must be signed in to change notification settings - Fork 652
/
mlx5dv_objects.pyx
214 lines (174 loc) · 6.17 KB
/
mlx5dv_objects.pyx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
"""
This module wraps mlx5dv_<obj> C structs, such as mlx5dv_cq, mlx5dv_qp etc.
It exposes to the users the mlx5 driver-specific attributes for ibv objects by
extracting them via mlx5dv_init_obj() API by using Mlx5DvObj class, which holds
all the (currently) supported Mlx5Dv<Obj> objects.
Note: This is not be confused with Mlx5<Obj> which holds the ibv_<obj>_ex that
was created using mlx5dv_create_<obj>().
"""
from libc.stdint cimport uintptr_t, uint32_t
from pyverbs.pyverbs_error import PyverbsUserError, PyverbsRDMAError
cimport pyverbs.providers.mlx5.mlx5_enums as dve
cimport pyverbs.libibverbs as v
cdef class Mlx5DvPD(PyverbsObject):
@property
def pdn(self):
""" The protection domain object number """
return self.dv_pd.pdn
@property
def comp_mask(self):
return self.dv_pd.comp_mask
cdef class Mlx5DvCQ(PyverbsObject):
@property
def cqe_size(self):
return self.dv_cq.cqe_size
@property
def comp_mask(self):
return self.dv_cq.comp_mask
@property
def cqn(self):
return self.dv_cq.cqn
@property
def buf(self):
return <uintptr_t><void*>self.dv_cq.buf
@property
def cq_uar(self):
return <uintptr_t><void*>self.dv_cq.cq_uar
@property
def dbrec(self):
return <uintptr_t><uint32_t*>self.dv_cq.dbrec
@property
def cqe_cnt(self):
return self.dv_cq.cqe_cnt
cdef class Mlx5DvQP(PyverbsObject):
@property
def rqn(self):
""" The receive queue number of the QP"""
return self.dv_qp.rqn
@property
def sqn(self):
""" The send queue number of the QP"""
return self.dv_qp.sqn
@property
def tirn(self):
"""
The number of the transport interface receive object that attached
to the RQ of the QP
"""
return self.dv_qp.tirn
@property
def tisn(self):
"""
The number of the transport interface send object that attached
to the SQ of the QP
"""
return self.dv_qp.tisn
@property
def comp_mask(self):
return self.dv_qp.comp_mask
@comp_mask.setter
def comp_mask(self, val):
self.dv_qp.comp_mask = val
@property
def uar_mmap_offset(self):
return self.uar_mmap_offset
cdef class Mlx5DvSRQ(PyverbsObject):
@property
def stride(self):
return self.dv_srq.stride
@property
def head(self):
return self.dv_srq.stride
@property
def tail(self):
return self.dv_srq.stride
@property
def comp_mask(self):
return self.dv_srq.comp_mask
@comp_mask.setter
def comp_mask(self, val):
self.dv_srq.comp_mask = val
@property
def srqn(self):
""" The shared receive queue object number """
return self.dv_srq.srqn
cdef class Mlx5DvObj(PyverbsObject):
"""
Mlx5DvObj represents mlx5dv_obj C struct.
"""
def __init__(self, obj_type=None, **kwargs):
"""
Retrieves DV objects from ibv object to be able to extract attributes
(such as cqe_size of a CQ).
Currently supports CQ, QP, PD and SRQ objects.
The initialized objects can be accessed using self.dvobj (e.g. self.dvcq).
:param obj_type: Bitmask which defines what objects was provided.
Currently it supports: MLX5DV_OBJ_CQ, MLX5DV_OBJ_QP,
MLX5DV_OBJ_SRQ and MLX5DV_OBJ_PD.
:param kwargs: List of objects (cq, qp, pd, srq) from which to extract
data and their comp_masks if applicable. If comp_mask is
not provided by user, mask all by default.
"""
self.dv_pd = self.dv_cq = self.dv_qp = self.dv_srq = None
if obj_type is None:
return
self.init_obj(obj_type, **kwargs)
def init_obj(self, obj_type, **kwargs):
"""
Initialize DV objects.
The objects are re-initialized if they're already extracted.
"""
supported_obj_types = dve.MLX5DV_OBJ_CQ | dve.MLX5DV_OBJ_QP | \
dve.MLX5DV_OBJ_PD | dve.MLX5DV_OBJ_SRQ
if obj_type & supported_obj_types is False:
raise PyverbsUserError('Invalid obj_type was provided')
cq = kwargs.get('cq') if obj_type | dve.MLX5DV_OBJ_CQ else None
qp = kwargs.get('qp') if obj_type | dve.MLX5DV_OBJ_QP else None
pd = kwargs.get('pd') if obj_type | dve.MLX5DV_OBJ_PD else None
srq = kwargs.get('srq') if obj_type | dve.MLX5DV_OBJ_SRQ else None
if cq is qp is pd is srq is None:
raise PyverbsUserError("No supported object was provided.")
if cq:
dv_cq = Mlx5DvCQ()
self.obj.cq.in_ = <v.ibv_cq*>cq.cq
self.obj.cq.out = &(dv_cq.dv_cq)
self.dv_cq = dv_cq
if qp:
dv_qp = Mlx5DvQP()
comp_mask = kwargs.get('qp_comp_mask')
dv_qp.comp_mask = comp_mask if comp_mask else \
dv.MLX5DV_QP_MASK_UAR_MMAP_OFFSET | \
dv.MLX5DV_QP_MASK_RAW_QP_HANDLES | \
dv.MLX5DV_QP_MASK_RAW_QP_TIR_ADDR
self.obj.qp.in_ = <v.ibv_qp*>qp.qp
self.obj.qp.out = &(dv_qp.dv_qp)
self.dv_qp = dv_qp
if pd:
dv_pd = Mlx5DvPD()
self.obj.pd.in_ = <v.ibv_pd*>pd.pd
self.obj.pd.out = &(dv_pd.dv_pd)
self.dv_pd = dv_pd
if srq:
dv_srq = Mlx5DvSRQ()
comp_mask = kwargs.get('srq_comp_mask')
dv_srq.comp_mask = comp_mask if comp_mask else dv.MLX5DV_SRQ_MASK_SRQN
self.obj.srq.in_ = <v.ibv_srq*>srq.srq
self.obj.srq.out = &(dv_srq.dv_srq)
self.dv_srq = dv_srq
rc = dv.mlx5dv_init_obj(&self.obj, obj_type)
if rc != 0:
raise PyverbsRDMAError("Failed to initialize Mlx5DvObj", rc)
@property
def dvcq(self):
return self.dv_cq
@property
def dvqp(self):
return self.dv_qp
@property
def dvpd(self):
return self.dv_pd
@property
def dvsrq(self):
return self.dv_srq