efa: Add create extended QP direct verb

Add direct verb to create extended IBV QP, with specific driver properties. With the EFA extended init attributes, users can create extended QP with type SRD (Scalable Reliable Datagram). Signed-off-by: Daniel Kranzdorf <[email protected]> Reviewed-by: Firas JahJah <[email protected]> Reviewed-by: Yossi Leybovich <[email protected]> Signed-off-by: Gal Pressman <[email protected]>
linux-rdma · Sep 5, 2019 · 885531b · 885531b
1 parent 0fb4e37
commit 885531b
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 7 deletions.
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols
@@ -91,5 +91,6 @@ libefa.so.1 ibverbs-providers #MINVER#
  EFA_1.0@EFA_1.0 24
  EFA_1.1@EFA_1.1 26
  efadv_create_driver_qp@EFA_1.0 24
+ efadv_create_qp_ex@EFA_1.1 26
  efadv_query_device@EFA_1.1 26
  efadv_query_ah@EFA_1.1 26
diff --git a/providers/efa/efadv.h b/providers/efa/efadv.h
@@ -24,6 +24,17 @@ struct ibv_qp *efadv_create_driver_qp(struct ibv_pd *ibvpd,
  struct ibv_qp_init_attr *attr,
  uint32_t driver_qp_type);
 
+struct efadv_qp_init_attr {
+ uint64_t comp_mask;
+ uint32_t driver_qp_type;
+ uint8_t reserved[4];
+};
+
+struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
+ struct ibv_qp_init_attr_ex *attr_ex,
+ struct efadv_qp_init_attr *efa_attr,
+ uint32_t inlen);
+
 struct efadv_device_attr {
  uint64_t comp_mask;
  uint32_t max_sq_wr;

diff --git a/providers/efa/libefa.map b/providers/efa/libefa.map
@@ -8,6 +8,7 @@ EFA_1.0 {
 
 EFA_1.1 {
  global:
+ efadv_create_qp_ex;
  efadv_query_ah;
  efadv_query_device;
 } EFA_1.0;
diff --git a/providers/efa/man/efadv_create_qp_ex.3.md b/providers/efa/man/efadv_create_qp_ex.3.md
@@ -0,0 +1,74 @@
+---
+layout: page
+title: EFADV_CREATE_QP_EX
+section: 3
+tagline: Verbs
+date: 2019-08-06
+header: "EFA Direct Verbs Manual"
+footer: efa
+---
+
+# NAME
+
+efadv_create_qp_ex - Create EFA specific extended Queue Pair
+
+# SYNOPSIS
+
+```c
+#include <infiniband/efadv.h>
+
+struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
+ struct ibv_qp_init_attr_ex *attr_ex,
+ struct efadv_qp_init_attr *efa_attr,
+ uint32_t inlen);
+```
+
+# DESCRIPTION
+
+**efadv_create_qp_ex()** creates device-specific extended Queue Pair.
+
+The argument attr_ex is an ibv_qp_init_attr_ex struct,
+as defined in <infiniband/verbs.h>.
+
+Use ibv_qp_to_qp_ex() to get the ibv_qp_ex for accessing the send ops
+iterator interface, when QP create attr IBV_QP_INIT_ATTR_SEND_OPS_FLAGS is used.
+
+Scalable Reliable Datagram (SRD) transport provides reliable out-of-order
+delivery, transparently utilizing multiple network paths to reduce network tail
+latency. Its interface is similar to UD, in particular it supports message size
+up to MTU, with error handling extended to support reliable communication.
+
+Compatibility is handled using the comp_mask and inlen fields.
+
+```c
+struct efadv_qp_init_attr {
+ uint64_t comp_mask;
+ uint32_t driver_qp_type;
+ uint8_t reserved[4];
+};
+```
+
+*inlen*
+: In: Size of struct efadv_qp_init_attr.
+
+*comp_mask*
+: Compatibility mask.
+
+*driver_qp_type*
+: The type of QP to be created:
+
+ EFADV_QP_DRIVER_TYPE_SRD:
+  Create an SRD QP.
+
+# RETURN VALUE
+
+efadv_create_qp_ex() returns a pointer to the created QP, or NULL if the request fails.
+
+# SEE ALSO
+
+**efadv**(7), **ibv_create_qp_ex**(3)
+
+# AUTHORS
+
+Gal Pressman <[email protected]>
+Daniel Kranzdorf <[email protected]>
diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c
@@ -673,10 +673,15 @@ static void efa_unlock_cqs(struct ibv_qp *ibvqp)
 }
 
 static int efa_check_qp_attr(struct efa_dev *dev,
- struct ibv_qp_init_attr_ex *attr)
+ struct ibv_qp_init_attr_ex *attr,
+ struct efadv_qp_init_attr *efa_attr)
 {
 #define EFA_CREATE_QP_SUPP_ATTR_MASK IBV_QP_INIT_ATTR_PD
 
+ if (attr->qp_type == IBV_QPT_DRIVER &&
+ efa_attr->driver_qp_type != EFADV_QP_DRIVER_TYPE_SRD)
+ return EOPNOTSUPP;
+
  if (!check_comp_mask(attr->comp_mask, EFA_CREATE_QP_SUPP_ATTR_MASK))
  return EOPNOTSUPP;
 
@@ -712,7 +717,7 @@ static int efa_check_qp_limits(struct efa_dev *dev,
 
 static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
  struct ibv_qp_init_attr_ex *attr,
- uint32_t driver_qp_type)
+ struct efadv_qp_init_attr *efa_attr)
 {
  struct efa_context *ctx = to_efa_context(ibvctx);
  struct efa_dev *dev = to_efa_dev(ibvctx->device);
@@ -724,7 +729,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
  struct efa_qp *qp;
  int err;
 
- err = efa_check_qp_attr(dev, attr);
+ err = efa_check_qp_attr(dev, attr, efa_attr);
  if (err)
  goto err_out;
 
@@ -748,7 +753,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
  req.sq_ring_size = (attr->cap.max_send_wr) *
  sizeof(struct efa_io_tx_wqe);
  if (attr->qp_type == IBV_QPT_DRIVER)
- req.driver_qp_type = driver_qp_type;
+ req.driver_qp_type = efa_attr->driver_qp_type;
 
  err = ibv_cmd_create_qp_ex(ibvctx, &qp->verbs_qp, sizeof(qp->verbs_qp),
  attr, &req.ibv_cmd, sizeof(req),
@@ -816,7 +821,7 @@ struct ibv_qp *efa_create_qp(struct ibv_pd *ibvpd,
  attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD;
  attr_ex.pd = ibvpd;
 
- ibvqp = create_qp(ibvpd->context, &attr_ex, 0);
+ ibvqp = create_qp(ibvpd->context, &attr_ex, NULL);
  if (ibvqp)
  memcpy(attr, &attr_ex, sizeof(*attr));
 
@@ -831,14 +836,15 @@ struct ibv_qp *efa_create_qp_ex(struct ibv_context *ibvctx,
  return NULL;
  }
 
- return create_qp(ibvctx, attr_ex, 0);
+ return create_qp(ibvctx, attr_ex, NULL);
 }
 
 struct ibv_qp *efadv_create_driver_qp(struct ibv_pd *ibvpd,
  struct ibv_qp_init_attr *attr,
  uint32_t driver_qp_type)
 {
  struct ibv_qp_init_attr_ex attr_ex = {};
+ struct efadv_qp_init_attr efa_attr = {};
  struct ibv_qp *ibvqp;
 
  if (!is_efa_dev(ibvpd->context->device)) {
@@ -854,14 +860,35 @@ struct ibv_qp *efadv_create_driver_qp(struct ibv_pd *ibvpd,
  memcpy(&attr_ex, attr, sizeof(*attr));
  attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD;
  attr_ex.pd = ibvpd;
+ efa_attr.driver_qp_type = driver_qp_type;
 
- ibvqp = create_qp(ibvpd->context, &attr_ex, driver_qp_type);
+ ibvqp = create_qp(ibvpd->context, &attr_ex, &efa_attr);
  if (ibvqp)
  memcpy(attr, &attr_ex, sizeof(*attr));
 
  return ibvqp;
 }
 
+struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
+ struct ibv_qp_init_attr_ex *attr_ex,
+ struct efadv_qp_init_attr *efa_attr,
+ uint32_t inlen)
+{
+ if (!is_efa_dev(ibvctx->device)) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
+ if (attr_ex->qp_type != IBV_QPT_DRIVER ||
+ !vext_field_avail(struct efadv_qp_init_attr,
+ driver_qp_type, inlen)) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ return create_qp(ibvctx, attr_ex, efa_attr);
+}
+
 int efa_modify_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr,
  int attr_mask)
 {