prov/efa: Support unsolicited RDMA write with immediate receive

This patch onboards Libfabric with the rdma-core feature that supports unsolicited RDMA write with immediate receive linux-rdma/rdma-core#1459 . When a rdma-write with imm is unsolicited, libfabric won't release the rx pkt and bump the rx pkt counters. Signed-off-by: Shi Jin <[email protected]>
shijin-aws · May 13, 2024 · e8fbd11 · e8fbd11
1 parent 4ee2cbe
commit e8fbd11
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 5 deletions.
diff --git a/prov/efa/configure.m4 b/prov/efa/configure.m4
@@ -71,6 +71,7 @@ AC_DEFUN([FI_EFA_CONFIGURE],[
  have_rdma_size=0
  have_caps_rnr_retry=0
  have_caps_rdma_write=0
+ have_caps_unsolicited_write_recv=0
  have_ibv_is_fork_initialized=0
  efa_support_data_in_order_aligned_128_byte=0
  efadv_support_extended_cq=0
@@ -96,6 +97,11 @@ AC_DEFUN([FI_EFA_CONFIGURE],[
  [have_caps_rdma_write=0],
  [[#include <infiniband/efadv.h>]])
 
+ AC_CHECK_DECL(EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV,
+ [have_caps_unsolicited_write_recv=1],
+ [have_caps_unsolicited_write_recv=0],
+ [[#include <infiniband/efadv.h>]])
+
  AC_CHECK_DECL([ibv_is_fork_initialized],
  [have_ibv_is_fork_initialized=1],
  [have_ibv_is_fork_initialized=0],
@@ -164,6 +170,9 @@ AC_DEFUN([FI_EFA_CONFIGURE],[
  AC_DEFINE_UNQUOTED([HAVE_CAPS_RDMA_WRITE],
  [$have_caps_rdma_write],
  [Indicates if EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE is defined])
+ AC_DEFINE_UNQUOTED([HAVE_CAPS_UNSOLICITED_WRITE_RECV],
+ [$have_caps_unsolicited_write_recv],
+ [Indicates if EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV is defined])
  AC_DEFINE_UNQUOTED([HAVE_IBV_IS_FORK_INITIALIZED],
  [$have_ibv_is_fork_initialized],
  [Indicates if libibverbs has ibv_is_fork_initialized])

diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -177,6 +177,10 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex)
  init_attr_ex);
  } else {
  assert(init_attr_ex->qp_type == IBV_QPT_DRIVER);
+#if HAVE_CAPS_UNSOLICITED_WRITE_RECV
+ if (efa_device_support_unsolicited_write_recv())
+ efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;
+#endif
  efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
  (*qp)->ibv_qp = efadv_create_qp_ex(
  init_attr_ex->pd->context, init_attr_ex, &efa_attr,

diff --git a/prov/efa/src/efa_cq.h b/prov/efa/src/efa_cq.h
@@ -130,6 +130,9 @@ static inline int efa_cq_ibv_cq_ex_open(struct fi_cq_attr *attr,
  .wc_flags = EFADV_WC_EX_WITH_SGID,
  };
 
+ if (efa_device_support_unsolicited_write_recv())
+ efadv_cq_init_attr.wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED;
+
  *ibv_cq_ex = efadv_create_cq(ibv_ctx, &init_attr_ex,
  &efadv_cq_init_attr,
  sizeof(efadv_cq_init_attr));

diff --git a/prov/efa/src/efa_device.c b/prov/efa/src/efa_device.c
@@ -268,6 +268,26 @@ bool efa_device_support_rdma_write(void)
 }
 #endif
 
+/**
+ * @brief check whether efa device support unsolicited write recv
+ *
+ * @return a boolean indicating unsolicited write recv
+ */
+#if HAVE_CAPS_UNSOLICITED_WRITE_RECV
+bool efa_device_support_unsolicited_write_recv(void)
+{
+ if (g_device_cnt <=0)
+ return false;
+
+ return g_device_list[0].device_caps & EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV;
+}
+#else
+bool efa_device_support_unsolicited_write_recv(void)
+{
+ return false;
+}
+#endif
+
 #ifndef _WIN32
 
 static char *get_sysfs_path(void)

diff --git a/prov/efa/src/efa_device.h b/prov/efa/src/efa_device.h
@@ -35,6 +35,8 @@ bool efa_device_support_rdma_read(void);
 
 bool efa_device_support_rdma_write(void);
 
+bool efa_device_support_unsolicited_write_recv(void);
+
 int efa_device_get_driver(struct efa_device *efa_device,
  char **efa_driver);
 

diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c
@@ -71,6 +71,30 @@ static struct fi_ops efa_rdm_cq_fi_ops = {
  .ops_open = fi_no_ops_open,
 };
 
+
+#if HAVE_CAPS_UNSOLICITED_WRITE_RECV
+/**
+ * @brief Check whether a completion consumes recv buffer
+ *
+ * @param ibv_cq_ex extended ibv cq
+ * @return true the wc consumes a recv buffer
+ * @return false the wc doesn't consume a recv buffer
+ */
+static inline
+bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
+{
+ return efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex));
+}
+
+#else
+
+static inline
+bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
+{
+ return false;
+}
+
+#endif
 /**
  * @brief handle rdma-core CQ completion resulted from IBV_WRITE_WITH_IMM
  *
@@ -120,11 +144,15 @@ void efa_rdm_cq_proc_ibv_recv_rdma_with_imm_completion(
 
  efa_cntr_report_rx_completion(&ep->base_ep.util_ep, flags);
 
- /* Recv with immediate will consume a pkt_entry, but the pkt is not
- filled, so free the pkt_entry and record we have one less posted
- packet now. */
- ep->efa_rx_pkts_posted--;
- efa_rdm_pke_release_rx(pkt_entry);
+ if (!efa_rdm_cq_wc_is_unsolicited(ibv_cq_ex)) {
+ /**
+ * Recv with immediate will consume a pkt_entry, but the pkt is not
+ * filled, so free the pkt_entry and record we have one less posted
+ * packet now.
+ */
+ ep->efa_rx_pkts_posted--;
+ efa_rdm_pke_release_rx(pkt_entry);
+ }
 }
 
 #if HAVE_EFADV_CQ_EX