providers/efa: Add device name to trace points

For multi-device platforms we must have the device name to analyze per device traces Signed-off-by: Yehuda Yitschak <[email protected]> Signed-off-by: Michael Margolin <[email protected]>
linux-rdma · Nov 19, 2023 · c61eb23 · c61eb23
1 parent 0bffa5d
commit c61eb23
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 11 deletions.
diff --git a/providers/efa/efa.h b/providers/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2019-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef __EFA_H__
@@ -78,6 +78,7 @@ struct efa_cq {
  pthread_spinlock_t lock;
  struct efa_wq *cur_wq;
  struct efa_io_cdesc_common *cur_cqe;
+ struct ibv_device *dev;
  struct efa_sub_cq sub_cq_arr[];
 };
 
@@ -136,6 +137,7 @@ struct efa_qp {
  int page_size;
  int sq_sig_all;
  int wr_session_err;
+ struct ibv_device *dev;
 };
 
 struct efa_mr {

diff --git a/providers/efa/efa_trace.h b/providers/efa/efa_trace.h
@@ -26,13 +26,15 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
  /* Input arguments */
  LTTNG_UST_TP_ARGS(
+ char *, dev_name,
  uint64_t, wr_id,
  uint32_t, qp_num,
  int, num_sge
  ),
 
  /* Output event fields */
  LTTNG_UST_TP_FIELDS(
+ lttng_ust_field_string(dev_name, dev_name)
  lttng_ust_field_integer(uint64_t, wr_id, wr_id)
  lttng_ust_field_integer(uint32_t, qp_num, qp_num)
  lttng_ust_field_integer(int, num_sge, num_sge)
@@ -48,6 +50,7 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
  /* Input arguments */
  LTTNG_UST_TP_ARGS(
+ char *, dev_name,
  uint64_t, wr_id,
  uint32_t, src_qp_num,
  uint32_t, dst_qp_num,
@@ -56,6 +59,7 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
  /* Output event fields */
  LTTNG_UST_TP_FIELDS(
+ lttng_ust_field_string(dev_name, dev_name)
  lttng_ust_field_integer(uint64_t, wr_id, wr_id)
  lttng_ust_field_integer(uint32_t, src_qp_num, src_qp_num)
  lttng_ust_field_integer(uint32_t, dst_qp_num, dst_qp_num)
@@ -72,6 +76,7 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
  /* Input arguments */
  LTTNG_UST_TP_ARGS(
+ char *, dev_name,
  uint64_t, wr_id,
  int, status,
  uint32_t, qp_num,
@@ -81,6 +86,7 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
  /* Output event fields */
  LTTNG_UST_TP_FIELDS(
+ lttng_ust_field_string(dev_name, dev_name)
  lttng_ust_field_integer(uint64_t, wr_id, wr_id)
  lttng_ust_field_integer(int, status, status)
  lttng_ust_field_integer(uint32_t, qp_num, qp_num)

diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c
@@ -596,8 +596,8 @@ static void efa_process_cqe(struct efa_cq *cq, struct ibv_wc *wc,
  */
  wc->wr_id = cq->cur_wq->wrid[wrid_idx];
 
- rdma_tracepoint(rdma_core_efa, process_completion, wc->wr_id, wc->status, wc->qp_num,
- wc->opcode, wc->byte_len);
+ rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, wc->wr_id, wc->status,
+ wc->qp_num, wc->opcode, wc->byte_len);
 }
 
 static void efa_process_ex_cqe(struct efa_cq *cq, struct efa_qp *qp)
@@ -618,8 +618,8 @@ static void efa_process_ex_cqe(struct efa_cq *cq, struct efa_qp *qp)
  ibvcqx->wr_id = cq->cur_wq->wrid[wrid_idx];
  ibvcqx->status = to_ibv_status(cqe->status);
 
- rdma_tracepoint(rdma_core_efa, process_completion, ibvcqx->wr_id, ibvcqx->status,
- qp->verbs_qp.qp.qp_num, efa_wc_read_opcode(ibvcqx),
+ rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, ibvcqx->wr_id,
+ ibvcqx->status, qp->verbs_qp.qp.qp_num, efa_wc_read_opcode(ibvcqx),
  efa_wc_read_byte_len(ibvcqx));
 }
 
@@ -861,6 +861,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
  cq->buf_size = resp.q_mmap_size;
  cq->num_sub_cqs = num_sub_cqs;
  cq->cqe_size = cqe_size;
+ cq->dev = ibvctx->device;
 
  cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED,
  ibvctx->cmd_fd, resp.q_mmap_key);
@@ -974,7 +975,7 @@ int efa_destroy_cq(struct ibv_cq *ibvcq)
  return err;
  }
 
- munmap(cq->db, to_efa_dev(cq->verbs_cq.cq.context->device)->pg_sz);
+ munmap(cq->db, to_efa_dev(cq->dev)->pg_sz);
  munmap(cq->buf, cq->buf_size);
 
  pthread_spin_destroy(&cq->lock);
@@ -1406,6 +1407,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
  ibvqp = &qp->verbs_qp.qp;
  ibvqp->state = IBV_QPS_RESET;
  qp->sq_sig_all = attr->sq_sig_all;
+ qp->dev = ibvctx->device;
 
  err = efa_rq_initialize(qp, &resp);
  if (err)
@@ -1860,8 +1862,8 @@ int efa_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
  efa_sq_ring_doorbell(sq, wq->pc);
  mmio_wc_start();
  }
- rdma_tracepoint(rdma_core_efa, post_send, wr->wr_id, ibvqp->qp_num,
- meta_desc->dest_qp_num, ah->efa_ah);
+ rdma_tracepoint(rdma_core_efa, post_send, qp->dev->name, wr->wr_id,
+ ibvqp->qp_num, meta_desc->dest_qp_num, ah->efa_ah);
  wr = wr->next;
  }
 
@@ -2141,8 +2143,8 @@ static void efa_send_wr_set_addr(struct ibv_qp_ex *ibvqpx,
  tx_wqe->meta.ah = ah->efa_ah;
  tx_wqe->meta.qkey = remote_qkey;
 
- rdma_tracepoint(rdma_core_efa, post_send, ibvqpx->wr_id, ibvqpx->qp_base.qp_num,
- remote_qpn, ah->efa_ah);
+ rdma_tracepoint(rdma_core_efa, post_send, qp->dev->name, ibvqpx->wr_id,
+ ibvqpx->qp_base.qp_num, remote_qpn, ah->efa_ah);
 }
 
 static void efa_send_wr_start(struct ibv_qp_ex *ibvqpx)
@@ -2356,7 +2358,8 @@ int efa_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
  /* reset descriptor for next iov */
  memset(&rx_buf, 0, sizeof(rx_buf));
  }
- rdma_tracepoint(rdma_core_efa, post_recv, wr->wr_id, ibvqp->qp_num, wr->num_sge);
+ rdma_tracepoint(rdma_core_efa, post_recv, qp->dev->name, wr->wr_id,
+ ibvqp->qp_num, wr->num_sge);
  wr = wr->next;
  }