mlx5: Add tracepoint for post send

Add initial support for MLX5 LTTng tracing using compiled out macros if LTTng flag isn't provided. Signed-off-by: zhenwei pi <[email protected]>
linux-rdma · Oct 19, 2023 · ec4b65b · ec4b65b
1 parent 657f025
commit ec4b65b
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 0 deletions.
diff --git a/providers/mlx5/CMakeLists.txt b/providers/mlx5/CMakeLists.txt
@@ -10,8 +10,13 @@ if (MLX5_MW_DEBUG)
  add_definitions("-DMW_DEBUG")
 endif()
 
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+ set(TRACE_FILE mlx5_trace.c)
+endif()
+
 rdma_shared_provider(mlx5 libmlx5.map
  1 1.24.${PACKAGE_VERSION}
+ ${TRACE_FILE}
  buf.c
  cq.c
  dbrec.c
@@ -47,3 +52,8 @@ publish_headers(infiniband
 )
 
 rdma_pkg_config("mlx5" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
+
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+ target_include_directories(mlx5 PUBLIC ".")
+ target_link_libraries(mlx5 LINK_PRIVATE LTTng::UST)
+endif()
diff --git a/providers/mlx5/mlx5_trace.c b/providers/mlx5/mlx5_trace.c
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#define LTTNG_UST_TRACEPOINT_CREATE_PROBES
+#define LTTNG_UST_TRACEPOINT_DEFINE
+
+#include "mlx5_trace.h"
diff --git a/providers/mlx5/mlx5_trace.h b/providers/mlx5/mlx5_trace.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#if defined(LTTNG_ENABLED)
+
+#undef LTTNG_UST_TRACEPOINT_PROVIDER
+#define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_mlx5
+
+#undef LTTNG_UST_TRACEPOINT_INCLUDE
+#define LTTNG_UST_TRACEPOINT_INCLUDE "mlx5_trace.h"
+
+#if !defined(__MLX5_TRACE_H__) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ)
+#define __MLX5_TRACE_H__
+
+#include <lttng/tracepoint.h>
+#include <infiniband/verbs.h>
+
+LTTNG_UST_TRACEPOINT_EVENT(
+ /* Tracepoint provider name */
+ rdma_core_mlx5,
+
+ /* Tracepoint name */
+ post_send,
+
+ /* Input arguments */
+ LTTNG_UST_TP_ARGS(
+ char *, dev,
+ uint32_t, src_qp_num,
+ char *, opcode,
+ uint32_t, bytes
+ ),
+
+ /* Output event fields */
+ LTTNG_UST_TP_FIELDS(
+ lttng_ust_field_string(dev, dev)
+ lttng_ust_field_integer(uint32_t, src_qp_num, src_qp_num)
+ lttng_ust_field_string(opcode, opcode)
+ lttng_ust_field_integer(uint32_t, bytes, bytes)
+ )
+)
+
+#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
+
+#endif /* __MLX5_TRACE_H__*/
+
+#include <lttng/tracepoint-event.h>
+
+#else
+
+#ifndef __MLX5_TRACE_H__
+#define __MLX5_TRACE_H__
+
+#define MLX5_TP_rdma_core_mlx5 ""
+#define MLX5_TP_post_send ""
+static inline void dummy_tracepoint(const char *p, const char *n, ...) {};
+
+#define rdma_tracepoint(P, N, arg...) \
+ dummy_tracepoint(MLX5_TP_##P, MLX5_TP_##N, arg)
+
+#endif /* __MLX5_TRACE_H__*/
+
+#endif /* defined(LTTNG_ENABLED) */
diff --git a/providers/mlx5/qp.c b/providers/mlx5/qp.c
@@ -42,6 +42,7 @@
 
 #include "mlx5.h"
 #include "mlx5_ifc.h"
+#include "mlx5_trace.h"
 #include "wqe.h"
 
 #define MLX5_ATOMIC_SIZE 8
@@ -823,6 +824,7 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
  uint8_t fence;
  uint8_t next_fence;
  uint32_t max_tso = 0;
+ unsigned int length = 0;
  FILE *fp = to_mctx(ibqp->context)->dbg_fp; /* The compiler ignores in non-debug mode */
 
  mlx5_spin_lock(&qp->sq.lock);
@@ -1137,6 +1139,14 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
  if (mlx5_debug_mask & MLX5_DBG_QP_SEND)
  dump_wqe(to_mctx(ibqp->context), idx, size, qp);
 #endif
+ for (i = 0; i < wr->num_sge; i++)
+ length += wr->sg_list[i].length;
+
+ rdma_tracepoint(rdma_core_mlx5, post_send,
+ ibqp->context->device->name,
+ ibqp->qp_num,
+ (char *)ibv_wr_opcode_str(wr->opcode),
+ length);
  }
 
 out: