providers/rxe: Add tracepoint for post send

Add initial support for RXE LTTng tracing using compiled out macros if
LTTng flag isn't provided.

Once LTTng records tracepoints, an example of python script to analyze
per-qp traffic:

 #!/usr/bin/env python3
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
 # Copyright (c) 2023, Bytedance. All rights reserved.  See COPYING file

import argparse
import bt2
import collections
import sys

def stats(args):
	stats = {}

	tcm_it = bt2.TraceCollectionMessageIterator(args.tracepoints)
	for msg in tcm_it:
		if type(msg) is not bt2._EventMessageConst:
			continue

		event = msg.event
		if event.cls.name != "rdma_core_rxe:post_send":
			continue

		# device filter
		dev = str(event.payload_field["dev"])
		if args.device != None and args.device != dev:
			continue;

		# QPN filter
		qpn = event.payload_field['src_qp_num']
		if args.qpn != None and args.qpn != qpn:
			continue;

		# opcode filter
		opcode = event.payload_field['opcode']
		if args.opcode != None and args.opcode != opcode:
			continue;

		key = dev + "-" + str(qpn)
		qpstats = stats.get(key)
		if qpstats is None:
			qpstats = {}
			qpstats["dev"] = dev
			qpstats["qpn"] = qpn
			qpstats["bytes"] = 0
			qpstats["ops"] = 0
			stats[key] = qpstats

		bytes = event.payload_field['bytes']
		qpstats["bytes"] += bytes
		qpstats["ops"] += 1

		qpstat = qpstats.get(opcode)
		if qpstat is None:
			qpstats[opcode] = bytes
		else:
			qpstats[opcode] += bytes

	# sorted by total bytes of a QP
	sorted_stats = sorted(stats.items(),
		key=lambda item: (item[1].get("bytes", 0)),
		reverse=True)

	# show header
	print("%16s %10s %10s %10s" % ("Device", "QPN", "Bytes", "Operations"))

	# show QP stats
	for qpstats in sorted_stats:
		stats = qpstats[1]
		print("%16s %10d %10d %10d" %
			(stats["dev"], stats["qpn"], stats["bytes"], stats["ops"]))

def parse_arg():
	examples = """LTTng tracing examples:
	# lttng create rdma-core-stats-session
	# lttng list --userspace
	# lttng enable-event --userspace rdma_core_ibverbs:post_send
	# lttng start
	# ...
	# lttng destroy
	# ibv_stats -t /path/to/lttng/tracepoints
	"""

	parser = argparse.ArgumentParser(
			description="Summarize IB verbs statistics",
			formatter_class=argparse.RawDescriptionHelpFormatter,
			epilog=examples)

	parser.add_argument("-t", "--tracepoints",
		help="specify the LTTng tracepoints directory")

	parser.add_argument("-d", "--device",
		help="specify IB device filter")

	parser.add_argument("-q", "--qpn", type=int,
		help="specify source QP number filter")

	parser.add_argument("-o", "--opcode",
 		help="specify opcode filter")

	return parser.parse_args()

if __name__ == '__main__':
	args = parse_arg()

	if args.tracepoints is None:
		print("Missing LTTng tracepoints directory(use -t/--tracepoints)")
		sys.exit(0)

	stats(args)

Signed-off-by: zhenwei pi <[email protected]>

providers/rxe/CMakeLists.txt

@@ -1,3 +1,13 @@
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+ set(TRACE_FILE rxe_trace.c)
+endif()
+
 rdma_provider(rxe
+ ${TRACE_FILE}
  rxe.c
  )
+
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+ target_include_directories("rxe-rdmav${IBVERBS_PABI_VERSION}" PUBLIC ".")
+ target_link_libraries("rxe-rdmav${IBVERBS_PABI_VERSION}" LINK_PRIVATE LTTng::UST)
+endif()

providers/rxe/rxe.c

@@ -55,6 +55,7 @@
 #include "rxe_queue.h"
 #include "rxe-abi.h"
 #include "rxe.h"
+#include "rxe_trace.h"
 
 static void rxe_free_context(struct ibv_context *ibctx);
 
@@ -1619,6 +1620,11 @@ static int post_one_send(struct rxe_qp *qp, struct rxe_wq *sq,
  return ENOMEM;
 
  advance_producer(sq->queue);
+ rdma_tracepoint(rdma_core_rxe, post_send,
+ qp->vqp.qp.context->device->name,
+ qp->vqp.qp.qp_num,
+ (char *)ibv_wr_opcode_str(ibwr->opcode),
+ length);
 
  return 0;
 }

providers/rxe/rxe_trace.c

@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#define LTTNG_UST_TRACEPOINT_CREATE_PROBES
+#define LTTNG_UST_TRACEPOINT_DEFINE
+
+#include "rxe_trace.h"

providers/rxe/rxe_trace.h

@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#if defined(LTTNG_ENABLED)
+
+#undef LTTNG_UST_TRACEPOINT_PROVIDER
+#define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_rxe
+
+#undef LTTNG_UST_TRACEPOINT_INCLUDE
+#define LTTNG_UST_TRACEPOINT_INCLUDE "rxe_trace.h"
+
+#if !defined(__RXE_TRACE_H__) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ)
+#define __RXE_TRACE_H__
+
+#include <lttng/tracepoint.h>
+#include <infiniband/verbs.h>
+
+LTTNG_UST_TRACEPOINT_EVENT(
+ /* Tracepoint provider name */
+ rdma_core_rxe,
+
+ /* Tracepoint name */
+ post_send,
+
+ /* Input arguments */
+ LTTNG_UST_TP_ARGS(
+ char *, dev,
+ uint32_t, src_qp_num,
+ char *, opcode,
+ uint32_t, bytes
+ ),
+
+ /* Output event fields */
+ LTTNG_UST_TP_FIELDS(
+ lttng_ust_field_string(dev, dev)
+ lttng_ust_field_integer(uint32_t, src_qp_num, src_qp_num)
+ lttng_ust_field_string(opcode, opcode)
+ lttng_ust_field_integer(uint32_t, bytes, bytes)
+ )
+)
+
+#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
+
+#endif /* __RXE_TRACE_H__*/
+
+#include <lttng/tracepoint-event.h>
+
+#else
+
+#ifndef __RXE_TRACE_H__
+#define __RXE_TRACE_H__
+
+#define rdma_tracepoint(arg...)
+
+#endif /* __RXE_TRACE_H__*/
+
+#endif /* defined(LTTNG_ENABLED) */