diff --git a/man/man8/tcprtt.8 b/man/man8/tcprtt.8 new file mode 100644 index 000000000000..5c3bb89b9ca3 --- /dev/null +++ b/man/man8/tcprtt.8 @@ -0,0 +1,76 @@ +.TH tcprtt 8 "2020-08-23" "USER COMMANDS" +.SH NAME +tcprtt \- Trace TCP RTT of established connections. Uses Linux eBPF/bcc. +.SH SYNOPSIS +.B tcprtt [\-h] [\-T] [\-D] [\-m] [\-i INTERVAL] [\-d DURATION] +.SH DESCRIPTION +This tool traces established connections RTT(round-trip time) to analyze the +quality of network. This can be useful for general troubleshooting to +distinguish the network latency is from user process or physical network. + +Since this uses BPF, only the root user can use this tool. +.SH REQUIREMENTS +CONFIG_BPF and bcc. +.SH OPTIONS +.TP +\-h +Print usage message. +.TP +\-T +Include a time column on output (HH:MM:SS). +.TP +\-D +Show debug infomation of bpf text. +.TP +\-m +Output histogram in milliseconds. +.TP +\-i INTERVAL +Print output every interval seconds. +.TP +\-d DURATION +Total duration of trace in seconds. +.TP +\-p SPORT +Filter for source port. +.TP +\-P DPORT +Filter for destination port. +.TP +\-a SADDR +Filter for source address. +.TP +\-A DADDR +Filter for destination address. +.SH EXAMPLES +.TP +Trace TCP RTT and print 1 second summaries, 10 times: +# +.B tcprtt \-i 1 \-d 10 +.TP +Summarize in millisecond, and timestamps: +# +.B tcprtt \-m \-T +.TP +Only trace TCP RTT for destination address 192.168.1.100 and destination port 80: +# +.B tcprtt \-i 1 \-d 10 -A 192.168.1.100 -P 80 +.SH OVERHEAD +This traces the kernel tcp_rcv_established function and collects TCP RTT. The +rate of this depends on your server application. If it is a web or proxy server +accepting many tens of thousands of connections per second. +.SH SOURCE +This is from bcc. +.IP +https://github.com/iovisor/bcc +.PP +Also look in the bcc distribution for a companion _examples.txt file containing +example usage, output, and commentary for this tool. +.SH OS +Linux +.SH STABILITY +Unstable - in development. +.SH AUTHOR +zhenwei pi +.SH SEE ALSO +tcptracer(8), tcpconnect(8), funccount(8), tcpdump(8) diff --git a/tools/tcprtt.py b/tools/tcprtt.py new file mode 100755 index 000000000000..81832cf0f81c --- /dev/null +++ b/tools/tcprtt.py @@ -0,0 +1,169 @@ +#!/usr/bin/python +# @lint-avoid-python-3-compatibility-imports +# +# tcprtt Summarize TCP RTT as a histogram. For Linux, uses BCC, eBPF. +# +# USAGE: tcprtt [-h] [-T] [-D] [-m] [-i INTERVAL] [-d DURATION] +# [-p SPORT] [-P DPORT] [-a SADDR] [-A DADDR] +# +# Copyright (c) 2020 zhenwei pi +# Licensed under the Apache License, Version 2.0 (the "License") +# +# 23-AUG-2020 zhenwei pi Created this. + +from __future__ import print_function +from bcc import BPF +from time import sleep, strftime +import socket, struct +import argparse + +# arguments +examples = """examples: + ./tcprtt # summarize TCP RTT + ./tcprtt -i 1 -d 10 # print 1 second summaries, 10 times + ./tcprtt -m -T # summarize in millisecond, and timestamps + ./tcprtt -p # filter for source port + ./tcprtt -P # filter for destination port + ./tcprtt -a # filter for source address + ./tcprtt -A # filter for destination address + ./tcprtt -D # show debug bpf text +""" +parser = argparse.ArgumentParser( + description="Summarize TCP RTT as a histogram", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) +parser.add_argument("-i", "--interval", + help="summary interval, seconds") +parser.add_argument("-d", "--duration", type=int, default=99999, + help="total duration of trace, seconds") +parser.add_argument("-T", "--timestamp", action="store_true", + help="include timestamp on output") +parser.add_argument("-m", "--milliseconds", action="store_true", + help="millisecond histogram") +parser.add_argument("-p", "--sport", + help="source port") +parser.add_argument("-P", "--dport", + help="destination port") +parser.add_argument("-a", "--saddr", + help="source address") +parser.add_argument("-A", "--daddr", + help="destination address") +parser.add_argument("-D", "--debug", action="store_true", + help="print BPF program before starting (for debugging purposes)") +parser.add_argument("--ebpf", action="store_true", + help=argparse.SUPPRESS) +args = parser.parse_args() +if not args.interval: + args.interval = args.duration + +# define BPF program +bpf_text = """ +#ifndef KBUILD_MODNAME +#define KBUILD_MODNAME "bcc" +#endif +#include +#include +#include +#include +#include + +BPF_HISTOGRAM(hist_srtt); + +int trace_tcp_rcv(struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb) +{ + struct tcp_sock *ts = tcp_sk(sk); + u32 srtt = ts->srtt_us >> 3; + const struct inet_sock *inet = inet_sk(sk); + + SPORTFILTER + DPORTFILTER + SADDRFILTER + DADDRFILTER + FACTOR + + hist_srtt.increment(bpf_log2l(srtt)); + + return 0; +} +""" + +# filter for source port +if args.sport: + bpf_text = bpf_text.replace(b'SPORTFILTER', + b"""u16 sport = 0; + bpf_probe_read_kernel(&sport, sizeof(sport), (void *)&inet->inet_sport); + if (ntohs(sport) != %d) + return 0;""" % int(args.sport)) +else: + bpf_text = bpf_text.replace(b'SPORTFILTER', b'') + +# filter for dest port +if args.dport: + bpf_text = bpf_text.replace(b'DPORTFILTER', + b"""u16 dport = 0; + bpf_probe_read_kernel(&dport, sizeof(dport), (void *)&inet->inet_dport); + if (ntohs(dport) != %d) + return 0;""" % int(args.dport)) +else: + bpf_text = bpf_text.replace(b'DPORTFILTER', b'') + +# filter for source address +if args.saddr: + bpf_text = bpf_text.replace(b'SADDRFILTER', + b"""u32 saddr = 0; + bpf_probe_read_kernel(&saddr, sizeof(saddr), (void *)&inet->inet_saddr); + if (saddr != %d) + return 0;""" % struct.unpack("=I", socket.inet_aton(args.saddr))[0]) +else: + bpf_text = bpf_text.replace(b'SADDRFILTER', b'') + +# filter for source address +if args.daddr: + bpf_text = bpf_text.replace(b'DADDRFILTER', + b"""u32 daddr = 0; + bpf_probe_read_kernel(&daddr, sizeof(daddr), (void *)&inet->inet_daddr); + if (daddr != %d) + return 0;""" % struct.unpack("=I", socket.inet_aton(args.daddr))[0]) +else: + bpf_text = bpf_text.replace(b'DADDRFILTER', b'') + +# show msecs or usecs[default] +if args.milliseconds: + bpf_text = bpf_text.replace('FACTOR', 'srtt /= 1000;') + label = "msecs" +else: + bpf_text = bpf_text.replace('FACTOR', '') + label = "usecs" + +# debug/dump ebpf enable or not +if args.debug or args.ebpf: + print(bpf_text) + if args.ebpf: + exit() + +# load BPF program +b = BPF(text=bpf_text) +b.attach_kprobe(event="tcp_rcv_established", fn_name="trace_tcp_rcv") + +print("Tracing TCP RTT... Hit Ctrl-C to end.") + +# output +exiting = 0 if args.interval else 1 +dist = b.get_table("hist_srtt") +seconds = 0 +while (1): + try: + sleep(int(args.interval)) + seconds = seconds + int(args.interval) + except KeyboardInterrupt: + exiting = 1 + + print() + if args.timestamp: + print("%-8s\n" % strftime("%H:%M:%S"), end="") + + dist.print_log2_hist(label, "srtt") + dist.clear() + + if exiting or seconds >= args.duration: + exit() diff --git a/tools/tcprtt_example.txt b/tools/tcprtt_example.txt new file mode 100644 index 000000000000..9a3d2356b327 --- /dev/null +++ b/tools/tcprtt_example.txt @@ -0,0 +1,83 @@ +Demonstrations of tcprtt, the Linux eBPF/bcc version. + + +This program traces TCP RTT(round-trip time) to analyze the quality of +network, then help us to distinguish the network latency trouble is from +user process or physical network. + +For example, wrk show the http request latency distribution: +# wrk -d 30 -c 10 --latency http://192.168.122.100/index.html +Running 30s test @ http://192.168.122.100/index.html + 2 threads and 10 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 86.75ms 153.76ms 1.54s 90.85% + Req/Sec 160.91 76.07 424.00 67.06% + Latency Distribution + 50% 14.55ms + 75% 119.21ms + 90% 230.22ms + 99% 726.90ms + 9523 requests in 30.02s, 69.62MB read + Socket errors: connect 0, read 0, write 0, timeout 1 + +During wrk testing, run tcprtt: +# ./tcprtt -i 1 -d 10 -m +Tracing TCP RTT... Hit Ctrl-C to end. + msecs : count distribution + 0 -> 1 : 4 | | + 2 -> 3 : 0 | | + 4 -> 7 : 1055 |****************************************| + 8 -> 15 : 26 | | + 16 -> 31 : 0 | | + 32 -> 63 : 0 | | + 64 -> 127 : 18 | | + 128 -> 255 : 14 | | + 256 -> 511 : 14 | | + 512 -> 1023 : 12 | | + +The wrk output shows that the latency of web service is not stable, and tcprtt +also shows unstable TCP RTT. So in this situation, we need to make sure the +quality of network is good or not firstly. + + +Use filter for address and(or) port. Ex, only collect source address 192.168.122.200 +and destination address 192.168.122.100 and destination port 80. +# ./tcprtt -i 1 -d 10 -m -a 192.168.122.200 -A 192.168.122.100 -P 80 + + +Full USAGE: + +# ./tcprtt -h +usage: tcprtt [-h] [-i INTERVAL] [-d DURATION] [-T] [-m] [-p SPORT] + [-P DPORT] [-a SADDR] [-A DADDR] [-D] + +Summarize TCP RTT as a histogram + +optional arguments: + -h, --help show this help message and exit + -i INTERVAL, --interval INTERVAL + summary interval, seconds + -d DURATION, --duration DURATION + total duration of trace, seconds + -T, --timestamp include timestamp on output + -m, --milliseconds millisecond histogram + -p SPORT, --sport SPORT + source port + -P DPORT, --dport DPORT + destination port + -a SADDR, --saddr SADDR + source address + -A DADDR, --daddr DADDR + destination address + -D, --debug print BPF program before starting (for debugging + purposes) + +examples: + ./tcprtt # summarize TCP RTT + ./tcprtt -i 1 -d 10 # print 1 second summaries, 10 times + ./tcprtt -m -T # summarize in millisecond, and timestamps + ./tcprtt -p # filter for source port + ./tcprtt -P # filter for destination port + ./tcprtt -a # filter for source address + ./tcprtt -A # filter for destination address + ./tcprtt -D # show debug bpf text