# tcprtt Summarize TCP RTT as a histogram. For Linux, uses BCC, eBPF.
#
# USAGE: tcprtt [-h] [-T] [-D] [-m] [-i INTERVAL] [-d DURATION]
-# [-p LPORT] [-P RPORT] [-a LADDR] [-A RADDR] [-b] [-B]
+# [-p LPORT] [-P RPORT] [-a LADDR] [-A RADDR] [-b] [-B] [-e]
#
# Copyright (c) 2020 zhenwei pi
# Licensed under the Apache License, Version 2.0 (the "License")
from socket import inet_ntop, AF_INET
import socket, struct
import argparse
+import ctypes
# arguments
examples = """examples:
./tcprtt -b # show sockets histogram by local address
./tcprtt -B # show sockets histogram by remote address
./tcprtt -D # show debug bpf text
+ ./tcprtt -e # show extension summary(average)
"""
parser = argparse.ArgumentParser(
description="Summarize TCP RTT as a histogram",
help="show sockets histogram by local address")
parser.add_argument("-B", "--byraddr", action="store_true",
help="show sockets histogram by remote address")
+parser.add_argument("-e", "--extension", action="store_true",
+ help="show extension summary(average)")
parser.add_argument("-D", "--debug", action="store_true",
help="print BPF program before starting (for debugging purposes)")
parser.add_argument("--ebpf", action="store_true",
u64 slot;
} sock_key_t;
-STORAGE
+typedef struct sock_latenty {
+ u64 latency;
+ u64 count;
+} sock_latency_t;
+
+BPF_HISTOGRAM(hist_srtt, sock_key_t);
+BPF_HASH(latency, u64, sock_latency_t);
int trace_tcp_rcv(struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb)
{
struct tcp_sock *ts = tcp_sk(sk);
u32 srtt = ts->srtt_us >> 3;
const struct inet_sock *inet = inet_sk(sk);
+
+ /* filters */
u16 sport = 0;
u16 dport = 0;
u32 saddr = 0;
u32 daddr = 0;
+ /* for histogram */
+ sock_key_t key;
+
+ /* for avg latency, if no saddr/daddr specified, use 0(addr) as key */
+ u64 addr = 0;
+
bpf_probe_read_kernel(&sport, sizeof(sport), (void *)&inet->inet_sport);
bpf_probe_read_kernel(&dport, sizeof(dport), (void *)&inet->inet_dport);
bpf_probe_read_kernel(&saddr, sizeof(saddr), (void *)&inet->inet_saddr);
FACTOR
- STORE
+ STORE_HIST
+ key.slot = bpf_log2l(srtt);
+ hist_srtt.increment(key);
+
+ STORE_LATENCY
return 0;
}
print_header = "srtt"
# show byladdr/byraddr histogram
if args.byladdr:
- bpf_text = bpf_text.replace('STORAGE',
- 'BPF_HISTOGRAM(hist_srtt, sock_key_t);')
- bpf_text = bpf_text.replace('STORE',
- b"""sock_key_t key;
- key.addr = saddr;
- key.slot = bpf_log2l(srtt);
- hist_srtt.increment(key);""")
- print_header = "Local Address: "
+ bpf_text = bpf_text.replace('STORE_HIST', 'key.addr = addr = saddr;')
+ print_header = "Local Address"
elif args.byraddr:
- bpf_text = bpf_text.replace('STORAGE',
- 'BPF_HISTOGRAM(hist_srtt, sock_key_t);')
- bpf_text = bpf_text.replace('STORE',
- b"""sock_key_t key;
- key.addr = daddr;
- key.slot = bpf_log2l(srtt);
- hist_srtt.increment(key);""")
- print_header = "Remote Address: "
+ bpf_text = bpf_text.replace('STORE_HIST', 'key.addr = addr = daddr;')
+ print_header = "Remote Addres"
+else:
+ bpf_text = bpf_text.replace('STORE_HIST', 'key.addr = addr = 0;')
+ print_header = "All Addresses"
+
+if args.extension:
+ bpf_text = bpf_text.replace('STORE_LATENCY', """
+ sock_latency_t newlat = {0};
+ sock_latency_t *lat;
+ lat = latency.lookup(&addr);
+ if (!lat) {
+ newlat.latency += srtt;
+ newlat.count += 1;
+ latency.update(&addr, &newlat);
+ } else {
+ lat->latency +=srtt;
+ lat->count += 1;
+ }
+ """)
else:
- bpf_text = bpf_text.replace('STORAGE', 'BPF_HISTOGRAM(hist_srtt);')
- bpf_text = bpf_text.replace('STORE', 'hist_srtt.increment(bpf_log2l(srtt));')
+ bpf_text = bpf_text.replace('STORE_LATENCY', '')
# debug/dump ebpf enable or not
if args.debug or args.ebpf:
print("Tracing TCP RTT... Hit Ctrl-C to end.")
def print_section(addr):
- if args.byladdr:
- return inet_ntop(AF_INET, struct.pack("I", addr)).encode()
- elif args.byraddr:
- return inet_ntop(AF_INET, struct.pack("I", addr)).encode()
+ addrstr = "*******"
+ if (addr):
+ addrstr = inet_ntop(AF_INET, struct.pack("I", addr)).encode()
+
+ avglat = ""
+ if args.extension:
+ lats = b.get_table("latency")
+ lat = lats[ctypes.c_ulong(addr)]
+ avglat = " [AVG %d]" % (lat.latency / lat.count)
+
+ return addrstr + avglat
# output
exiting = 0 if args.interval else 1
dist = b.get_table("hist_srtt")
+lathash = b.get_table("latency")
seconds = 0
while (1):
try:
dist.print_log2_hist(label, section_header=print_header, section_print_fn=print_section)
dist.clear()
+ lathash.clear()
if exiting or seconds >= args.duration:
exit()
Tracing at server side, show each clients with its own histogram.
For example, run tcprtt on a storage node to show initiators' rtt histogram:
-# ./tcprtt -i 1 -m --lport 3260 --byraddr
+# ./tcprtt -i 1 --lport 3260 --byraddr -e
Tracing TCP RTT... Hit Ctrl-C to end.
-Remote Address: = 10.131.90.16
- msecs : count distribution
+
+Remote Addres = 10.194.87.206 [AVG 170]
+ usecs : count distribution
0 -> 1 : 0 | |
2 -> 3 : 0 | |
4 -> 7 : 0 | |
- 8 -> 15 : 2 |****************************************|
-
-Remote Address: = 10.131.90.13
- msecs : count distribution
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 0 | |
+ 32 -> 63 : 31 | |
+ 64 -> 127 : 5150 |******************* |
+ 128 -> 255 : 10327 |****************************************|
+ 256 -> 511 : 1014 |*** |
+ 512 -> 1023 : 10 | |
+ 1024 -> 2047 : 7 | |
+ 2048 -> 4095 : 14 | |
+ 4096 -> 8191 : 10 | |
+
+Remote Addres = 10.194.87.197 [AVG 4293]
+ usecs : count distribution
+ 0 -> 1 : 0 | |
+ 2 -> 3 : 0 | |
+ 4 -> 7 : 0 | |
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 0 | |
+ 32 -> 63 : 0 | |
+ 64 -> 127 : 0 | |
+ 128 -> 255 : 0 | |
+ 256 -> 511 : 0 | |
+ 512 -> 1023 : 0 | |
+ 1024 -> 2047 : 3 |******** |
+ 2048 -> 4095 : 12 |********************************** |
+ 4096 -> 8191 : 14 |****************************************|
+
+Remote Addres = 10.194.88.148 [AVG 6215]
+ usecs : count distribution
+ 0 -> 1 : 0 | |
+ 2 -> 3 : 0 | |
+ 4 -> 7 : 0 | |
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 0 | |
+ 32 -> 63 : 0 | |
+ 64 -> 127 : 0 | |
+ 128 -> 255 : 0 | |
+ 256 -> 511 : 0 | |
+ 512 -> 1023 : 0 | |
+ 1024 -> 2047 : 0 | |
+ 2048 -> 4095 : 0 | |
+ 4096 -> 8191 : 2 |****************************************|
+
+Remote Addres = 10.194.87.90 [AVG 2188]
+ usecs : count distribution
0 -> 1 : 0 | |
2 -> 3 : 0 | |
- 4 -> 7 : 4 |************************** |
- 8 -> 15 : 6 |****************************************|
+ 4 -> 7 : 0 | |
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 0 | |
+ 32 -> 63 : 0 | |
+ 64 -> 127 : 0 | |
+ 128 -> 255 : 0 | |
+ 256 -> 511 : 15 |********* |
+ 512 -> 1023 : 30 |****************** |
+ 1024 -> 2047 : 50 |****************************** |
+ 2048 -> 4095 : 65 |****************************************|
+ 4096 -> 8191 : 22 |************* |
-Remote Address: = 10.131.89.153
- msecs : count distribution
- 0 -> 1 : 120 |****************************************|
- 2 -> 3 : 31 |********** |
- 4 -> 7 : 32 |********** |
+....
-Remote Address: = 10.131.89.150
- msecs : count distribution
- 0 -> 1 : 12 |****************************************|
- 2 -> 3 : 12 |****************************************|
- 4 -> 7 : 9 |****************************** |
- 8 -> 15 : 3 |********** |
-Remote Address: = 10.131.89.148
- msecs : count distribution
+Use -e(--extension) to show extension RTT:
+# ./tcprtt -i 1 -e
+
+All Addresses = ******* [AVG 324]
+ usecs : count distribution
0 -> 1 : 0 | |
2 -> 3 : 0 | |
- 4 -> 7 : 4 |****************************************|
-
-....
+ 4 -> 7 : 0 | |
+ 8 -> 15 : 0 | |
+ 16 -> 31 : 0 | |
+ 32 -> 63 : 0 | |
+ 64 -> 127 : 5360 |******** |
+ 128 -> 255 : 23834 |****************************************|
+ 256 -> 511 : 11276 |****************** |
+ 512 -> 1023 : 700 |* |
+ 1024 -> 2047 : 434 | |
+ 2048 -> 4095 : 356 | |
+ 4096 -> 8191 : 328 | |
+ 8192 -> 16383 : 91 | |
Full USAGE:
# ./tcprtt -h
-usage: tcprtt.py [-h] [-i INTERVAL] [-d DURATION] [-T] [-m] [-p LPORT]
- [-P RPORT] [-a LADDR] [-A RADDR] [-b] [-B] [-D]
+usage: tcprtt [-h] [-i INTERVAL] [-d DURATION] [-T] [-m] [-p LPORT]
+ [-P RPORT] [-a LADDR] [-A RADDR] [-b] [-B] [-e] [-D]
Summarize TCP RTT as a histogram
filter for remote address
-b, --byladdr show sockets histogram by local address
-B, --byraddr show sockets histogram by remote address
+ -e, --extension show extension summary(average)
-D, --debug print BPF program before starting (for debugging
purposes)
./tcprtt -b # show sockets histogram by local address
./tcprtt -B # show sockets histogram by remote address
./tcprtt -D # show debug bpf text
+ ./tcprtt -e # show extension summary(average)
projects / platform / upstream / bcc.git / commitdiff