1#!/usr/bin/python 2# @lint-avoid-python-3-compatibility-imports 3# 4# tcpconnect Trace TCP connect()s. 5# For Linux, uses BCC, eBPF. Embedded C. 6# 7# USAGE: tcpconnect [-h] [-c] [-t] [-p PID] [-P PORT [PORT ...]] [-4 | -6] 8# 9# All connection attempts are traced, even if they ultimately fail. 10# 11# This uses dynamic tracing of kernel functions, and will need to be updated 12# to match kernel changes. 13# 14# Copyright (c) 2015 Brendan Gregg. 15# Licensed under the Apache License, Version 2.0 (the "License") 16# 17# 25-Sep-2015 Brendan Gregg Created this. 18# 14-Feb-2016 " " Switch to bpf_perf_output. 19# 09-Jan-2019 Takuma Kume Support filtering by UID 20# 30-Jul-2019 Xiaozhou Liu Count connects. 21# 07-Oct-2020 Nabil Schear Correlate connects with DNS responses 22# 08-Mar-2021 Suresh Kumar Added LPORT option 23 24from __future__ import print_function 25from bcc import BPF 26from bcc.containers import filter_by_containers 27from bcc.utils import printb 28import argparse 29from socket import inet_ntop, ntohs, AF_INET, AF_INET6 30from struct import pack 31from time import sleep 32from datetime import datetime 33 34# arguments 35examples = """examples: 36 ./tcpconnect # trace all TCP connect()s 37 ./tcpconnect -t # include timestamps 38 ./tcpconnect -d # include DNS queries associated with connects 39 ./tcpconnect -p 181 # only trace PID 181 40 ./tcpconnect -P 80 # only trace port 80 41 ./tcpconnect -P 80,81 # only trace port 80 and 81 42 ./tcpconnect -4 # only trace IPv4 family 43 ./tcpconnect -6 # only trace IPv6 family 44 ./tcpconnect -U # include UID 45 ./tcpconnect -u 1000 # only trace UID 1000 46 ./tcpconnect -c # count connects per src ip and dest ip/port 47 ./tcpconnect -L # include LPORT while printing outputs 48 ./tcpconnect --cgroupmap mappath # only trace cgroups in this BPF map 49 ./tcpconnect --mntnsmap mappath # only trace mount namespaces in the map 50""" 51parser = argparse.ArgumentParser( 52 description="Trace TCP connects", 53 formatter_class=argparse.RawDescriptionHelpFormatter, 54 epilog=examples) 55parser.add_argument("-t", "--timestamp", action="store_true", 56 help="include timestamp on output") 57parser.add_argument("-p", "--pid", 58 help="trace this PID only") 59parser.add_argument("-P", "--port", 60 help="comma-separated list of destination ports to trace.") 61group = parser.add_mutually_exclusive_group() 62group.add_argument("-4", "--ipv4", action="store_true", 63 help="trace IPv4 family only") 64group.add_argument("-6", "--ipv6", action="store_true", 65 help="trace IPv6 family only") 66parser.add_argument("-L", "--lport", action="store_true", 67 help="include LPORT on output") 68parser.add_argument("-U", "--print-uid", action="store_true", 69 help="include UID on output") 70parser.add_argument("-u", "--uid", 71 help="trace this UID only") 72parser.add_argument("-c", "--count", action="store_true", 73 help="count connects per src ip and dest ip/port") 74parser.add_argument("--cgroupmap", 75 help="trace cgroups in this BPF map only") 76parser.add_argument("--mntnsmap", 77 help="trace mount namespaces in this BPF map only") 78parser.add_argument("-d", "--dns", action="store_true", 79 help="include likely DNS query associated with each connect") 80parser.add_argument("--ebpf", action="store_true", 81 help=argparse.SUPPRESS) 82args = parser.parse_args() 83debug = 0 84 85# define BPF program 86bpf_text = """ 87#include <uapi/linux/ptrace.h> 88#include <net/sock.h> 89#include <bcc/proto.h> 90 91BPF_HASH(currsock, u32, struct sock *); 92 93// separate data structs for ipv4 and ipv6 94struct ipv4_data_t { 95 u64 ts_us; 96 u32 pid; 97 u32 uid; 98 u32 saddr; 99 u32 daddr; 100 u64 ip; 101 u16 lport; 102 u16 dport; 103 char task[TASK_COMM_LEN]; 104}; 105BPF_PERF_OUTPUT(ipv4_events); 106 107struct ipv6_data_t { 108 u64 ts_us; 109 u32 pid; 110 u32 uid; 111 unsigned __int128 saddr; 112 unsigned __int128 daddr; 113 u64 ip; 114 u16 lport; 115 u16 dport; 116 char task[TASK_COMM_LEN]; 117}; 118BPF_PERF_OUTPUT(ipv6_events); 119 120// separate flow keys per address family 121struct ipv4_flow_key_t { 122 u32 saddr; 123 u32 daddr; 124 u16 dport; 125}; 126BPF_HASH(ipv4_count, struct ipv4_flow_key_t); 127 128struct ipv6_flow_key_t { 129 unsigned __int128 saddr; 130 unsigned __int128 daddr; 131 u16 dport; 132}; 133BPF_HASH(ipv6_count, struct ipv6_flow_key_t); 134 135int trace_connect_entry(struct pt_regs *ctx, struct sock *sk) 136{ 137 if (container_should_be_filtered()) { 138 return 0; 139 } 140 141 u64 pid_tgid = bpf_get_current_pid_tgid(); 142 u32 pid = pid_tgid >> 32; 143 u32 tid = pid_tgid; 144 FILTER_PID 145 146 u32 uid = bpf_get_current_uid_gid(); 147 FILTER_UID 148 149 // stash the sock ptr for lookup on return 150 currsock.update(&tid, &sk); 151 152 return 0; 153}; 154 155static int trace_connect_return(struct pt_regs *ctx, short ipver) 156{ 157 int ret = PT_REGS_RC(ctx); 158 u64 pid_tgid = bpf_get_current_pid_tgid(); 159 u32 pid = pid_tgid >> 32; 160 u32 tid = pid_tgid; 161 162 struct sock **skpp; 163 skpp = currsock.lookup(&tid); 164 if (skpp == 0) { 165 return 0; // missed entry 166 } 167 168 if (ret != 0) { 169 // failed to send SYNC packet, may not have populated 170 // socket __sk_common.{skc_rcv_saddr, ...} 171 currsock.delete(&tid); 172 return 0; 173 } 174 175 // pull in details 176 struct sock *skp = *skpp; 177 u16 lport = skp->__sk_common.skc_num; 178 u16 dport = skp->__sk_common.skc_dport; 179 180 FILTER_PORT 181 182 FILTER_FAMILY 183 184 if (ipver == 4) { 185 IPV4_CODE 186 } else /* 6 */ { 187 IPV6_CODE 188 } 189 190 currsock.delete(&tid); 191 192 return 0; 193} 194 195int trace_connect_v4_return(struct pt_regs *ctx) 196{ 197 return trace_connect_return(ctx, 4); 198} 199 200int trace_connect_v6_return(struct pt_regs *ctx) 201{ 202 return trace_connect_return(ctx, 6); 203} 204""" 205 206struct_init = {'ipv4': 207 {'count': 208 """ 209 struct ipv4_flow_key_t flow_key = {}; 210 flow_key.saddr = skp->__sk_common.skc_rcv_saddr; 211 flow_key.daddr = skp->__sk_common.skc_daddr; 212 flow_key.dport = ntohs(dport); 213 ipv4_count.increment(flow_key);""", 214 'trace': 215 """ 216 struct ipv4_data_t data4 = {.pid = pid, .ip = ipver}; 217 data4.uid = bpf_get_current_uid_gid(); 218 data4.ts_us = bpf_ktime_get_ns() / 1000; 219 data4.saddr = skp->__sk_common.skc_rcv_saddr; 220 data4.daddr = skp->__sk_common.skc_daddr; 221 data4.lport = lport; 222 data4.dport = ntohs(dport); 223 bpf_get_current_comm(&data4.task, sizeof(data4.task)); 224 ipv4_events.perf_submit(ctx, &data4, sizeof(data4));""" 225 }, 226 'ipv6': 227 {'count': 228 """ 229 struct ipv6_flow_key_t flow_key = {}; 230 bpf_probe_read_kernel(&flow_key.saddr, sizeof(flow_key.saddr), 231 skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 232 bpf_probe_read_kernel(&flow_key.daddr, sizeof(flow_key.daddr), 233 skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 234 flow_key.dport = ntohs(dport); 235 ipv6_count.increment(flow_key);""", 236 'trace': 237 """ 238 struct ipv6_data_t data6 = {.pid = pid, .ip = ipver}; 239 data6.uid = bpf_get_current_uid_gid(); 240 data6.ts_us = bpf_ktime_get_ns() / 1000; 241 bpf_probe_read_kernel(&data6.saddr, sizeof(data6.saddr), 242 skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 243 bpf_probe_read_kernel(&data6.daddr, sizeof(data6.daddr), 244 skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 245 data6.lport = lport; 246 data6.dport = ntohs(dport); 247 bpf_get_current_comm(&data6.task, sizeof(data6.task)); 248 ipv6_events.perf_submit(ctx, &data6, sizeof(data6));""" 249 } 250 } 251 252# This defines an additional BPF program that instruments udp_recvmsg system 253# call to locate DNS response packets on UDP port 53. When these packets are 254# located, the data is copied to user-space where python will parse them with 255# dnslib. 256# 257# uses a percpu array of length 1 to store the dns_data_t off the stack to 258# allow for a maximum DNS packet length of 512 bytes. 259dns_bpf_text = """ 260#include <net/inet_sock.h> 261 262#define MAX_PKT 512 263struct dns_data_t { 264 u8 pkt[MAX_PKT]; 265}; 266 267BPF_PERF_OUTPUT(dns_events); 268 269// store msghdr pointer captured on syscall entry to parse on syscall return 270BPF_HASH(tbl_udp_msg_hdr, u64, struct msghdr *); 271 272// single element per-cpu array to hold the current event off the stack 273BPF_PERCPU_ARRAY(dns_data,struct dns_data_t,1); 274 275int trace_udp_recvmsg(struct pt_regs *ctx) 276{ 277 __u64 pid_tgid = bpf_get_current_pid_tgid(); 278 struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); 279 struct inet_sock *is = inet_sk(sk); 280 281 // only grab port 53 packets, 13568 is ntohs(53) 282 if (is->inet_dport == 13568) { 283 struct msghdr *msghdr = (struct msghdr *)PT_REGS_PARM2(ctx); 284 tbl_udp_msg_hdr.update(&pid_tgid, &msghdr); 285 } 286 return 0; 287} 288 289int trace_udp_ret_recvmsg(struct pt_regs *ctx) 290{ 291 __u64 pid_tgid = bpf_get_current_pid_tgid(); 292 u32 zero = 0; 293 struct msghdr **msgpp = tbl_udp_msg_hdr.lookup(&pid_tgid); 294 if (msgpp == 0) 295 return 0; 296 297 struct msghdr *msghdr = (struct msghdr *)*msgpp; 298 if (msghdr->msg_iter.TYPE_FIELD != ITER_IOVEC) 299 goto delete_and_return; 300 301 int copied = (int)PT_REGS_RC(ctx); 302 if (copied < 0) 303 goto delete_and_return; 304 size_t buflen = (size_t)copied; 305 306 if (buflen > msghdr->msg_iter.iov->iov_len) 307 goto delete_and_return; 308 309 if (buflen > MAX_PKT) 310 buflen = MAX_PKT; 311 312 struct dns_data_t *data = dns_data.lookup(&zero); 313 if (!data) // this should never happen, just making the verifier happy 314 return 0; 315 316 void *iovbase = msghdr->msg_iter.iov->iov_base; 317 bpf_probe_read(data->pkt, buflen, iovbase); 318 dns_events.perf_submit(ctx, data, buflen); 319 320delete_and_return: 321 tbl_udp_msg_hdr.delete(&pid_tgid); 322 return 0; 323} 324 325""" 326 327if args.count and args.dns: 328 print("Error: you may not specify -d/--dns with -c/--count.") 329 exit() 330 331# code substitutions 332if args.count: 333 bpf_text = bpf_text.replace("IPV4_CODE", struct_init['ipv4']['count']) 334 bpf_text = bpf_text.replace("IPV6_CODE", struct_init['ipv6']['count']) 335else: 336 bpf_text = bpf_text.replace("IPV4_CODE", struct_init['ipv4']['trace']) 337 bpf_text = bpf_text.replace("IPV6_CODE", struct_init['ipv6']['trace']) 338 339if args.pid: 340 bpf_text = bpf_text.replace('FILTER_PID', 341 'if (pid != %s) { return 0; }' % args.pid) 342if args.port: 343 dports = [int(dport) for dport in args.port.split(',')] 344 dports_if = ' && '.join(['dport != %d' % ntohs(dport) for dport in dports]) 345 bpf_text = bpf_text.replace('FILTER_PORT', 346 'if (%s) { currsock.delete(&tid); return 0; }' % dports_if) 347if args.ipv4: 348 bpf_text = bpf_text.replace('FILTER_FAMILY', 349 'if (ipver != 4) { return 0; }') 350elif args.ipv6: 351 bpf_text = bpf_text.replace('FILTER_FAMILY', 352 'if (ipver != 6) { return 0; }') 353if args.uid: 354 bpf_text = bpf_text.replace('FILTER_UID', 355 'if (uid != %s) { return 0; }' % args.uid) 356bpf_text = filter_by_containers(args) + bpf_text 357 358bpf_text = bpf_text.replace('FILTER_PID', '') 359bpf_text = bpf_text.replace('FILTER_PORT', '') 360bpf_text = bpf_text.replace('FILTER_FAMILY', '') 361bpf_text = bpf_text.replace('FILTER_UID', '') 362 363if args.dns: 364 if BPF.kernel_struct_has_field(b'iov_iter', b'iter_type') == 1: 365 dns_bpf_text = dns_bpf_text.replace('TYPE_FIELD', 'iter_type') 366 else: 367 dns_bpf_text = dns_bpf_text.replace('TYPE_FIELD', 'type') 368 bpf_text += dns_bpf_text 369 370if debug or args.ebpf: 371 print(bpf_text) 372 if args.ebpf: 373 exit() 374 375# process event 376def print_ipv4_event(cpu, data, size): 377 event = b["ipv4_events"].event(data) 378 global start_ts 379 if args.timestamp: 380 if start_ts == 0: 381 start_ts = event.ts_us 382 printb(b"%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), nl="") 383 if args.print_uid: 384 printb(b"%-6d" % event.uid, nl="") 385 dest_ip = inet_ntop(AF_INET, pack("I", event.daddr)).encode() 386 if args.lport: 387 printb(b"%-7d %-12.12s %-2d %-16s %-6d %-16s %-6d %s" % (event.pid, 388 event.task, event.ip, 389 inet_ntop(AF_INET, pack("I", event.saddr)).encode(), event.lport, 390 dest_ip, event.dport, print_dns(dest_ip))) 391 else: 392 printb(b"%-7d %-12.12s %-2d %-16s %-16s %-6d %s" % (event.pid, 393 event.task, event.ip, 394 inet_ntop(AF_INET, pack("I", event.saddr)).encode(), 395 dest_ip, event.dport, print_dns(dest_ip))) 396 397def print_ipv6_event(cpu, data, size): 398 event = b["ipv6_events"].event(data) 399 global start_ts 400 if args.timestamp: 401 if start_ts == 0: 402 start_ts = event.ts_us 403 printb(b"%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), nl="") 404 if args.print_uid: 405 printb(b"%-6d" % event.uid, nl="") 406 dest_ip = inet_ntop(AF_INET6, event.daddr).encode() 407 if args.lport: 408 printb(b"%-7d %-12.12s %-2d %-16s %-6d %-16s %-6d %s" % (event.pid, 409 event.task, event.ip, 410 inet_ntop(AF_INET6, event.saddr).encode(), event.lport, 411 dest_ip, event.dport, print_dns(dest_ip))) 412 else: 413 printb(b"%-7d %-12.12s %-2d %-16s %-16s %-6d %s" % (event.pid, 414 event.task, event.ip, 415 inet_ntop(AF_INET6, event.saddr).encode(), 416 dest_ip, event.dport, print_dns(dest_ip))) 417 418def depict_cnt(counts_tab, l3prot='ipv4'): 419 for k, v in sorted(counts_tab.items(), 420 key=lambda counts: counts[1].value, reverse=True): 421 depict_key = "" 422 if l3prot == 'ipv4': 423 depict_key = "%-25s %-25s %-20s" % \ 424 ((inet_ntop(AF_INET, pack('I', k.saddr))), 425 inet_ntop(AF_INET, pack('I', k.daddr)), k.dport) 426 else: 427 depict_key = "%-25s %-25s %-20s" % \ 428 ((inet_ntop(AF_INET6, k.saddr)), 429 inet_ntop(AF_INET6, k.daddr), k.dport) 430 431 print("%s %-10d" % (depict_key, v.value)) 432 433def print_dns(dest_ip): 434 if not args.dns: 435 return b"" 436 437 dnsname, timestamp = dns_cache.get(dest_ip, (None, None)) 438 if timestamp is not None: 439 diff = datetime.now() - timestamp 440 diff = float(diff.seconds) * 1000 + float(diff.microseconds) / 1000 441 else: 442 diff = 0 443 if dnsname is None: 444 dnsname = b"No DNS Query" 445 if dest_ip == b"127.0.0.1" or dest_ip == b"::1": 446 dnsname = b"localhost" 447 retval = b"%s" % dnsname 448 if diff > DELAY_DNS: 449 retval += b" (%.3fms)" % diff 450 return retval 451 452if args.dns: 453 try: 454 import dnslib 455 from cachetools import TTLCache 456 except ImportError: 457 print("Error: The python packages dnslib and cachetools are required " 458 "to use the -d/--dns option.") 459 print("Install this package with:") 460 print("\t$ pip3 install dnslib cachetools") 461 print(" or") 462 print("\t$ sudo apt-get install python3-dnslib python3-cachetools " 463 "(on Ubuntu 18.04+)") 464 exit(1) 465 466 # 24 hours 467 DEFAULT_TTL = 86400 468 469 # Cache Size in entries 470 DNS_CACHE_SIZE = 10240 471 472 # delay in ms in which to warn users of long delay between the query 473 # and the connect that used the IP 474 DELAY_DNS = 100 475 476 dns_cache = TTLCache(maxsize=DNS_CACHE_SIZE, ttl=DEFAULT_TTL) 477 478 # process event 479 def save_dns(cpu, data, size): 480 event = b["dns_events"].event(data) 481 payload = event.pkt[:size] 482 483 # pass the payload to dnslib for parsing 484 dnspkt = dnslib.DNSRecord.parse(payload) 485 # lets only look at responses 486 if dnspkt.header.qr != 1: 487 return 488 # must be some questions in there 489 if dnspkt.header.q != 1: 490 return 491 # make sure there are answers 492 if dnspkt.header.a == 0 and dnspkt.header.aa == 0: 493 return 494 495 # lop off the trailing . 496 question = ("%s" % dnspkt.q.qname)[:-1].encode('utf-8') 497 498 for answer in dnspkt.rr: 499 # skip all but A and AAAA records 500 if answer.rtype == 1 or answer.rtype == 28: 501 dns_cache[str(answer.rdata).encode('utf-8')] = (question, 502 datetime.now()) 503 504# initialize BPF 505b = BPF(text=bpf_text) 506b.attach_kprobe(event="tcp_v4_connect", fn_name="trace_connect_entry") 507b.attach_kprobe(event="tcp_v6_connect", fn_name="trace_connect_entry") 508b.attach_kretprobe(event="tcp_v4_connect", fn_name="trace_connect_v4_return") 509b.attach_kretprobe(event="tcp_v6_connect", fn_name="trace_connect_v6_return") 510if args.dns: 511 b.attach_kprobe(event="udp_recvmsg", fn_name="trace_udp_recvmsg") 512 b.attach_kretprobe(event="udp_recvmsg", fn_name="trace_udp_ret_recvmsg") 513 514print("Tracing connect ... Hit Ctrl-C to end") 515if args.count: 516 try: 517 while True: 518 sleep(99999999) 519 except KeyboardInterrupt: 520 pass 521 522 # header 523 print("\n%-25s %-25s %-20s %-10s" % ( 524 "LADDR", "RADDR", "RPORT", "CONNECTS")) 525 depict_cnt(b["ipv4_count"]) 526 depict_cnt(b["ipv6_count"], l3prot='ipv6') 527# read events 528else: 529 # header 530 if args.timestamp: 531 print("%-9s" % ("TIME(s)"), end="") 532 if args.print_uid: 533 print("%-6s" % ("UID"), end="") 534 if args.lport: 535 print("%-7s %-12s %-2s %-16s %-6s %-16s %-6s" % ("PID", "COMM", "IP", "SADDR", 536 "LPORT", "DADDR", "DPORT"), end="") 537 else: 538 print("%-7s %-12s %-2s %-16s %-16s %-6s" % ("PID", "COMM", "IP", "SADDR", 539 "DADDR", "DPORT"), end="") 540 if args.dns: 541 print(" QUERY") 542 else: 543 print() 544 545 start_ts = 0 546 547 # read events 548 b["ipv4_events"].open_perf_buffer(print_ipv4_event) 549 b["ipv6_events"].open_perf_buffer(print_ipv6_event) 550 if args.dns: 551 b["dns_events"].open_perf_buffer(save_dns) 552 while True: 553 try: 554 b.perf_buffer_poll() 555 except KeyboardInterrupt: 556 exit() 557