1#!/usr/bin/env python 2# @lint-avoid-python-3-compatibility-imports 3# 4# ustat Activity stats from high-level languages, including exceptions, 5# method calls, class loads, garbage collections, and more. 6# For Linux, uses BCC, eBPF. 7# 8# USAGE: ustat [-l {java,node,perl,php,python,ruby,tcl}] [-C] 9# [-S {cload,excp,gc,method,objnew,thread}] [-r MAXROWS] [-d] 10# [interval [count]] 11# 12# This uses in-kernel eBPF maps to store per process summaries for efficiency. 13# Newly-created processes might only be traced at the next interval, if the 14# relevant USDT probe requires enabling through a semaphore. 15# 16# Copyright 2016 Sasha Goldshtein 17# Licensed under the Apache License, Version 2.0 (the "License") 18# 19# 26-Oct-2016 Sasha Goldshtein Created this. 20 21from __future__ import print_function 22import argparse 23from bcc import BPF, USDT, USDTException 24import os 25import sys 26from subprocess import call 27from time import sleep, strftime 28 29class Category(object): 30 THREAD = "THREAD" 31 METHOD = "METHOD" 32 OBJNEW = "OBJNEW" 33 CLOAD = "CLOAD" 34 EXCP = "EXCP" 35 GC = "GC" 36 37class Probe(object): 38 def __init__(self, language, procnames, events): 39 """ 40 Initialize a new probe object with a specific language, set of process 41 names to monitor for that language, and a dictionary of events and 42 categories. The dictionary is a mapping of USDT probe names (such as 43 'gc__start') to event categories supported by this tool -- from the 44 Category class. 45 """ 46 self.language = language 47 self.procnames = procnames 48 self.events = events 49 50 def _find_targets(self): 51 """Find pids where the comm is one of the specified list""" 52 self.targets = {} 53 all_pids = [int(pid) for pid in os.listdir('/proc') if pid.isdigit()] 54 for pid in all_pids: 55 try: 56 comm = open('/proc/%d/comm' % pid).read().strip() 57 if comm in self.procnames: 58 cmdline = open('/proc/%d/cmdline' % pid).read() 59 self.targets[pid] = cmdline.replace('\0', ' ') 60 except IOError: 61 continue # process may already have terminated 62 63 def _enable_probes(self): 64 self.usdts = [] 65 for pid in self.targets: 66 try: 67 usdt = USDT(pid=pid) 68 except USDTException: 69 # avoid race condition on pid going away. 70 print("failed to instrument %d" % pid, file=sys.stderr) 71 continue 72 for event in self.events: 73 try: 74 usdt.enable_probe(event, "%s_%s" % (self.language, event)) 75 except Exception: 76 # This process might not have a recent version of the USDT 77 # probes enabled, or might have been compiled without USDT 78 # probes at all. The process could even have been shut down 79 # and the pid been recycled. We have to gracefully handle 80 # the possibility that we can't attach probes to it at all. 81 pass 82 self.usdts.append(usdt) 83 84 def _generate_tables(self): 85 text = """ 86BPF_HASH(%s_%s_counts, u32, u64); // pid to event count 87 """ 88 return str.join('', [text % (self.language, event) 89 for event in self.events]) 90 91 def _generate_functions(self): 92 text = """ 93int %s_%s(void *ctx) { 94 u64 *valp, zero = 0; 95 u32 tgid = bpf_get_current_pid_tgid() >> 32; 96 valp = %s_%s_counts.lookup_or_init(&tgid, &zero); 97 ++(*valp); 98 return 0; 99} 100 """ 101 lang = self.language 102 return str.join('', [text % (lang, event, lang, event) 103 for event in self.events]) 104 105 def get_program(self): 106 self._find_targets() 107 self._enable_probes() 108 return self._generate_tables() + self._generate_functions() 109 110 def get_usdts(self): 111 return self.usdts 112 113 def get_counts(self, bpf): 114 """Return a map of event counts per process""" 115 event_dict = dict([(category, 0) for category in self.events.values()]) 116 result = dict([(pid, event_dict.copy()) for pid in self.targets]) 117 for event, category in self.events.items(): 118 counts = bpf["%s_%s_counts" % (self.language, event)] 119 for pid, count in counts.items(): 120 if pid.value not in result: 121 print("result was not found for %d" % pid.value, file=sys.stderr) 122 continue 123 result[pid.value][category] = count.value 124 counts.clear() 125 return result 126 127 def cleanup(self): 128 self.usdts = None 129 130class Tool(object): 131 def _parse_args(self): 132 examples = """examples: 133 ./ustat # stats for all languages, 1 second refresh 134 ./ustat -C # don't clear the screen 135 ./ustat -l java # Java processes only 136 ./ustat 5 # 5 second summaries 137 ./ustat 5 10 # 5 second summaries, 10 times only 138 """ 139 parser = argparse.ArgumentParser( 140 description="Activity stats from high-level languages.", 141 formatter_class=argparse.RawDescriptionHelpFormatter, 142 epilog=examples) 143 parser.add_argument("-l", "--language", 144 choices=["java", "node", "perl", "php", "python", "ruby", "tcl"], 145 help="language to trace (default: all languages)") 146 parser.add_argument("-C", "--noclear", action="store_true", 147 help="don't clear the screen") 148 parser.add_argument("-S", "--sort", 149 choices=[cat.lower() for cat in dir(Category) if cat.isupper()], 150 help="sort by this field (descending order)") 151 parser.add_argument("-r", "--maxrows", default=20, type=int, 152 help="maximum rows to print, default 20") 153 parser.add_argument("-d", "--debug", action="store_true", 154 help="Print the resulting BPF program (for debugging purposes)") 155 parser.add_argument("interval", nargs="?", default=1, type=int, 156 help="output interval, in seconds") 157 parser.add_argument("count", nargs="?", default=99999999, type=int, 158 help="number of outputs") 159 parser.add_argument("--ebpf", action="store_true", 160 help=argparse.SUPPRESS) 161 self.args = parser.parse_args() 162 163 def _create_probes(self): 164 probes_by_lang = { 165 "java": Probe("java", ["java"], { 166 "gc__begin": Category.GC, 167 "mem__pool__gc__begin": Category.GC, 168 "thread__start": Category.THREAD, 169 "class__loaded": Category.CLOAD, 170 "object__alloc": Category.OBJNEW, 171 "method__entry": Category.METHOD, 172 "ExceptionOccurred__entry": Category.EXCP 173 }), 174 "node": Probe("node", ["node"], { 175 "gc__start": Category.GC 176 }), 177 "perl": Probe("perl", ["perl"], { 178 "sub__entry": Category.METHOD 179 }), 180 "php": Probe("php", ["php"], { 181 "function__entry": Category.METHOD, 182 "compile__file__entry": Category.CLOAD, 183 "exception__thrown": Category.EXCP 184 }), 185 "python": Probe("python", ["python"], { 186 "function__entry": Category.METHOD, 187 "gc__start": Category.GC 188 }), 189 "ruby": Probe("ruby", ["ruby", "irb"], { 190 "method__entry": Category.METHOD, 191 "cmethod__entry": Category.METHOD, 192 "gc__mark__begin": Category.GC, 193 "gc__sweep__begin": Category.GC, 194 "object__create": Category.OBJNEW, 195 "hash__create": Category.OBJNEW, 196 "string__create": Category.OBJNEW, 197 "array__create": Category.OBJNEW, 198 "require__entry": Category.CLOAD, 199 "load__entry": Category.CLOAD, 200 "raise": Category.EXCP 201 }), 202 "tcl": Probe("tcl", ["tclsh", "wish"], { 203 "proc__entry": Category.METHOD, 204 "obj__create": Category.OBJNEW 205 }), 206 } 207 208 if self.args.language: 209 self.probes = [probes_by_lang[self.args.language]] 210 else: 211 self.probes = probes_by_lang.values() 212 213 def _attach_probes(self): 214 program = str.join('\n', [p.get_program() for p in self.probes]) 215 if self.args.debug or self.args.ebpf: 216 print(program) 217 if self.args.ebpf: 218 exit() 219 for probe in self.probes: 220 print("Attached to %s processes:" % probe.language, 221 str.join(', ', map(str, probe.targets))) 222 self.bpf = BPF(text=program) 223 usdts = [usdt for probe in self.probes for usdt in probe.get_usdts()] 224 # Filter out duplicates when we have multiple processes with the same 225 # uprobe. We are attaching to these probes manually instead of using 226 # the USDT support from the bcc module, because the USDT class attaches 227 # to each uprobe with a specific pid. When there is more than one 228 # process from some language, we end up attaching more than once to the 229 # same uprobe (albeit with different pids), which is not allowed. 230 # Instead, we use a global attach (with pid=-1). 231 uprobes = set([(path, func, addr) for usdt in usdts 232 for (path, func, addr, _) 233 in usdt.enumerate_active_probes()]) 234 for (path, func, addr) in uprobes: 235 self.bpf.attach_uprobe(name=path, fn_name=func, addr=addr, pid=-1) 236 237 def _detach_probes(self): 238 for probe in self.probes: 239 probe.cleanup() # Cleans up USDT contexts 240 self.bpf.cleanup() # Cleans up all attached probes 241 self.bpf = None 242 243 def _loop_iter(self): 244 self._attach_probes() 245 try: 246 sleep(self.args.interval) 247 except KeyboardInterrupt: 248 self.exiting = True 249 250 if not self.args.noclear: 251 call("clear") 252 else: 253 print() 254 with open("/proc/loadavg") as stats: 255 print("%-8s loadavg: %s" % (strftime("%H:%M:%S"), stats.read())) 256 print("%-6s %-20s %-10s %-6s %-10s %-8s %-6s %-6s" % ( 257 "PID", "CMDLINE", "METHOD/s", "GC/s", "OBJNEW/s", 258 "CLOAD/s", "EXC/s", "THR/s")) 259 260 line = 0 261 counts = {} 262 targets = {} 263 for probe in self.probes: 264 counts.update(probe.get_counts(self.bpf)) 265 targets.update(probe.targets) 266 if self.args.sort: 267 sort_field = self.args.sort.upper() 268 counts = sorted(counts.items(), 269 key=lambda kv: -kv[1].get(sort_field, 0)) 270 else: 271 counts = sorted(counts.items(), key=lambda kv: kv[0]) 272 for pid, stats in counts: 273 print("%-6d %-20s %-10d %-6d %-10d %-8d %-6d %-6d" % ( 274 pid, targets[pid][:20], 275 stats.get(Category.METHOD, 0) / self.args.interval, 276 stats.get(Category.GC, 0) / self.args.interval, 277 stats.get(Category.OBJNEW, 0) / self.args.interval, 278 stats.get(Category.CLOAD, 0) / self.args.interval, 279 stats.get(Category.EXCP, 0) / self.args.interval, 280 stats.get(Category.THREAD, 0) / self.args.interval 281 )) 282 line += 1 283 if line >= self.args.maxrows: 284 break 285 self._detach_probes() 286 287 def run(self): 288 self._parse_args() 289 self._create_probes() 290 print('Tracing... Output every %d secs. Hit Ctrl-C to end' % 291 self.args.interval) 292 countdown = self.args.count 293 self.exiting = False 294 while True: 295 self._loop_iter() 296 countdown -= 1 297 if self.exiting or countdown == 0: 298 print("Detaching...") 299 exit() 300 301if __name__ == "__main__": 302 try: 303 Tool().run() 304 except KeyboardInterrupt: 305 pass 306