• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import argparse
22import atexit
23import hashlib
24import os
25import shutil
26import signal
27import subprocess
28import sys
29import tempfile
30import time
31import uuid
32
33
34TRACE_TO_TEXT_SHAS = {
35    'linux': 'aba0e660818bfc249992ebbceb13a2e4c9a62c3a',
36    'mac': 'c7d1b9d904f008bfb95125b204837eff946c3ed7',
37}
38TRACE_TO_TEXT_PATH = tempfile.gettempdir()
39TRACE_TO_TEXT_BASE_URL = ('https://storage.googleapis.com/perfetto/')
40
41NULL = open(os.devnull)
42NOOUT = {
43    'stdout': NULL,
44    'stderr': NULL,
45}
46
47UUID = str(uuid.uuid4())
48
49def check_hash(file_name, sha_value):
50  with open(file_name, 'rb') as fd:
51    # TODO(fmayer): Chunking.
52    file_hash = hashlib.sha1(fd.read()).hexdigest()
53    return file_hash == sha_value
54
55
56def load_trace_to_text(platform):
57  sha_value = TRACE_TO_TEXT_SHAS[platform]
58  file_name = 'trace_to_text-' + platform + '-' + sha_value
59  local_file = os.path.join(TRACE_TO_TEXT_PATH, file_name)
60
61  if os.path.exists(local_file):
62    if not check_hash(local_file, sha_value):
63      os.remove(local_file)
64    else:
65      return local_file
66
67  url = TRACE_TO_TEXT_BASE_URL + file_name
68  subprocess.check_call(['curl', '-L', '-#', '-o', local_file, url])
69  if not check_hash(local_file, sha_value):
70    os.remove(local_file)
71    raise ValueError("Invalid signature.")
72  os.chmod(local_file, 0o755)
73  return local_file
74
75
76PACKAGES_LIST_CFG = '''data_sources {
77  config {
78    name: "android.packages_list"
79  }
80}
81'''
82
83CFG_INDENT = '      '
84CFG = '''buffers {{
85  size_kb: 32768
86}}
87
88data_sources {{
89  config {{
90    name: "android.heapprofd"
91    heapprofd_config {{
92
93      shmem_size_bytes: {shmem_size}
94      sampling_interval_bytes: {interval}
95{target_cfg}
96{continuous_dump_cfg}
97    }}
98  }}
99}}
100
101duration_ms: {duration}
102write_into_file: true
103flush_timeout_ms: 30000
104flush_period_ms: 604800000
105'''
106
107# flush_period_ms of 1 week to suppress trace_processor_shell warning.
108
109CONTINUOUS_DUMP = """
110      continuous_dump_config {{
111        dump_phase_ms: 0
112        dump_interval_ms: {dump_interval}
113      }}
114"""
115
116PROFILE_LOCAL_PATH = '/tmp/profile-' + UUID
117
118IS_INTERRUPTED = False
119
120def sigint_handler(sig, frame):
121  global IS_INTERRUPTED
122  IS_INTERRUPTED = True
123
124
125def print_no_profile_error():
126  print("No profiles generated", file=sys.stderr)
127  print(
128    "If this is unexpected, check "
129    "https://docs.perfetto.dev/#/heapprofd?id=troubleshooting.",
130    file=sys.stderr)
131
132SDK = {
133    'R': 30,
134}
135
136def release_or_newer(release):
137  sdk = int(subprocess.check_output(
138    ['adb', 'shell', 'getprop', 'ro.system.build.version.sdk']
139  ).decode('utf-8').strip())
140  if sdk >= SDK[release]:
141    return True
142  codename = subprocess.check_output(
143    ['adb', 'shell', 'getprop', 'ro.build.version.codename']
144  ).decode('utf-8').strip()
145  return codename == release
146
147def main(argv):
148  parser = argparse.ArgumentParser()
149  parser.add_argument(
150      "-i",
151      "--interval",
152      help="Sampling interval. "
153      "Default 4096 (4KiB)",
154      type=int,
155      default=4096)
156  parser.add_argument(
157      "-d",
158      "--duration",
159      help="Duration of profile (ms). "
160      "Default 7 days.",
161      type=int,
162      default=604800000)
163  parser.add_argument(
164      "--no-start", help="Do not start heapprofd.", action='store_true')
165  parser.add_argument(
166      "-p",
167      "--pid",
168      help="Comma-separated list of PIDs to "
169      "profile.",
170      metavar="PIDS")
171  parser.add_argument(
172      "-n",
173      "--name",
174      help="Comma-separated list of process "
175      "names to profile.",
176      metavar="NAMES")
177  parser.add_argument(
178      "-c",
179      "--continuous-dump",
180      help="Dump interval in ms. 0 to disable continuous dump.",
181      type=int,
182      default=0)
183  parser.add_argument(
184      "--disable-selinux",
185      action="store_true",
186      help="Disable SELinux enforcement for duration of "
187      "profile.")
188  parser.add_argument(
189      "--no-versions",
190      action="store_true",
191      help="Do not get version information about APKs.")
192  parser.add_argument(
193      "--no-running",
194      action="store_true",
195      help="Do not target already running processes. Requires Android 11.")
196  parser.add_argument(
197      "--no-startup",
198      action="store_true",
199      help="Do not target processes that start during "
200      "the profile. Requires Android 11.")
201  parser.add_argument(
202      "--shmem-size",
203      help="Size of buffer between client and "
204      "heapprofd. Default 8MiB. Needs to be a power of two "
205      "multiple of 4096, at least 8192.",
206      type=int,
207      default=8 * 1048576)
208  parser.add_argument(
209      "--block-client",
210      help="When buffer is full, block the "
211      "client to wait for buffer space. Use with caution as "
212      "this can significantly slow down the client. "
213      "This is the default",
214      action="store_true")
215  parser.add_argument(
216      "--block-client-timeout",
217      help="If --block-client is given, do not block any allocation for "
218      "longer than this timeout (us).",
219      type=int)
220  parser.add_argument(
221      "--no-block-client",
222      help="When buffer is full, stop the "
223      "profile early.",
224      action="store_true")
225  parser.add_argument(
226      "--idle-allocations",
227      help="Keep track of how many "
228      "bytes were unused since the last dump, per "
229      "callstack",
230      action="store_true")
231  parser.add_argument(
232      "--dump-at-max",
233      help="Dump the maximum memory usage "
234      "rather than at the time of the dump.",
235      action="store_true")
236  parser.add_argument(
237      "--disable-fork-teardown",
238      help="Do not tear down client in forks. This can be useful for programs "
239      "that use vfork. Android 11+ only.",
240      action="store_true")
241  parser.add_argument(
242      "--simpleperf",
243      action="store_true",
244      help="Get simpleperf profile of heapprofd. This is "
245      "only for heapprofd development.")
246  parser.add_argument(
247      "--trace-to-text-binary",
248      help="Path to local trace to text. For debugging.")
249  parser.add_argument(
250      "--print-config",
251      action="store_true",
252      help="Print config instead of running. For debugging.")
253  parser.add_argument(
254      "-o",
255      "--output",
256      help="Output directory.",
257      metavar="DIRECTORY",
258      default=None)
259
260  args = parser.parse_args()
261
262  # TODO(fmayer): Maybe feature detect whether we can remove traces instead of
263  # this.
264  uuid_trace = release_or_newer('R')
265  if uuid_trace:
266    profile_device_path = '/data/misc/perfetto-traces/profile-' + UUID
267  else:
268    user = subprocess.check_output(
269      ['adb', 'shell', 'whoami']).decode('utf-8').strip()
270    profile_device_path = '/data/misc/perfetto-traces/profile-' + user
271  perfetto_cmd = ('CFG=\'{cfg}\'; echo ${{CFG}} | '
272                  'perfetto --txt -c - -o ' + profile_device_path + ' -d')
273
274  fail = False
275  if args.block_client and args.no_block_client:
276    print(
277        "FATAL: Both block-client and no-block-client given.", file=sys.stderr)
278    fail = True
279  if args.pid is None and args.name is None:
280    print("FATAL: Neither PID nor NAME given.", file=sys.stderr)
281    fail = True
282  if args.duration is None:
283    print("FATAL: No duration given.", file=sys.stderr)
284    fail = True
285  if args.interval is None:
286    print("FATAL: No interval given.", file=sys.stderr)
287    fail = True
288  if args.shmem_size % 4096:
289    print("FATAL: shmem-size is not a multiple of 4096.", file=sys.stderr)
290    fail = True
291  if args.shmem_size < 8192:
292    print("FATAL: shmem-size is less than 8192.", file=sys.stderr)
293    fail = True
294  if args.shmem_size & (args.shmem_size - 1):
295    print("FATAL: shmem-size is not a power of two.", file=sys.stderr)
296    fail = True
297
298  target_cfg = ""
299  if not args.no_block_client:
300    target_cfg += "block_client: true\n"
301  if args.block_client_timeout:
302    target_cfg += "block_client_timeout_us: %s\n" % args.block_client_timeout
303  if args.idle_allocations:
304    target_cfg += "idle_allocations: true\n"
305  if args.no_startup:
306    target_cfg += "no_startup: true\n"
307  if args.no_running:
308    target_cfg += "no_running: true\n"
309  if args.dump_at_max:
310    target_cfg += "dump_at_max: true\n"
311  if args.disable_fork_teardown:
312    target_cfg += "disable_fork_teardown: true\n"
313  if args.pid:
314    for pid in args.pid.split(','):
315      try:
316        pid = int(pid)
317      except ValueError:
318        print("FATAL: invalid PID %s" % pid, file=sys.stderr)
319        fail = True
320      target_cfg += '{}pid: {}\n'.format(CFG_INDENT, pid)
321  if args.name:
322    for name in args.name.split(','):
323      target_cfg += '{}process_cmdline: "{}"\n'.format(CFG_INDENT, name)
324
325  if fail:
326    parser.print_help()
327    return 1
328
329  trace_to_text_binary = args.trace_to_text_binary
330  if trace_to_text_binary is None:
331    platform = None
332    if sys.platform.startswith('linux'):
333      platform = 'linux'
334    elif sys.platform.startswith('darwin'):
335      platform = 'mac'
336    else:
337      print("Invalid platform: {}".format(sys.platform), file=sys.stderr)
338      return 1
339
340    trace_to_text_binary = load_trace_to_text(platform)
341
342  continuous_dump_cfg = ""
343  if args.continuous_dump:
344    continuous_dump_cfg = CONTINUOUS_DUMP.format(
345        dump_interval=args.continuous_dump)
346  cfg = CFG.format(
347      interval=args.interval,
348      duration=args.duration,
349      target_cfg=target_cfg,
350      continuous_dump_cfg=continuous_dump_cfg,
351      shmem_size=args.shmem_size)
352  if not args.no_versions:
353    cfg += PACKAGES_LIST_CFG
354
355  if args.print_config:
356    print(cfg)
357    return 0
358
359  if args.disable_selinux:
360    enforcing = subprocess.check_output(['adb', 'shell', 'getenforce'])
361    atexit.register(
362        subprocess.check_call,
363        ['adb', 'shell', 'su root setenforce %s' % enforcing])
364    subprocess.check_call(['adb', 'shell', 'su root setenforce 0'])
365
366  if not args.no_start:
367    heapprofd_prop = subprocess.check_output(
368        ['adb', 'shell', 'getprop persist.heapprofd.enable'])
369    if heapprofd_prop.strip() != '1':
370      subprocess.check_call(
371          ['adb', 'shell', 'setprop persist.heapprofd.enable 1'])
372      atexit.register(subprocess.check_call,
373                      ['adb', 'shell', 'setprop persist.heapprofd.enable 0'])
374
375
376  if args.simpleperf:
377    subprocess.check_call([
378        'adb', 'shell', 'mkdir -p /data/local/tmp/heapprofd_profile && '
379        'cd /data/local/tmp/heapprofd_profile &&'
380        '(nohup simpleperf record -g -p $(pidof heapprofd) 2>&1 &) '
381        '> /dev/null'
382    ])
383
384  profile_target = PROFILE_LOCAL_PATH
385  if args.output is not None:
386    profile_target = args.output
387  else:
388    os.mkdir(profile_target)
389
390  if not os.path.isdir(profile_target):
391    print("Output directory {} not found".format(profile_target),
392            file=sys.stderr)
393    return 1
394
395  if os.listdir(profile_target):
396    print("Output directory {} not empty".format(profile_target),
397            file=sys.stderr)
398    return 1
399
400  perfetto_pid = subprocess.check_output(
401      ['adb', 'exec-out',
402       perfetto_cmd.format(cfg=cfg)]).strip()
403  try:
404    perfetto_pid = int(perfetto_pid.strip())
405  except ValueError:
406    print("Failed to invoke perfetto: {}".format(perfetto_pid), file=sys.stderr)
407    return 1
408
409  old_handler = signal.signal(signal.SIGINT, sigint_handler)
410  print("Profiling active. Press Ctrl+C to terminate.")
411  print("You may disconnect your device.")
412  print()
413  exists = True
414  device_connected = True
415  while not device_connected or (exists and not IS_INTERRUPTED):
416    exists = subprocess.call(
417        ['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)], **NOOUT) == 0
418    device_connected = subprocess.call(['adb', 'shell', 'true'], **NOOUT) == 0
419    time.sleep(1)
420  signal.signal(signal.SIGINT, old_handler)
421  if IS_INTERRUPTED:
422    # Not check_call because it could have existed in the meantime.
423    subprocess.call(['adb', 'shell', 'kill', '-INT', str(perfetto_pid)])
424  if args.simpleperf:
425    subprocess.check_call(['adb', 'shell', 'killall', '-INT', 'simpleperf'])
426    print("Waiting for simpleperf to exit.")
427    while subprocess.call(
428        ['adb', 'shell', '[ -f /proc/$(pidof simpleperf)/exe ]'], **NOOUT) == 0:
429      time.sleep(1)
430    subprocess.check_call(
431        ['adb', 'pull', '/data/local/tmp/heapprofd_profile', '/tmp'])
432    print("Pulled simpleperf profile to /tmp/heapprofd_profile")
433
434  # Wait for perfetto cmd to return.
435  while exists:
436    exists = subprocess.call(
437        ['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)]) == 0
438    time.sleep(1)
439
440  subprocess.check_call([
441      'adb', 'pull', profile_device_path,
442      os.path.join(profile_target, 'raw-trace')
443  ], stdout=NULL)
444  if uuid_trace:
445    subprocess.check_call(
446          ['adb', 'shell', 'rm', profile_device_path], stdout=NULL)
447
448  trace_to_text_output = subprocess.check_output(
449      [trace_to_text_binary, 'profile',
450          os.path.join(profile_target, 'raw-trace')],
451      env=os.environ)
452  profile_path = None
453  for word in trace_to_text_output.decode('utf-8').split():
454    if 'heap_profile-' in word:
455      profile_path = word
456  if profile_path is None:
457    print_no_profile_error();
458    return 1
459
460  profile_files = os.listdir(profile_path)
461  if not profile_files:
462    print_no_profile_error();
463    return 1
464
465  for profile_file in profile_files:
466    shutil.copy(os.path.join(profile_path, profile_file), profile_target)
467
468  subprocess.check_call(
469      ['gzip'] +
470      [os.path.join(profile_target, x) for x in profile_files])
471
472  symlink_path = None
473  if args.output is None:
474      symlink_path = os.path.join(
475          os.path.dirname(profile_target), "heap_profile-latest")
476      if os.path.lexists(symlink_path):
477        os.unlink(symlink_path)
478      os.symlink(profile_target, symlink_path)
479
480  binary_path = os.getenv('PERFETTO_BINARY_PATH')
481  if binary_path is not None:
482      with open(os.path.join(profile_path, 'symbols'), 'w') as fd:
483          ret = subprocess.call([
484              trace_to_text_binary, 'symbolize',
485              os.path.join(profile_target, 'raw-trace')],
486              env=os.environ,
487              stdout=fd)
488          if ret != 0:
489              print("Failed to symbolize. Continuing without symbols.",
490                    file=sys.stderr)
491
492  if symlink_path is not None:
493    print("Wrote profiles to {} (symlink {})".format(
494        profile_target, symlink_path))
495  else:
496    print("Wrote profiles to {}".format(profile_target))
497
498  print("These can be viewed using pprof. Googlers: head to pprof/ and "
499        "upload them.")
500
501
502if __name__ == '__main__':
503  sys.exit(main(sys.argv))
504