• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import absolute_import
17from __future__ import division
18from __future__ import print_function
19
20import argparse
21import atexit
22import os
23import shutil
24import signal
25import subprocess
26import sys
27import tempfile
28import time
29import uuid
30
31from perfetto.prebuilts.manifests.traceconv import *
32from perfetto.prebuilts.perfetto_prebuilts import *
33
34NULL = open(os.devnull)
35NOOUT = {
36    'stdout': NULL,
37    'stderr': NULL,
38}
39
40UUID = str(uuid.uuid4())[-6:]
41
42PACKAGES_LIST_CFG = '''data_sources {
43  config {
44    name: "android.packages_list"
45  }
46}
47'''
48
49CFG_INDENT = '      '
50CFG = '''buffers {{
51  size_kb: 63488
52}}
53
54data_sources {{
55  config {{
56    name: "android.heapprofd"
57    heapprofd_config {{
58      shmem_size_bytes: {shmem_size}
59      sampling_interval_bytes: {interval}
60{target_cfg}
61    }}
62  }}
63}}
64
65duration_ms: {duration}
66write_into_file: true
67flush_timeout_ms: 30000
68flush_period_ms: 604800000
69'''
70
71# flush_period_ms of 1 week to suppress trace_processor_shell warning.
72
73CONTINUOUS_DUMP = """
74      continuous_dump_config {{
75        dump_phase_ms: 0
76        dump_interval_ms: {dump_interval}
77      }}
78"""
79
80PROFILE_LOCAL_PATH = os.path.join(tempfile.gettempdir(), UUID)
81
82IS_INTERRUPTED = False
83
84
85def sigint_handler(sig, frame):
86  global IS_INTERRUPTED
87  IS_INTERRUPTED = True
88
89
90def print_no_profile_error():
91  print("No profiles generated", file=sys.stderr)
92  print(
93      "If this is unexpected, check "
94      "https://perfetto.dev/docs/data-sources/native-heap-profiler#troubleshooting.",
95      file=sys.stderr)
96
97
98def known_issues_url(number):
99  return ('https://perfetto.dev/docs/data-sources/native-heap-profiler'
100          '#known-issues-android{}'.format(number))
101
102
103KNOWN_ISSUES = {
104    '10': known_issues_url(10),
105    'Q': known_issues_url(10),
106    '11': known_issues_url(11),
107    'R': known_issues_url(11),
108}
109
110
111def maybe_known_issues():
112  release_or_codename = subprocess.check_output(
113      ['adb', 'shell', 'getprop',
114       'ro.build.version.release_or_codename']).decode('utf-8').strip()
115  return KNOWN_ISSUES.get(release_or_codename, None)
116
117
118SDK = {
119    'R': 30,
120}
121
122
123def release_or_newer(release):
124  sdk = int(
125      subprocess.check_output(
126          ['adb', 'shell', 'getprop',
127           'ro.system.build.version.sdk']).decode('utf-8').strip())
128  if sdk >= SDK[release]:
129    return True
130  codename = subprocess.check_output(
131      ['adb', 'shell', 'getprop',
132       'ro.build.version.codename']).decode('utf-8').strip()
133  return codename == release
134
135
136ORDER = ['-n', '-p', '-i', '-o']
137
138
139def arg_order(action):
140  result = len(ORDER)
141  for opt in action.option_strings:
142    if opt in ORDER:
143      result = min(ORDER.index(opt), result)
144  return result, action.option_strings[0].strip('-')
145
146
147def print_options(parser):
148  for action in sorted(parser._actions, key=arg_order):
149    if action.help is argparse.SUPPRESS:
150      continue
151    opts = ', '.join('`' + x + '`' for x in action.option_strings)
152    metavar = '' if action.metavar is None else ' _' + action.metavar + '_'
153    print('{}{}'.format(opts, metavar))
154    print(':    {}'.format(action.help))
155    print()
156
157
158def main(argv):
159  parser = argparse.ArgumentParser()
160  parser.add_argument(
161      "-i",
162      "--interval",
163      help="Sampling interval. "
164      "Default 4096 (4KiB)",
165      type=int,
166      default=4096)
167  parser.add_argument(
168      "-d",
169      "--duration",
170      help="Duration of profile (ms). 0 to run until interrupted. "
171      "Default: until interrupted by user.",
172      type=int,
173      default=0)
174  # This flag is a no-op now. We never start heapprofd explicitly using system
175  # properties.
176  parser.add_argument(
177      "--no-start", help="Do not start heapprofd.", action='store_true')
178  parser.add_argument(
179      "-p",
180      "--pid",
181      help="Comma-separated list of PIDs to "
182      "profile.",
183      metavar="PIDS")
184  parser.add_argument(
185      "-n",
186      "--name",
187      help="Comma-separated list of process "
188      "names to profile.",
189      metavar="NAMES")
190  parser.add_argument(
191      "-c",
192      "--continuous-dump",
193      help="Dump interval in ms. 0 to disable continuous dump.",
194      type=int,
195      default=0)
196  parser.add_argument(
197      "--heaps",
198      help="Comma-separated list of heaps to collect, e.g: malloc,art. "
199      "Requires Android 12.",
200      metavar="HEAPS")
201  parser.add_argument(
202      "--all-heaps",
203      action="store_true",
204      help="Collect allocations from all heaps registered by target.")
205  parser.add_argument(
206      "--no-android-tree-symbolization",
207      action="store_true",
208      help="Do not symbolize using currently lunched target in the "
209      "Android tree.")
210  parser.add_argument(
211      "--disable-selinux",
212      action="store_true",
213      help="Disable SELinux enforcement for duration of "
214      "profile.")
215  parser.add_argument(
216      "--no-versions",
217      action="store_true",
218      help="Do not get version information about APKs.")
219  parser.add_argument(
220      "--no-running",
221      action="store_true",
222      help="Do not target already running processes. Requires Android 11.")
223  parser.add_argument(
224      "--no-startup",
225      action="store_true",
226      help="Do not target processes that start during "
227      "the profile. Requires Android 11.")
228  parser.add_argument(
229      "--shmem-size",
230      help="Size of buffer between client and "
231      "heapprofd. Default 8MiB. Needs to be a power of two "
232      "multiple of 4096, at least 8192.",
233      type=int,
234      default=8 * 1048576)
235  parser.add_argument(
236      "--block-client",
237      help="When buffer is full, block the "
238      "client to wait for buffer space. Use with caution as "
239      "this can significantly slow down the client. "
240      "This is the default",
241      action="store_true")
242  parser.add_argument(
243      "--block-client-timeout",
244      help="If --block-client is given, do not block any allocation for "
245      "longer than this timeout (us).",
246      type=int)
247  parser.add_argument(
248      "--no-block-client",
249      help="When buffer is full, stop the "
250      "profile early.",
251      action="store_true")
252  parser.add_argument(
253      "--idle-allocations",
254      help="Keep track of how many "
255      "bytes were unused since the last dump, per "
256      "callstack",
257      action="store_true")
258  parser.add_argument(
259      "--dump-at-max",
260      help="Dump the maximum memory usage "
261      "rather than at the time of the dump.",
262      action="store_true")
263  parser.add_argument(
264      "--disable-fork-teardown",
265      help="Do not tear down client in forks. This can be useful for programs "
266      "that use vfork. Android 11+ only.",
267      action="store_true")
268  parser.add_argument(
269      "--simpleperf",
270      action="store_true",
271      help="Get simpleperf profile of heapprofd. This is "
272      "only for heapprofd development.")
273  parser.add_argument(
274      "--traceconv-binary", help="Path to local trace to text. For debugging.")
275  parser.add_argument(
276      "--no-annotations",
277      help="Do not suffix the pprof function names with Android ART mode "
278      "annotations such as [jit].",
279      action="store_true")
280  parser.add_argument(
281      "--print-config",
282      action="store_true",
283      help="Print config instead of running. For debugging.")
284  parser.add_argument(
285      "-o",
286      "--output",
287      help="Output directory.",
288      metavar="DIRECTORY",
289      default=None)
290  parser.add_argument(
291      "--print-options", action="store_true", help=argparse.SUPPRESS)
292
293  args = parser.parse_args()
294  if args.print_options:
295    print_options(parser)
296    return 0
297  fail = False
298  if args.block_client and args.no_block_client:
299    print(
300        "FATAL: Both block-client and no-block-client given.", file=sys.stderr)
301    fail = True
302  if args.pid is None and args.name is None:
303    print("FATAL: Neither PID nor NAME given.", file=sys.stderr)
304    fail = True
305  if args.duration is None:
306    print("FATAL: No duration given.", file=sys.stderr)
307    fail = True
308  if args.interval is None:
309    print("FATAL: No interval given.", file=sys.stderr)
310    fail = True
311  if args.shmem_size % 4096:
312    print("FATAL: shmem-size is not a multiple of 4096.", file=sys.stderr)
313    fail = True
314  if args.shmem_size < 8192:
315    print("FATAL: shmem-size is less than 8192.", file=sys.stderr)
316    fail = True
317  if args.shmem_size & (args.shmem_size - 1):
318    print("FATAL: shmem-size is not a power of two.", file=sys.stderr)
319    fail = True
320
321  target_cfg = ""
322  if not args.no_block_client:
323    target_cfg += CFG_INDENT + "block_client: true\n"
324  if args.block_client_timeout:
325    target_cfg += (
326        CFG_INDENT +
327        "block_client_timeout_us: %s\n" % args.block_client_timeout)
328  if args.no_startup:
329    target_cfg += CFG_INDENT + "no_startup: true\n"
330  if args.no_running:
331    target_cfg += CFG_INDENT + "no_running: true\n"
332  if args.dump_at_max:
333    target_cfg += CFG_INDENT + "dump_at_max: true\n"
334  if args.disable_fork_teardown:
335    target_cfg += CFG_INDENT + "disable_fork_teardown: true\n"
336  if args.all_heaps:
337    target_cfg += CFG_INDENT + "all_heaps: true\n"
338  if args.pid:
339    for pid in args.pid.split(','):
340      try:
341        pid = int(pid)
342      except ValueError:
343        print("FATAL: invalid PID %s" % pid, file=sys.stderr)
344        fail = True
345      target_cfg += CFG_INDENT + 'pid: {}\n'.format(pid)
346  if args.name:
347    for name in args.name.split(','):
348      target_cfg += CFG_INDENT + 'process_cmdline: "{}"\n'.format(name)
349  if args.heaps:
350    for heap in args.heaps.split(','):
351      target_cfg += CFG_INDENT + 'heaps: "{}"\n'.format(heap)
352
353  if fail:
354    parser.print_help()
355    return 1
356
357  traceconv_binary = args.traceconv_binary
358
359  if args.continuous_dump:
360    target_cfg += CONTINUOUS_DUMP.format(dump_interval=args.continuous_dump)
361  cfg = CFG.format(
362      interval=args.interval,
363      duration=args.duration,
364      target_cfg=target_cfg,
365      shmem_size=args.shmem_size)
366  if not args.no_versions:
367    cfg += PACKAGES_LIST_CFG
368
369  if args.print_config:
370    print(cfg)
371    return 0
372
373  # Do this AFTER print_config so we do not download traceconv only to
374  # print out the config.
375  if traceconv_binary is None:
376    traceconv_binary = get_perfetto_prebuilt(TRACECONV_MANIFEST, soft_fail=True)
377
378  known_issues = maybe_known_issues()
379  if known_issues:
380    print('If you are experiencing problems, please see the known issues for '
381          'your release: {}.'.format(known_issues))
382
383  # TODO(fmayer): Maybe feature detect whether we can remove traces instead of
384  # this.
385  uuid_trace = release_or_newer('R')
386  if uuid_trace:
387    profile_device_path = '/data/misc/perfetto-traces/profile-' + UUID
388  else:
389    user = subprocess.check_output(['adb', 'shell',
390                                    'whoami']).decode('utf-8').strip()
391    profile_device_path = '/data/misc/perfetto-traces/profile-' + user
392
393  perfetto_cmd = ('CFG=\'{cfg}\'; echo ${{CFG}} | '
394                  'perfetto --txt -c - -o ' + profile_device_path + ' -d')
395
396  if args.disable_selinux:
397    enforcing = subprocess.check_output(['adb', 'shell',
398                                         'getenforce']).decode('utf-8').strip()
399    atexit.register(
400        subprocess.check_call,
401        ['adb', 'shell', 'su root setenforce %s' % enforcing])
402    subprocess.check_call(['adb', 'shell', 'su root setenforce 0'])
403
404  if args.simpleperf:
405    subprocess.check_call([
406        'adb', 'shell', 'mkdir -p /data/local/tmp/heapprofd_profile && '
407        'cd /data/local/tmp/heapprofd_profile &&'
408        '(nohup simpleperf record -g -p $(pidof heapprofd) 2>&1 &) '
409        '> /dev/null'
410    ])
411
412  profile_target = PROFILE_LOCAL_PATH
413  if args.output is not None:
414    profile_target = args.output
415  else:
416    os.mkdir(profile_target)
417
418  if not os.path.isdir(profile_target):
419    print(
420        "Output directory {} not found".format(profile_target), file=sys.stderr)
421    return 1
422
423  if os.listdir(profile_target):
424    print(
425        "Output directory {} not empty".format(profile_target), file=sys.stderr)
426    return 1
427
428  perfetto_pid = subprocess.check_output(
429      ['adb', 'exec-out', perfetto_cmd.format(cfg=cfg)]).strip()
430  try:
431    perfetto_pid = int(perfetto_pid.strip())
432  except ValueError:
433    print("Failed to invoke perfetto: {}".format(perfetto_pid), file=sys.stderr)
434    return 1
435
436  old_handler = signal.signal(signal.SIGINT, sigint_handler)
437  print("Profiling active. Press Ctrl+C to terminate.")
438  print("You may disconnect your device.")
439  print()
440  exists = True
441  device_connected = True
442  while not device_connected or (exists and not IS_INTERRUPTED):
443    exists = subprocess.call(
444        ['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)], **NOOUT) == 0
445    device_connected = subprocess.call(['adb', 'shell', 'true'], **NOOUT) == 0
446    time.sleep(1)
447  print("Waiting for profiler shutdown...")
448  signal.signal(signal.SIGINT, old_handler)
449  if IS_INTERRUPTED:
450    # Not check_call because it could have existed in the meantime.
451    subprocess.call(['adb', 'shell', 'kill', '-INT', str(perfetto_pid)])
452  if args.simpleperf:
453    subprocess.check_call(['adb', 'shell', 'killall', '-INT', 'simpleperf'])
454    print("Waiting for simpleperf to exit.")
455    while subprocess.call(
456        ['adb', 'shell', '[ -f /proc/$(pidof simpleperf)/exe ]'], **NOOUT) == 0:
457      time.sleep(1)
458    subprocess.check_call(
459        ['adb', 'pull', '/data/local/tmp/heapprofd_profile', profile_target])
460    print("Pulled simpleperf profile to " + profile_target +
461          "/heapprofd_profile")
462
463  # Wait for perfetto cmd to return.
464  while exists:
465    exists = subprocess.call(
466        ['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)]) == 0
467    time.sleep(1)
468
469  profile_host_path = os.path.join(profile_target, 'raw-trace')
470  subprocess.check_call(['adb', 'pull', profile_device_path, profile_host_path],
471                        stdout=NULL)
472  if uuid_trace:
473    subprocess.check_call(['adb', 'shell', 'rm', profile_device_path],
474                          stdout=NULL)
475
476  if traceconv_binary is None:
477    print('Wrote profile to {}'.format(profile_host_path))
478    print(
479        'This file can be opened using the Perfetto UI, https://ui.perfetto.dev'
480    )
481    return 0
482
483  binary_path = os.getenv('PERFETTO_BINARY_PATH')
484  if not args.no_android_tree_symbolization:
485    product_out = os.getenv('ANDROID_PRODUCT_OUT')
486    if product_out:
487      product_out_symbols = product_out + '/symbols'
488    else:
489      product_out_symbols = None
490
491    if binary_path is None:
492      binary_path = product_out_symbols
493    elif product_out_symbols is not None:
494      binary_path += ":" + product_out_symbols
495
496  trace_file = os.path.join(profile_target, 'raw-trace')
497  concat_files = [trace_file]
498
499  if binary_path is not None:
500    with open(os.path.join(profile_target, 'symbols'), 'w') as fd:
501      ret = subprocess.call([
502          traceconv_binary, 'symbolize',
503          os.path.join(profile_target, 'raw-trace')
504      ],
505                            env=dict(
506                                os.environ, PERFETTO_BINARY_PATH=binary_path),
507                            stdout=fd)
508    if ret == 0:
509      concat_files.append(os.path.join(profile_target, 'symbols'))
510    else:
511      print("Failed to symbolize. Continuing without symbols.", file=sys.stderr)
512
513  proguard_map = os.getenv('PERFETTO_PROGUARD_MAP')
514  if proguard_map is not None:
515    with open(os.path.join(profile_target, 'deobfuscation-packets'), 'w') as fd:
516      ret = subprocess.call([
517          traceconv_binary, 'deobfuscate',
518          os.path.join(profile_target, 'raw-trace')
519      ],
520                            env=dict(
521                                os.environ, PERFETTO_PROGUARD_MAP=proguard_map),
522                            stdout=fd)
523    if ret == 0:
524      concat_files.append(os.path.join(profile_target, 'deobfuscation-packets'))
525    else:
526      print(
527          "Failed to deobfuscate. Continuing without deobfuscated.",
528          file=sys.stderr)
529
530  if len(concat_files) > 1:
531    with open(os.path.join(profile_target, 'symbolized-trace'), 'wb') as out:
532      for fn in concat_files:
533        with open(fn, 'rb') as inp:
534          while True:
535            buf = inp.read(4096)
536            if not buf:
537              break
538            out.write(buf)
539    trace_file = os.path.join(profile_target, 'symbolized-trace')
540
541  conversion_args = [traceconv_binary, 'profile'] + (
542      ['--no-annotations'] if args.no_annotations else []) + [trace_file]
543  traceconv_output = subprocess.check_output(conversion_args)
544  profile_path = None
545  for word in traceconv_output.decode('utf-8').split():
546    if 'heap_profile-' in word:
547      profile_path = word
548  if profile_path is None:
549    print_no_profile_error()
550    return 1
551
552  profile_files = os.listdir(profile_path)
553  if not profile_files:
554    print_no_profile_error()
555    return 1
556
557  for profile_file in profile_files:
558    shutil.copy(os.path.join(profile_path, profile_file), profile_target)
559
560  symlink_path = None
561  if not sys.platform.startswith('win'):
562    subprocess.check_call(
563        ['gzip'] + [os.path.join(profile_target, x) for x in profile_files])
564    if args.output is None:
565      symlink_path = os.path.join(
566          os.path.dirname(profile_target), "heap_profile-latest")
567      if os.path.lexists(symlink_path):
568        os.unlink(symlink_path)
569      os.symlink(profile_target, symlink_path)
570
571  if symlink_path is not None:
572    print("Wrote profiles to {} (symlink {})".format(profile_target,
573                                                     symlink_path))
574  else:
575    print("Wrote profiles to {}".format(profile_target))
576
577  print("The raw-trace file can be viewed using https://ui.perfetto.dev.")
578  print("The heap_dump.* files can be viewed using pprof/ (Googlers only) " +
579        "or https://www.speedscope.app/.")
580  print("The two above are equivalent. The raw-trace contains the union of " +
581        "all the heap dumps.")
582
583
584if __name__ == '__main__':
585  sys.exit(main(sys.argv))
586