#!/usr/bin/env python # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Sample Usage: # $ python update_profiles.py 500000 git_master ALL --profdata-suffix 2019-04-15 # # Additional/frequently-used arguments: # -b BUG adds a 'Bug: ' to the commit message when adding the profiles. # --do-not-merge adds a 'DO NOT MERGE' tag to the commit message to restrict # automerge of profiles from release branches. # # Try '-h' for a full list of command line arguments. import argparse import os import shutil import subprocess import sys import tempfile import zipfile import utils X20_BASE_LOCATION = '/google/data/ro/teams/android-pgo-data' class Benchmark(object): def __init__(self, name): self.name = name def x20_profile_location(self): raise NotImplementedError() def apct_job_name(self): raise NotImplementedError() def profdata_file(self, suffix=''): profdata = os.path.join(self.name, '{}.profdata'.format(self.name)) if suffix: profdata += '.' + suffix return profdata def profraw_files(self): raise NotImplementedError() def merge_profraws(self, profile_dir, output): profraws = [os.path.join(profile_dir, p) for p in self.profraw_files(profile_dir)] utils.run_llvm_profdata(profraws, output) class NativeExeBenchmark(Benchmark): def apct_job_name(self): return 'pgo-collector' def x20_profile_location(self): return os.path.join(X20_BASE_LOCATION, 'raw') def profraw_files(self, profile_dir): if self.name == 'hwui': return ['hwuimacro.profraw', 'hwuimacro_64.profraw', 'hwuimicro.profraw', 'hwuimicro_64.profraw', 'skia_nanobench.profraw', 'skia_nanobench_64.profraw'] elif self.name == 'hwbinder': return ['hwbinder.profraw', 'hwbinder_64.profraw'] class APKBenchmark(Benchmark): def apct_job_name(self): return 'apk-pgo-collector' def x20_profile_location(self): return os.path.join(X20_BASE_LOCATION, 'apk-raw') def profdata_file(self, suffix=''): profdata = os.path.join('art', '{}_arm_arm64.profdata'.format(self.name)) if suffix: profdata += '.' + suffix return profdata def profraw_files(self, profile_dir): return os.listdir(profile_dir) def BenchmarkFactory(benchmark_name): if benchmark_name == 'dex2oat': return APKBenchmark(benchmark_name) elif benchmark_name in ['hwui', 'hwbinder']: return NativeExeBenchmark(benchmark_name) else: raise RuntimeError('Unknown benchmark ' + benchmark_name) def extract_profiles(benchmark, branch, build, output_dir): # The APCT results are stored in # ///// # # The PGO files are in _data_local_tmp_.zip in the above directory. profile_base = os.path.join(benchmark.x20_profile_location(), branch, build, benchmark.apct_job_name()) invocation_dirs = os.listdir(profile_base) if len(invocation_dirs) == 0: raise RuntimeError('No invocations found in {}'.format(profile_base)) if len(invocation_dirs) > 1: # TODO Add option to pick/select an invocation from the command line. raise RuntimeError('More than one invocation found in {}'.format(profile_base)) profile_dir = os.path.join(profile_base, invocation_dirs[0]) zipfiles = [f for f in os.listdir(profile_dir) if f.startswith('_data_local_tmp')] if len(zipfiles) != 1: raise RuntimeError('Expected one zipfile in {}. Found {}'.format(profile_dir, len(zipfiles))) zipfile_name = os.path.join(profile_dir, zipfiles[0]) zip_ref = zipfile.ZipFile(zipfile_name) zip_ref.extractall(output_dir) zip_ref.close() KNOWN_BENCHMARKS = ['ALL', 'dex2oat', 'hwui', 'hwbinder'] def parse_args(): """Parses and returns command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument( 'build', metavar='BUILD', help='Build number to pull from the build server.') parser.add_argument( '-b', '--bug', type=int, help='Bug to reference in commit message.') parser.add_argument( '--use-current-branch', action='store_true', help='Do not repo start a new branch for the update.') parser.add_argument( '--add-do-not-merge', action='store_true', help='Add \'DO NOT MERGE\' to the commit message.') parser.add_argument( '--profdata-suffix', type=str, default='', help='Suffix to append to merged profdata file') parser.add_argument( 'branch', metavar='BRANCH', help='Fetch profiles for BRANCH (e.g. git_qt-release)') parser.add_argument( 'benchmark', metavar='BENCHMARK', help='Update profiles for BENCHMARK. Choices are {}'.format(KNOWN_BENCHMARKS)) parser.add_argument( '--skip-cleanup', '-sc', action='store_true', default=False, help='Skip the cleanup, and leave intermediate files (in /tmp/pgo-profiles-*)') return parser.parse_args() def get_current_profile(benchmark): profile = benchmark.profdata_file() dirname, basename = os.path.split(profile) old_profiles = [f for f in os.listdir(dirname) if f.startswith(basename)] if len(old_profiles) == 0: return '' return os.path.join(dirname, old_profiles[0]) def main(): args = parse_args() if args.benchmark == 'ALL': worklist = KNOWN_BENCHMARKS[1:] else: worklist = [args.benchmark] profiles_project = os.path.join(utils.android_build_top(), 'toolchain', 'pgo-profiles') os.chdir(profiles_project) if not args.use_current_branch: branch_name = 'update-profiles-' + args.build utils.check_call(['repo', 'start', branch_name, '.']) for benchmark_name in worklist: benchmark = BenchmarkFactory(benchmark_name) # Existing profile file, which gets 'rm'-ed from 'git' down below. current_profile = get_current_profile(benchmark) # Extract profiles to a temporary directory. After extraction, we # expect to find one subdirectory with profraw files under the temporary # directory. extract_dir = tempfile.mkdtemp(prefix='pgo-profiles-'+benchmark_name) extract_profiles(benchmark, args.branch, args.build, extract_dir) if len(os.listdir(extract_dir)) != 1: raise RuntimeError("Expected one subdir under {}".format(extract_dir)) extract_subdir = os.path.join(extract_dir, os.listdir(extract_dir)[0]) # Merge profiles. profdata = benchmark.profdata_file(args.profdata_suffix) benchmark.merge_profraws(extract_subdir, profdata) # Construct 'git' commit message. message_lines = [ 'Update PGO profiles for {}'.format(benchmark_name), '', 'The profiles are from build {}.'.format(args.build), '' ] if args.add_do_not_merge: message_lines[0] = '[DO NOT MERGE] ' + message_lines[0] if args.bug: message_lines.append('') message_lines.append('Bug: http://b/{}'.format(args.bug)) message_lines.append('Test: Build (TH)') message = '\n'.join(message_lines) # Invoke git: Delete current profile, add new profile and commit these # changes. if current_profile: utils.check_call(['git', 'rm', current_profile]) utils.check_call(['git', 'add', profdata]) utils.check_call(['git', 'commit', '-m', message]) if not args.skip_cleanup: shutil.rmtree(extract_dir) if __name__ == '__main__': main()