• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright (C) 2019 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# Sample Usage:
18# $ python update_profiles.py 500000 ALL --profdata-suffix 2019-04-15
19#
20# Additional/frequently-used arguments:
21#   -b BUG adds a 'Bug: <BUG>' to the commit message when adding the profiles.
22#   --do-not-merge adds a 'DO NOT MERGE' tag to the commit message to restrict
23#                  automerge of profiles from release branches.
24#
25# Try '-h' for a full list of command line arguments.
26
27import argparse
28import os
29import shutil
30import subprocess
31import sys
32import tempfile
33import zipfile
34
35import utils
36
37from android_build_client import AndroidBuildClient
38
39
40class Benchmark(object):
41
42    def __init__(self, name):
43        self.name = name
44
45    def apct_test_tag(self):
46        raise NotImplementedError()
47
48    def profdata_file(self, suffix=''):
49        profdata = os.path.join(self.name, '{}.profdata'.format(self.name))
50        if suffix:
51            profdata += '.' + suffix
52        return profdata
53
54    def profraw_files(self):
55        raise NotImplementedError()
56
57    def merge_profraws(self, profile_dir, output):
58        profraws = [
59            os.path.join(profile_dir, p)
60            for p in self.profraw_files(profile_dir)
61        ]
62        utils.run_llvm_profdata(profraws, output)
63
64
65class APKBenchmark(Benchmark):
66
67    def apct_test_tag(self):
68        return 'apct/perf/pgo/apk-profile-collector'
69
70    def profdata_file(self, suffix=''):
71        profdata = os.path.join('art',
72                                '{}_arm_arm64.profdata'.format(self.name))
73        if suffix:
74            profdata += '.' + suffix
75        return profdata
76
77    def profraw_files(self, profile_dir):
78        return os.listdir(profile_dir)
79
80
81def BenchmarkFactory(benchmark_name):
82    if benchmark_name == 'dex2oat':
83        return APKBenchmark(benchmark_name)
84    else:
85        raise RuntimeError('Unknown benchmark ' + benchmark_name)
86
87
88def extract_profiles(build, test_tag, build_client, output_dir):
89    pgo_zip = build_client.download_pgo_zip(build, test_tag, output_dir)
90
91    zipfile_name = os.path.join(pgo_zip)
92    zip_ref = zipfile.ZipFile(zipfile_name)
93    zip_ref.extractall(output_dir)
94    zip_ref.close()
95
96
97KNOWN_BENCHMARKS = ['ALL', 'dex2oat']
98
99
100def parse_args():
101    """Parses and returns command line arguments."""
102    parser = argparse.ArgumentParser()
103
104    parser.add_argument(
105        'build',
106        metavar='BUILD',
107        help='Build number to pull from the build server.')
108
109    parser.add_argument(
110        '-b', '--bug', type=int, help='Bug to reference in commit message.')
111
112    parser.add_argument(
113        '--use-current-branch',
114        action='store_true',
115        help='Do not repo start a new branch for the update.')
116
117    parser.add_argument(
118        '--add-do-not-merge',
119        action='store_true',
120        help='Add \'DO NOT MERGE\' to the commit message.')
121
122    parser.add_argument(
123        '--profdata-suffix',
124        type=str,
125        default='',
126        help='Suffix to append to merged profdata file')
127
128    parser.add_argument(
129        'benchmark',
130        metavar='BENCHMARK',
131        help='Update profiles for BENCHMARK.  Choices are {}'.format(
132            KNOWN_BENCHMARKS))
133
134    parser.add_argument(
135        '--skip-cleanup',
136        '-sc',
137        action='store_true',
138        default=False,
139        help='Skip the cleanup, and leave intermediate files (in /tmp/pgo-profiles-*)'
140    )
141
142    return parser.parse_args()
143
144
145def get_current_profile(benchmark):
146    profile = benchmark.profdata_file()
147    dirname, basename = os.path.split(profile)
148
149    old_profiles = [f for f in os.listdir(dirname) if f.startswith(basename)]
150    if len(old_profiles) == 0:
151        return ''
152    return os.path.join(dirname, old_profiles[0])
153
154
155def main():
156    args = parse_args()
157    utils.check_gcertstatus()
158
159    if args.benchmark == 'ALL':
160        worklist = KNOWN_BENCHMARKS[1:]
161    else:
162        worklist = [args.benchmark]
163
164    profiles_project = os.path.join(utils.android_build_top(), 'toolchain',
165                                    'pgo-profiles')
166    os.chdir(profiles_project)
167
168    if not args.use_current_branch:
169        branch_name = 'update-profiles-' + args.build
170        utils.check_call(['repo', 'start', branch_name, '.'])
171
172    build_client = AndroidBuildClient()
173
174    for benchmark_name in worklist:
175        benchmark = BenchmarkFactory(benchmark_name)
176
177        # Existing profile file, which gets 'rm'-ed from 'git' down below.
178        current_profile = get_current_profile(benchmark)
179
180        # Extract profiles to a temporary directory.  After extraction, we
181        # expect to find one subdirectory with profraw files under the temporary
182        # directory.
183        extract_dir = tempfile.mkdtemp(prefix='pgo-profiles-' + benchmark_name)
184        extract_profiles(args.build, benchmark.apct_test_tag(), build_client,
185                         extract_dir)
186
187        extract_subdirs = [
188            os.path.join(extract_dir, sub)
189            for sub in os.listdir(extract_dir)
190            if os.path.isdir(os.path.join(extract_dir, sub))
191        ]
192        if len(extract_subdirs) != 1:
193            raise RuntimeError(
194                'Expected one subdir under {}'.format(extract_dir))
195
196        # Merge profiles.
197        profdata = benchmark.profdata_file(args.profdata_suffix)
198        benchmark.merge_profraws(extract_subdirs[0], profdata)
199
200        # Construct 'git' commit message.
201        message_lines = [
202            'Update PGO profiles for {}'.format(benchmark_name), '',
203            'The profiles are from build {}.'.format(args.build), ''
204        ]
205
206        if args.add_do_not_merge:
207            message_lines[0] = '[DO NOT MERGE] ' + message_lines[0]
208
209        if args.bug:
210            message_lines.append('')
211            message_lines.append('Bug: http://b/{}'.format(args.bug))
212        message_lines.append('Test: Build (TH)')
213        message = '\n'.join(message_lines)
214
215        # Invoke git: Delete current profile, add new profile and commit these
216        # changes.
217        if current_profile:
218            utils.check_call(['git', 'rm', current_profile])
219        utils.check_call(['git', 'add', profdata])
220        utils.check_call(['git', 'commit', '-m', message])
221
222        if not args.skip_cleanup:
223            shutil.rmtree(extract_dir)
224
225
226if __name__ == '__main__':
227    main()
228