1#!/usr/bin/env python 2# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file 3# for details. All rights reserved. Use of this source code is governed by a 4# BSD-style license that can be found in the LICENSE file. 5 6# Clone and build AOSP, using D8 instead of JACK or DX, 7# then run the Android-CTS on the emulator and compare results 8# to a baseline. 9# 10# This script uses the repo manifest file 'third_party/aosp_manifest.xml' 11# which is a snapshot of the aosp repo set. 12# The manifest file can be updated with the following commands: 13# 14# cd build/aosp 15# repo manifest -o ../../third_party/aosp_manifest.xml -r 16# 17# The baseline is a set of `test_result.xml` files in 18# third_party/android_cts_baseline/jack. The current test considered a success 19# if all tests pass that consistently pass in the baseline. 20 21from __future__ import print_function 22from glob import glob 23from itertools import chain 24from os.path import join 25from shutil import copy2 26from subprocess import check_call, Popen 27import argparse 28import os 29import re 30import sys 31import time 32 33import gradle 34import utils 35 36CTS_BASELINE_FILES_DIR = join(utils.REPO_ROOT, 37 'third_party/android_cts_baseline/jack') 38AOSP_MANIFEST_XML = join(utils.REPO_ROOT, 'third_party', 39 'aosp_manifest.xml') 40AOSP_HELPER_SH = join(utils.REPO_ROOT, 'scripts', 'aosp_helper.sh') 41 42D8_JAR = join(utils.REPO_ROOT, 'build/libs/d8.jar') 43COMPATDX_JAR = join(utils.REPO_ROOT, 'build/libs/compatdx.jar') 44D8LOGGER_JAR = join(utils.REPO_ROOT, 'build/libs/d8logger.jar') 45 46AOSP_ROOT = join(utils.REPO_ROOT, 'build/aosp') 47 48AOSP_MANIFEST_URL = 'https://android.googlesource.com/platform/manifest' 49AOSP_PRESET = 'aosp_x86-eng' 50 51AOSP_OUT = join(AOSP_ROOT, 'out') 52OUT_IMG = join(AOSP_ROOT, 'out_img') # output dir for android img build 53OUT_CTS = join(AOSP_ROOT, 'out_cts') # output dir for CTS build 54RESULTS_DIR_BASE = join(OUT_CTS, 'host/linux-x86/cts/android-cts/results') 55CTS_TRADEFED = join(OUT_CTS, 56 'host/linux-x86/cts/android-cts/tools/cts-tradefed') 57 58J_OPTION = '-j8' 59 60EXIT_FAILURE = 1 61 62def parse_arguments(): 63 parser = argparse.ArgumentParser( 64 description = 'Download the AOSP source tree, build an Android image' 65 ' and the CTS targets and run CTS with the emulator on the image.') 66 parser.add_argument('--tool', 67 choices = ['jack', 'dx', 'd8'], 68 default = 'd8', 69 help='compiler tool to use') 70 parser.add_argument('--d8log', 71 metavar = 'FILE', 72 help = 'Enable logging d8 (compatdx) calls to the specified file. Works' 73 ' only with --tool=d8') 74 parser.add_argument('--save-result', 75 metavar = 'FILE', 76 help = 'Save final test_result.xml to the specified file.') 77 parser.add_argument('--no-baseline', 78 action = 'store_true', 79 help = "Don't compare results to baseline hence don't return failure if" 80 ' they differ.') 81 parser.add_argument('--clean-dex', 82 action = 'store_true', 83 help = 'Remove AOSP/dex files always, before the build. By default they' 84 " are removed only if '--tool=d8' and they're older then the D8 tool") 85 return parser.parse_args() 86 87# return False on error 88def remove_aosp_out(): 89 if os.path.exists(AOSP_OUT): 90 if os.path.islink(AOSP_OUT): 91 os.remove(AOSP_OUT) 92 else: 93 print("The AOSP out directory ('" + AOSP_OUT + "') is expected" 94 " to be a symlink", file = sys.stderr) 95 return False 96 return True 97 98# Return list of fully qualified names of tests passing in 99# all the files. 100def consistently_passing_tests_from_test_results(filenames): 101 tree = {} 102 module = None 103 testcase = None 104 # Build a tree with leaves True|False|None for passing, failing and flaky 105 # tests. 106 for f in filenames: 107 for x in utils.read_cts_test_result(f): 108 if type(x) is utils.CtsModule: 109 module = tree.setdefault(x.name, {}) 110 elif type(x) is utils.CtsTestCase: 111 testcase = module.setdefault(x.name, {}) 112 else: 113 outcome = testcase.setdefault(x.name, x.outcome) 114 if outcome is not None and outcome != x.outcome: 115 testcase[x.name] = None 116 117 result = [] 118 for module_name, module in tree.iteritems(): 119 for test_case_name, test_case in module.iteritems(): 120 result.extend(['{}/{}/{}'.format(module_name, test_case_name, test_name) 121 for test_name, test in test_case.iteritems() 122 if test]) 123 124 return result 125 126def setup_and_clean(tool_is_d8, clean_dex): 127 # Two output dirs, one for the android image and one for cts tests. 128 # The output is compiled with d8 and jack, respectively. 129 utils.makedirs_if_needed(AOSP_ROOT) 130 utils.makedirs_if_needed(OUT_IMG) 131 utils.makedirs_if_needed(OUT_CTS) 132 133 # remove dex files older than the current d8 tool 134 counter = 0 135 if tool_is_d8 or clean_dex: 136 if not clean_dex: 137 d8jar_mtime = os.path.getmtime(D8_JAR) 138 dex_files = (chain.from_iterable(glob(join(x[0], '*.dex')) 139 for x in os.walk(OUT_IMG))) 140 for f in dex_files: 141 if clean_dex or os.path.getmtime(f) <= d8jar_mtime: 142 os.remove(f) 143 counter += 1 144 if counter > 0: 145 print('Removed {} dex files.'.format(counter)) 146 147def checkout_aosp(): 148 # checkout AOSP source 149 manifests_dir = join(AOSP_ROOT, '.repo', 'manifests') 150 utils.makedirs_if_needed(manifests_dir) 151 152 copy2(AOSP_MANIFEST_XML, manifests_dir) 153 check_call(['repo', 'init', '-u', AOSP_MANIFEST_URL, '-m', 154 'aosp_manifest.xml', '--depth=1'], cwd = AOSP_ROOT) 155 156 check_call(['repo', 'sync', '-dq', J_OPTION], cwd = AOSP_ROOT) 157 158def Main(): 159 args = parse_arguments() 160 161 if args.d8log and args.tool != 'd8': 162 print("The '--d8log' option works only with '--tool=d8'.", 163 file = sys.stderr) 164 return EXIT_FAILURE 165 166 assert args.tool in ['jack', 'dx', 'd8'] 167 168 jack_option = 'ANDROID_COMPILE_WITH_JACK=' \ 169 + ('true' if args.tool == 'jack' else 'false') 170 171 # DX_ALT_JAR need to be cleared if not set, for 'make' to work properly 172 alt_jar_option = 'DX_ALT_JAR=' 173 if args.tool == 'd8': 174 if args.d8log: 175 alt_jar_option += D8LOGGER_JAR 176 os.environ['D8LOGGER_OUTPUT'] = args.d8log 177 else: 178 alt_jar_option += COMPATDX_JAR 179 180 gradle.RunGradle(['d8','d8logger', 'compatdx']) 181 182 setup_and_clean(args.tool == 'd8', args.clean_dex) 183 184 checkout_aosp() 185 186 # activate OUT_CTS and build Android CTS 187 # AOSP has no clean way to set the output directory. 188 # In order to do incremental builds we apply the following symlink-based 189 # workaround. 190 # Note: this does not work on windows, but the AOSP 191 # doesn't build, either 192 193 if not remove_aosp_out(): 194 return EXIT_FAILURE 195 print("-- Building CTS with 'make {} cts'.".format(J_OPTION)) 196 os.symlink(OUT_CTS, AOSP_OUT) 197 check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, 'cts'], 198 cwd = AOSP_ROOT) 199 200 # activate OUT_IMG and build the Android image 201 if not remove_aosp_out(): 202 return EXIT_FAILURE 203 print("-- Building Android image with 'make {} {} {}'." \ 204 .format(J_OPTION, jack_option, alt_jar_option)) 205 os.symlink(OUT_IMG, AOSP_OUT) 206 check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, jack_option, 207 alt_jar_option], cwd = AOSP_ROOT) 208 209 emulator_proc = Popen([AOSP_HELPER_SH, AOSP_PRESET, 210 'emulator', '-partition-size', '4096', '-wipe-data'], cwd = AOSP_ROOT) 211 212 if emulator_proc.poll() is not None: 213 print("Can't start Android Emulator.", file = sys.stderr) 214 215 check_call([AOSP_HELPER_SH, AOSP_PRESET, 'run-cts', 216 CTS_TRADEFED, 'run', 'cts'], cwd = AOSP_ROOT) 217 218 emulator_proc.terminate() 219 time.sleep(6) # aosp_helper waits to be killed in looping 'sleep 5' 220 221 # find the newest test_result.xml 222 result_dirs = \ 223 [f for f in glob(join(RESULTS_DIR_BASE, '*')) if os.path.isdir(f)] 224 if len(result_dirs) == 0: 225 print("Can't find result directories in ", RESULTS_DIR_BASE) 226 return EXIT_FAILURE 227 result_dirs.sort(key = os.path.getmtime) 228 results_xml = join(result_dirs[-1], 'test_result.xml') 229 230 # print summaries 231 re_summary = re.compile('<Summary ') 232 233 summaries = [('Summary from current test results: ', results_xml)] 234 235 for (title, result_file) in summaries: 236 print(title, result_file) 237 with open(result_file) as f: 238 for line in f: 239 if re_summary.search(line): 240 print(line) 241 break 242 243 if args.no_baseline: 244 r = 0 245 else: 246 print('Comparing test results to baseline:\n') 247 248 passing_tests = consistently_passing_tests_from_test_results([results_xml]) 249 baseline_results = glob(join(CTS_BASELINE_FILES_DIR, '*.xml')) 250 assert len(baseline_results) != 0 251 252 passing_tests_in_baseline = \ 253 consistently_passing_tests_from_test_results(baseline_results) 254 255 missing_or_failing_tests = \ 256 set(passing_tests_in_baseline) - set(passing_tests) 257 258 num_tests = len(missing_or_failing_tests) 259 if num_tests != 0: 260 if num_tests > 1: 261 text = '{} tests that consistently pass in the baseline' \ 262 ' are missing or failing in the current test:'.format(num_tests) 263 else: 264 text = '1 test that consistently passes in the baseline' \ 265 ' is missing or failing in the current test:' 266 print(text) 267 for t in missing_or_failing_tests: 268 print(t) 269 r = EXIT_FAILURE 270 else: 271 r = 0 272 273 if args.save_result: 274 copy2(results_xml, args.save_result) 275 276 return r 277 278if __name__ == '__main__': 279 sys.exit(Main()) 280