1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.ddmlib.testrunner.ITestRunListener; 20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.ddmlib.Log.LogLevel; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.util.ArrayUtil; 30 import com.android.tradefed.util.CommandResult; 31 import com.android.tradefed.util.CommandStatus; 32 import com.android.tradefed.util.FileUtil; 33 import com.android.tradefed.util.StreamUtil; 34 import com.android.tradefed.util.JsonUtil; 35 import com.android.tradefed.util.IRunUtil; 36 import com.android.tradefed.util.RunUtil; 37 import com.android.tradefed.testtype.IAbi; 38 39 import org.json.JSONArray; 40 import org.json.JSONObject; 41 import org.json.JSONException; 42 43 44 import java.io.File; 45 import java.io.FileNotFoundException; 46 import java.io.FileWriter; 47 import java.io.IOException; 48 import java.io.InputStream; 49 import java.nio.file.Paths; 50 import java.util.TreeSet; 51 import java.util.Set; 52 import java.util.Collection; 53 import java.util.ArrayList; 54 55 /** 56 * A Test that runs a vts multi device test package (part of Vendor Test Suite, 57 * VTS) on given device. 58 */ 59 60 @OptionClass(alias = "vtsmultidevicetest") 61 public class VtsMultiDeviceTest implements IDeviceTest, IRemoteTest, ITestFilterReceiver, 62 IRuntimeHintProvider, ITestCollector, IBuildReceiver, IAbiReceiver { 63 64 static final String ANDROIDDEVICE = "AndroidDevice"; 65 static final String BUILD = "build"; 66 static final String BUILD_ID = "build_id"; 67 static final String BUILD_TARGET = "build_target"; 68 static final String COVERAGE_PROPERTY = "ro.vts.coverage"; 69 static final String DATA_FILE_PATH = "data_file_path"; 70 static final String LOG_PATH = "log_path"; 71 static final String NAME = "name"; 72 static final String OS_NAME = "os.name"; 73 static final String WINDOWS = "Windows"; 74 static final String PYTHONPATH = "PYTHONPATH"; 75 static final String SERIAL = "serial"; 76 static final String TEST_SUITE = "test_suite"; 77 static final String VIRTUAL_ENV_PATH = "VIRTUALENVPATH"; 78 static final String ABI_NAME = "abi_name"; 79 static final String ABI_BITNESS = "abi_bitness"; 80 static final String SKIP_ON_32BIT_ABI = "skip_on_32bit_abi"; 81 static final String SKIP_ON_64BIT_ABI = "skip_on_64bit_abi"; 82 static final String SKIP_IF_THERMAL_THROTTLING = "skip_if_thermal_throttling"; 83 static final String RUN_32BIT_ON_64BIT_ABI = "run_32bit_on_64bit_abi"; 84 static final String VTS = "vts"; 85 static final String CONFIG_FILE_EXTENSION = ".config"; 86 static final String INCLUDE_FILTER = "include_filter"; 87 static final String EXCLUDE_FILTER = "exclude_filter"; 88 static final String BINARY_TEST_SOURCE = "binary_test_source"; 89 static final String BINARY_TEST_WORKING_DIRECTORY = "binary_test_working_directory"; 90 static final String BINARY_TEST_ENVP = "binary_test_envp"; 91 static final String BINARY_TEST_ARGS = "binary_test_args"; 92 static final String BINARY_TEST_LD_LIBRARY_PATH = "binary_test_ld_library_path"; 93 static final String BINARY_TEST_PROFILING_LIBRARY_PATH = "binary_test_profiling_library_path"; 94 static final String BINARY_TEST_DISABLE_FRAMEWORK = "binary_test_disable_framework"; 95 static final String BINARY_TEST_STOP_NATIVE_SERVERS = "binary_test_stop_native_servers"; 96 static final String BINARY_TEST_TYPE_GTEST = "gtest"; 97 static final String BINARY_TEST_TYPE_LLVMFUZZER = "llvmfuzzer"; 98 static final String BINARY_TEST_TYPE_HAL_HIDL_GTEST = "hal_hidl_gtest"; 99 static final String BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST = "hal_hidl_replay_test"; 100 static final String BINARY_TEST_TYPE_HOST_BINARY_TEST = "host_binary_test"; 101 static final String ENABLE_PROFILING = "enable_profiling"; 102 static final String ENABLE_COVERAGE = "enable_coverage"; 103 static final String NATIVE_SERVER_PROCESS_NAME = "native_server_process_name"; 104 static final String PASSTHROUGH_MODE = "passthrough_mode"; 105 static final String PRECONDITION_HWBINDER_SERVICE = "precondition_hwbinder_service"; 106 static final String PRECONDITION_FEATURE = "precondition_feature"; 107 static final String PRECONDITION_FILE_PATH_PREFIX = "precondition_file_path_prefix"; 108 static final String PRECONDITION_LSHAL = "precondition_lshal"; 109 static final String ENABLE_SYSTRACE = "enable_systrace"; 110 static final String HAL_HIDL_REPLAY_TEST_TRACE_PATHS = "hal_hidl_replay_test_trace_paths"; 111 static final String HAL_HIDL_PACKAGE_NAME = "hal_hidl_package_name"; 112 static final String SYSTRACE_PROCESS_NAME = "systrace_process_name"; 113 static final String TEMPLATE_BINARY_TEST_PATH = "vts/testcases/template/binary_test/binary_test"; 114 static final String TEMPLATE_GTEST_BINARY_TEST_PATH = "vts/testcases/template/gtest_binary_test/gtest_binary_test"; 115 static final String TEMPLATE_LLVMFUZZER_TEST_PATH = "vts/testcases/template/llvmfuzzer_test/llvmfuzzer_test"; 116 static final String TEMPLATE_HAL_HIDL_GTEST_PATH = "vts/testcases/template/hal_hidl_gtest/hal_hidl_gtest"; 117 static final String TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH = "vts/testcases/template/hal_hidl_replay_test/hal_hidl_replay_test"; 118 static final String TEMPLATE_HOST_BINARY_TEST_PATH = "vts/testcases/template/host_binary_test/host_binary_test"; 119 static final String TEST_RUN_SUMMARY_FILE_NAME = "test_run_summary.json"; 120 static final float DEFAULT_TARGET_VERSION = -1; 121 static final String DEFAULT_TESTCASE_CONFIG_PATH = "vts/tools/vts-tradefed/res/default/DefaultTestCase.config"; 122 123 private ITestDevice mDevice = null; 124 private IAbi mAbi = null; 125 126 @Option(name = "test-timeout", description = "maximum amount of time" 127 + "(im milliseconds) tests are allowed to run", 128 isTimeVal = true) 129 private long mTestTimeout = 1000 * 60 * 60 * 3; 130 131 @Option(name = "test-module-name", 132 description = "The name for a test module.") 133 private String mTestModuleName = null; 134 135 @Option(name = "test-case-path", 136 description = "The path for test case.") 137 private String mTestCasePath = null; 138 139 @Option(name = "test-config-path", 140 description = "The path for test case config file.") 141 private String mTestConfigPath = null; 142 143 @Option(name = "precondition-hwbinder-service", 144 description = "The name of a HW binder service needed to run the test.") 145 private String mPreconditionHwBinderServiceName = null; 146 147 @Option(name = "precondition-feature", 148 description = "The name of a `pm`-listable feature needed to run the test.") 149 private String mPreconditionFeature = null; 150 151 @Option(name = "precondition-file-path-prefix", 152 description = "The path prefix of a file (e.g., shared lib) needed to run the test.") 153 private String mPreconditionFilePathPrefix = null; 154 155 @Option(name = "precondition-lshal", 156 description = "The name of a `lshal`-listable feature needed to run the test.") 157 private String mPreconditionLshal = null; 158 159 @Option(name = "use-stdout-logs", 160 description = "Flag that determines whether to use std:out to parse output.") 161 private boolean mUseStdoutLogs = false; 162 163 @Option(name = "include-filter", 164 description = "The positive filter of the test names to run.") 165 private Set<String> mIncludeFilters = new TreeSet<>(); 166 167 @Option(name = "exclude-filter", 168 description = "The negative filter of the test names to run.") 169 private Set<String> mExcludeFilters = new TreeSet<>(); 170 171 @Option(name = "runtime-hint", description = "The hint about the test's runtime.", 172 isTimeVal = true) 173 private long mRuntimeHint = 60000; // 1 minute 174 175 @Option(name = "enable-profiling", description = "Enable profiling for the tests.") 176 private boolean mEnableProfiling = false; 177 178 @Option(name = "enable-systrace", description = "Enable systrace for the tests.") 179 private boolean mEnableSystrace = false; 180 181 @Option(name = "enable-coverage", 182 description = "Enable coverage for the tests. In order for coverage to be measured, " + 183 "ro.vts.coverage system must have value \"1\" to indicate the target " + 184 "build is coverage instrumented.") 185 private boolean mEnableCoverage = true; 186 187 // Another design option is to parse a string or use enum for host preference on BINDER, 188 // PASSTHROUGH and DEFAULT(which is BINDER). Also in the future, we might want to deal with 189 // the case of target preference on PASSTHROUGH (if host does not specify to use BINDER mode). 190 @Option(name = "passthrough-mode", description = "Set getStub to use passthrough mode. " 191 + "Value true means use passthrough mode if available; false for binderized mode if " 192 + "available. Default is false") 193 private boolean mPassthroughMode = false; 194 195 @Option(name = "skip-on-32bit-abi", 196 description = "Whether to skip tests on 32bit ABI.") 197 private boolean mSkipOn32BitAbi = false; 198 199 @Option(name = "skip-on-64bit-abi", 200 description = "Whether to skip tests on 64bit ABI.") 201 private boolean mSkipOn64BitAbi = false; 202 203 @Option(name = "skip-if-thermal-throttling", 204 description = "Whether to skip tests if target device suffers from thermal throttling.") 205 private boolean mSkipIfThermalThrottling = false; 206 207 @Option(name = "run-32bit-on-64bit-abi", 208 description = "Whether to run 32bit tests on 64bit ABI.") 209 private boolean mRun32bBitOn64BitAbi = false; 210 211 @Option(name = "binary-test-source", 212 description = "Binary test source paths relative to vts testcase directory on host." 213 + "Format of tags:" 214 + " <source>: source without tag." 215 + " <tag>::<source>: source with tag. Can be used to separate 32bit and 64" 216 + " bit tests with same file name." 217 + " <tag1>::<source1>, <tag1>::<source2>, <tag2>::<source3>: multiple" 218 + " sources connected by comma. White spaces in-between" 219 + " will be ignored." 220 + "Format of each source string:" 221 + " <source file>: push file and create test case." 222 + " Source is relative path of source file under vts's testcases" 223 + " folder. Source file will be pushed to working directory" 224 + " discarding original directory structure, and test cases will" 225 + " be created using the pushed file." 226 + " <source file>-><destination file>: push file and create test case." 227 + " Destination path is absolute path on device. Test cases will" 228 + " be created using the pushed file." 229 + " <source file>->: push file only." 230 + " Push the source file to its' tag's corresponding" 231 + " working directory. Test case will not be created on" 232 + " this file. This is equivalent to but simpler than specifying a" 233 + " working directory for the tag and use VtsFilePusher to push the" 234 + " file to the directory." 235 + " -><destination file>: create test case only." 236 + " Destination is absolute device side path." 237 + " Note: each path in source string can be a directory. However, the" 238 + " default binary test runner and gtest binary test runner does not" 239 + " support creating test cases from a directory. You will need to" 240 + " override the binary test runner's CreateTestCase method in python." 241 + " If you wish to push a source file to a specific destination and not" 242 + " create a test case from it, please use VtsFilePusher.") 243 private Collection<String> mBinaryTestSource = new ArrayList<>(); 244 245 @Option(name = "binary-test-working-directory", description = "Working directories for binary " 246 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 247 + "However, each tag should only has one working directory. This option is optional for " 248 + "binary tests. If not specified, different directories will be used for files with " 249 + "different tags.") 250 private Collection<String> mBinaryTestWorkingDirectory = new ArrayList<>(); 251 252 @Option(name = "binary-test-envp", description = "Additional environment path for binary " 253 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 254 + "There can be multiple instances of binary-test-envp for a same tag, which will " 255 + "later automatically be combined.") 256 private Collection<String> mBinaryTestEnvp = new ArrayList<>(); 257 258 @Option(name = "binary-test-args", description = "Additional args or flags for binary " 259 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 260 + "There can be multiple instances of binary-test-args for a same tag, which will " 261 + "later automatically be combined.") 262 private Collection<String> mBinaryTestArgs = new ArrayList<>(); 263 264 @Option(name = "binary-test-ld-library-path", description = "LD_LIBRARY_PATH for binary " 265 + "tests. Tags can be added to the front of each instance using '::' as delimiter. " 266 + "Multiple directories can be added under a same tag using ':' as delimiter. " 267 + "There can be multiple instances of ld-library-path for a same tag, which will " 268 + "later automatically be combined using ':' as delimiter. Paths without a tag " 269 + "will only used for binaries without tag. This option is optional for binary tests.") 270 private Collection<String> mBinaryTestLdLibraryPath = new ArrayList<>(); 271 272 @Option(name = "binary-test-profiling-library-path", description = "Path to lookup and load " 273 + "profiling libraries for tests with profiling enabled. Tags can be added to the " 274 + "front of each directory using '::' as delimiter. Only one directory could be " 275 + "specified for the same tag. This option is optional for binary tests. If not " 276 + "specified, default directories will be used for files with different tags.") 277 private Collection<String> mBinaryTestProfilingLibraryPath = new ArrayList<>(); 278 279 @Option(name = "binary-test-disable-framework", description = "Adb stop/start before/after test.") 280 private boolean mBinaryTestDisableFramework = false; 281 282 @Option(name = "binary-test-stop-native-servers", 283 description = "Set to stop all properly configured native servers during the testing.") 284 private boolean mBinaryTestStopNativeServers = false; 285 286 @Option(name = "native-server-process-name", 287 description = "Name of a native server process. The runner checks to make sure " 288 + "each specified native server process is not running after the framework stop.") 289 private Collection<String> mNativeServerProcessName = new ArrayList<>(); 290 291 @Option(name = "binary-test-type", description = "Binary test type. Only specify this when " 292 + "running an extended binary test without a python test file. Available options: gtest") 293 private String mBinaryTestType = ""; 294 295 @Option(name = "hal-hidl-replay-test-trace-path", description = "The path of a trace file to replay.") 296 private Collection<String> mHalHidlReplayTestTracePaths = new ArrayList<>(); 297 298 @Option(name = "hal-hidl-package-name", description = "The name of a target HIDL HAL package " 299 + "e.g., 'android.hardware.light@2.0'.") 300 private String mHalHidlPackageName = null; 301 302 @Option(name = "systrace-process-name", description = "Process name for systrace.") 303 private String mSystraceProcessName = null; 304 305 @Option(name = "collect-tests-only", 306 description = "Only invoke the test binary to collect list of applicable test cases. " 307 + "All test run callbacks will be triggered, but test execution will " 308 + "not be actually carried out.") 309 private boolean mCollectTestsOnly = false; 310 311 // This variable is set in order to include the directory that contains the 312 // python test cases. This is set before calling the method. 313 // {@link #doRunTest(IRunUtil, String, String)}. 314 public String mPythonPath = null; 315 316 @Option(name = "python-binary", description = "python binary to use " 317 + "(optional)") 318 private String mPythonBin = null; 319 private IRunUtil mRunUtil = null; 320 private IBuildInfo mBuildInfo = null; 321 private String mRunName = "VtsHostDrivenTest"; 322 // the path of a dir which contains the test data files. 323 private String mTestCaseDataDir = "./"; 324 325 /** 326 * @return the mRunUtil 327 */ getRunUtil()328 public IRunUtil getRunUtil() { 329 return mRunUtil; 330 } 331 332 /** 333 * @param mRunUtil the mRunUtil to set 334 */ setRunUtil(IRunUtil mRunUtil)335 public void setRunUtil(IRunUtil mRunUtil) { 336 this.mRunUtil = mRunUtil; 337 } 338 339 /** 340 * {@inheritDoc} 341 */ 342 @Override setDevice(ITestDevice device)343 public void setDevice(ITestDevice device) { 344 mDevice = device; 345 } 346 347 /** 348 * {@inheritDoc} 349 */ 350 @Override getDevice()351 public ITestDevice getDevice() { 352 return mDevice; 353 } 354 setTestCasePath(String path)355 public void setTestCasePath(String path){ 356 mTestCasePath = path; 357 } 358 setTestConfigPath(String path)359 public void setTestConfigPath(String path){ 360 mTestConfigPath = path; 361 } 362 363 /** 364 * {@inheritDoc} 365 */ 366 @Override addIncludeFilter(String filter)367 public void addIncludeFilter(String filter) { 368 mIncludeFilters.add(cleanFilter(filter)); 369 } 370 371 /** 372 * {@inheritDoc} 373 */ 374 @Override addAllIncludeFilters(Set<String> filters)375 public void addAllIncludeFilters(Set<String> filters) { 376 for (String filter : filters) { 377 mIncludeFilters.add(cleanFilter(filter)); 378 } 379 } 380 381 /** 382 * {@inheritDoc} 383 */ 384 @Override addExcludeFilter(String filter)385 public void addExcludeFilter(String filter) { 386 mExcludeFilters.add(cleanFilter(filter)); 387 } 388 389 /** 390 * {@inheritDoc} 391 */ 392 @Override addAllExcludeFilters(Set<String> filters)393 public void addAllExcludeFilters(Set<String> filters) { 394 for (String filter : filters) { 395 mExcludeFilters.add(cleanFilter(filter)); 396 } 397 } 398 399 /* 400 * Conforms filters using a {@link com.android.ddmlib.testrunner.TestIdentifier} format 401 * to be recognized by the GTest executable. 402 */ cleanFilter(String filter)403 private String cleanFilter(String filter) { 404 return filter.replace('#', '.'); 405 } 406 407 /** 408 * {@inheritDoc} 409 */ 410 @Override getRuntimeHint()411 public long getRuntimeHint() { 412 return mRuntimeHint; 413 } 414 415 /** 416 * {@inheritDoc} 417 */ 418 @Override setCollectTestsOnly(boolean shouldCollectTest)419 public void setCollectTestsOnly(boolean shouldCollectTest) { 420 mCollectTestsOnly = shouldCollectTest; 421 } 422 423 /** 424 * {@inheritDoc} 425 */ 426 @SuppressWarnings("deprecation") 427 @Override run(ITestInvocationListener listener)428 public void run(ITestInvocationListener listener) 429 throws IllegalArgumentException, DeviceNotAvailableException { 430 if (mDevice == null) { 431 throw new DeviceNotAvailableException("Device has not been set"); 432 } 433 434 if (mTestCasePath == null) { 435 if (!mBinaryTestSource.isEmpty()) { 436 String template; 437 switch (mBinaryTestType) { 438 case BINARY_TEST_TYPE_GTEST: 439 template = TEMPLATE_GTEST_BINARY_TEST_PATH; 440 break; 441 case BINARY_TEST_TYPE_HAL_HIDL_GTEST: 442 template = TEMPLATE_HAL_HIDL_GTEST_PATH; 443 break; 444 case BINARY_TEST_TYPE_HOST_BINARY_TEST: 445 template = TEMPLATE_HOST_BINARY_TEST_PATH; 446 break; 447 default: 448 template = TEMPLATE_BINARY_TEST_PATH; 449 } 450 CLog.i("Using default test case template at %s.", template); 451 setTestCasePath(template); 452 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST)) { 453 setTestCasePath(TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH); 454 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_LLVMFUZZER)) { 455 // Fuzz test don't need test-case-path. 456 setTestCasePath(TEMPLATE_LLVMFUZZER_TEST_PATH); 457 } else { 458 throw new IllegalArgumentException("test-case-path is not set."); 459 } 460 } 461 462 setPythonPath(); 463 464 if (mPythonBin == null) { 465 mPythonBin = getPythonBinary(); 466 } 467 468 if (mRunUtil == null){ 469 mRunUtil = new RunUtil(); 470 mRunUtil.setEnvVariable(PYTHONPATH, mPythonPath); 471 } 472 473 doRunTest(listener); 474 } 475 476 /** 477 * {@inheritDoc} 478 */ 479 @Override setBuild(IBuildInfo buildInfo)480 public void setBuild(IBuildInfo buildInfo) { 481 mBuildInfo = buildInfo; 482 } 483 484 /** 485 * Populate a jsonObject with default fields. 486 * This method uses deepMergeJsonObjects method from JsonUtil to merge the default config file with the target 487 * config file. Field already defined in target config file will not be overwritten. Also, JSONArray will not be 488 * deep merged. 489 * 490 * @param jsonObject the target json object to populate 491 * @param testCaseDataDir data file path 492 * @throws IOException 493 * @throws JSONException 494 */ populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir)495 private void populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir) 496 throws IOException, JSONException { 497 CLog.i("Populating default fields to json object from %s", DEFAULT_TESTCASE_CONFIG_PATH); 498 String content = FileUtil.readStringFromFile(new File(mTestCaseDataDir, DEFAULT_TESTCASE_CONFIG_PATH)); 499 JSONObject defaultJsonObject = new JSONObject(content); 500 501 JsonUtil.deepMergeJsonObjects(jsonObject, defaultJsonObject); 502 } 503 504 /** 505 * This method reads the provided VTS runner test json config, adds or updates some of its 506 * fields (e.g., to add build info and device serial IDs), and returns the updated json object. 507 * This method calls populateDefaultJsonFields to populate the config JSONObject if the config file is missing 508 * or some required fields is missing from the JSONObject. If test name is not specified, this method 509 * will use the config file's file name file without extension as test name. If config file is missing, this method 510 * will use the test case's class name as test name. 511 * 512 * @param log_path the path of a directory to store the VTS runner logs. 513 * @return the updated JSONObject as the new test config. 514 */ updateVtsRunnerTestConfig(JSONObject jsonObject)515 private void updateVtsRunnerTestConfig(JSONObject jsonObject) 516 throws IOException, JSONException, RuntimeException { 517 CLog.i("Load vendor test config %s", "/config/google-tradefed-vts-config.config"); 518 InputStream config = getClass().getResourceAsStream("/config/google-tradefed-vts-config.config"); 519 if (config != null) { 520 try { 521 String content = StreamUtil.getStringFromStream(config); 522 CLog.i("Loaded vendor test config %s", content); 523 if (content != null) { 524 JSONObject vendorConfigJson = new JSONObject(content); 525 JsonUtil.deepMergeJsonObjects(jsonObject, vendorConfigJson); 526 } 527 } catch(IOException e) { 528 throw new RuntimeException("Failed to read vendor config json file"); 529 } catch(JSONException e) { 530 throw new RuntimeException("Failed to build updated vendor config json data"); 531 } 532 } 533 534 CLog.i("Load original test config %s %s", mTestCaseDataDir, mTestConfigPath); 535 String content = null; 536 537 if (mTestConfigPath != null) { 538 content = FileUtil.readStringFromFile(new File(Paths.get(mTestCaseDataDir, mTestConfigPath).toString())); 539 } 540 541 CLog.i("Loaded original test config %s", content); 542 if (content != null) { 543 JsonUtil.deepMergeJsonObjects(jsonObject, new JSONObject(content)); 544 } 545 populateDefaultJsonFields(jsonObject, mTestCaseDataDir); 546 CLog.i("Built a Json object using the loaded original test config"); 547 548 JSONArray deviceArray = new JSONArray(); 549 JSONObject deviceItemObject = new JSONObject(); 550 deviceItemObject.put(SERIAL, mDevice.getSerialNumber()); 551 boolean coverageBuild = false; 552 try { 553 deviceItemObject.put("product_type", mDevice.getProductType()); 554 deviceItemObject.put("product_variant", mDevice.getProductVariant()); 555 deviceItemObject.put("build_alias", mDevice.getBuildAlias()); 556 deviceItemObject.put("build_id", mDevice.getBuildId()); 557 deviceItemObject.put("build_flavor", mDevice.getBuildFlavor()); 558 String coverageProperty = mDevice.getProperty(COVERAGE_PROPERTY); 559 coverageBuild = coverageProperty != null && coverageProperty.equals("1"); 560 } catch (DeviceNotAvailableException e) { 561 CLog.e("A device not available - continuing"); 562 throw new RuntimeException("Failed to get device information"); 563 } 564 deviceArray.put(deviceItemObject); 565 566 JSONArray testBedArray = (JSONArray) jsonObject.get("test_bed"); 567 if (testBedArray.length() == 0) { 568 JSONObject device = new JSONObject(); 569 String testName; 570 if (mTestModuleName != null) { 571 testName = mTestModuleName; 572 } else { 573 CLog.w("--test-module-name not set (not recommended); deriving automatically"); 574 if (mTestConfigPath != null) { 575 testName = new File(mTestConfigPath).getName(); 576 testName = testName.replace(CONFIG_FILE_EXTENSION, ""); 577 } else if (mTestCasePath != null) { 578 testName = new File(mTestCasePath).getName(); 579 } else { 580 throw new RuntimeException( 581 "Failed to derive test module name; use --test-module-name option"); 582 } 583 } 584 CLog.logAndDisplay(LogLevel.INFO, "Setting test name as %s", testName); 585 device.put(NAME, testName); 586 device.put(ANDROIDDEVICE, deviceArray); 587 testBedArray.put(device); 588 } else if (testBedArray.length() == 1) { 589 JSONObject device = (JSONObject) testBedArray.get(0); 590 device.put(ANDROIDDEVICE, deviceArray); 591 } else { 592 CLog.e("Multi-device not yet supported: %d devices requested", 593 testBedArray.length()); 594 throw new RuntimeException("Failed to produce VTS runner test config"); 595 } 596 jsonObject.put(DATA_FILE_PATH, mTestCaseDataDir); 597 CLog.i("Added %s = %s to the Json object", DATA_FILE_PATH, mTestCaseDataDir); 598 599 JSONObject build = new JSONObject(); 600 build.put(BUILD_ID, mBuildInfo.getBuildId()); 601 build.put(BUILD_TARGET, mBuildInfo.getBuildTargetName()); 602 jsonObject.put(BUILD, build); 603 CLog.i("Added %s to the Json object", BUILD); 604 605 JSONObject suite = new JSONObject(); 606 suite.put(NAME, mBuildInfo.getTestTag()); 607 suite.put(INCLUDE_FILTER, new JSONArray(mIncludeFilters)); 608 CLog.i("Added include filter to test suite: %s", mIncludeFilters); 609 suite.put(EXCLUDE_FILTER, new JSONArray(mExcludeFilters)); 610 CLog.i("Added exclude filter to test suite: %s", mExcludeFilters); 611 jsonObject.put(TEST_SUITE, suite); 612 CLog.i("Added %s to the Json object", TEST_SUITE); 613 614 if (mAbi != null) { 615 jsonObject.put(ABI_NAME, mAbi.getName()); 616 CLog.i("Added %s to the Json object", ABI_NAME); 617 jsonObject.put(ABI_BITNESS, mAbi.getBitness()); 618 CLog.i("Added %s to the Json object", ABI_BITNESS); 619 } 620 if (mSkipOn32BitAbi) { 621 jsonObject.put(SKIP_ON_32BIT_ABI, mSkipOn32BitAbi); 622 CLog.i("Added %s to the Json object", SKIP_ON_32BIT_ABI); 623 } 624 if (mSkipOn64BitAbi) { 625 jsonObject.put(SKIP_ON_64BIT_ABI, mSkipOn64BitAbi); 626 CLog.i("Added %s to the Json object", SKIP_ON_64BIT_ABI); 627 } else if (mRun32bBitOn64BitAbi) { 628 jsonObject.put(RUN_32BIT_ON_64BIT_ABI, mRun32bBitOn64BitAbi); 629 CLog.i("Added %s to the Json object", RUN_32BIT_ON_64BIT_ABI); 630 } 631 if (mSkipIfThermalThrottling) { 632 jsonObject.put(SKIP_IF_THERMAL_THROTTLING, mSkipIfThermalThrottling); 633 CLog.i("Added %s to the Json object", SKIP_IF_THERMAL_THROTTLING); 634 } 635 636 if (!mBinaryTestSource.isEmpty()) { 637 jsonObject.put(BINARY_TEST_SOURCE, new JSONArray(mBinaryTestSource)); 638 CLog.i("Added %s to the Json object", BINARY_TEST_SOURCE); 639 } 640 if (!mBinaryTestWorkingDirectory.isEmpty()) { 641 jsonObject.put(BINARY_TEST_WORKING_DIRECTORY, 642 new JSONArray(mBinaryTestWorkingDirectory)); 643 CLog.i("Added %s to the Json object", BINARY_TEST_WORKING_DIRECTORY); 644 } 645 if (!mBinaryTestEnvp.isEmpty()) { 646 jsonObject.put(BINARY_TEST_ENVP, new JSONArray(mBinaryTestEnvp)); 647 CLog.i("Added %s to the Json object", BINARY_TEST_ENVP); 648 } 649 if (!mBinaryTestArgs.isEmpty()) { 650 jsonObject.put(BINARY_TEST_ARGS, new JSONArray(mBinaryTestArgs)); 651 CLog.i("Added %s to the Json object", BINARY_TEST_ARGS); 652 } 653 if (!mBinaryTestLdLibraryPath.isEmpty()) { 654 jsonObject.put(BINARY_TEST_LD_LIBRARY_PATH, 655 new JSONArray(mBinaryTestLdLibraryPath)); 656 CLog.i("Added %s to the Json object", BINARY_TEST_LD_LIBRARY_PATH); 657 } 658 659 if (mEnableProfiling) { 660 jsonObject.put(ENABLE_PROFILING, mEnableProfiling); 661 CLog.i("Added %s to the Json object", ENABLE_PROFILING); 662 } 663 if (mEnableSystrace) { 664 jsonObject.put(ENABLE_SYSTRACE, mEnableSystrace); 665 CLog.i("Added %s to the Json object", ENABLE_SYSTRACE); 666 } 667 if (mEnableCoverage) { 668 if (coverageBuild) { 669 jsonObject.put(ENABLE_COVERAGE, mEnableCoverage); 670 CLog.i("Added %s to the Json object", ENABLE_COVERAGE); 671 } else { 672 CLog.i("Device build has coverage disabled"); 673 } 674 } 675 676 if (mPreconditionHwBinderServiceName != null) { 677 jsonObject.put(PRECONDITION_HWBINDER_SERVICE, mPreconditionHwBinderServiceName); 678 CLog.i("Added %s to the Json object", PRECONDITION_HWBINDER_SERVICE); 679 } 680 681 if (mPreconditionFeature != null) { 682 jsonObject.put(PRECONDITION_FEATURE, mPreconditionFeature); 683 CLog.i("Added %s to the Json object", PRECONDITION_FEATURE); 684 } 685 686 if (mPreconditionFilePathPrefix != null) { 687 jsonObject.put(PRECONDITION_FILE_PATH_PREFIX, mPreconditionFilePathPrefix); 688 CLog.i("Added %s to the Json object", PRECONDITION_FILE_PATH_PREFIX); 689 } 690 691 if (mPreconditionLshal != null) { 692 jsonObject.put(PRECONDITION_LSHAL, mPreconditionLshal); 693 CLog.i("Added %s to the Json object", PRECONDITION_LSHAL); 694 } 695 696 if (!mBinaryTestProfilingLibraryPath.isEmpty()) { 697 jsonObject.put(BINARY_TEST_PROFILING_LIBRARY_PATH, 698 new JSONArray(mBinaryTestProfilingLibraryPath)); 699 CLog.i("Added %s to the Json object", BINARY_TEST_PROFILING_LIBRARY_PATH); 700 } 701 702 if (mBinaryTestDisableFramework) { 703 jsonObject.put(BINARY_TEST_DISABLE_FRAMEWORK, mBinaryTestDisableFramework); 704 CLog.i("Added %s to the Json object", BINARY_TEST_DISABLE_FRAMEWORK); 705 } 706 707 if (mBinaryTestStopNativeServers) { 708 jsonObject.put(BINARY_TEST_STOP_NATIVE_SERVERS, mBinaryTestStopNativeServers); 709 CLog.i("Added %s to the Json object", BINARY_TEST_STOP_NATIVE_SERVERS); 710 } 711 712 if (!mNativeServerProcessName.isEmpty()) { 713 jsonObject.put(NATIVE_SERVER_PROCESS_NAME, new JSONArray(mNativeServerProcessName)); 714 CLog.i("Added %s to the Json object", NATIVE_SERVER_PROCESS_NAME); 715 } 716 717 if (!mHalHidlReplayTestTracePaths.isEmpty()) { 718 jsonObject.put(HAL_HIDL_REPLAY_TEST_TRACE_PATHS, 719 new JSONArray(mHalHidlReplayTestTracePaths)); 720 CLog.i("Added %s to the Json object", HAL_HIDL_REPLAY_TEST_TRACE_PATHS); 721 } 722 723 if (mHalHidlPackageName != null) { 724 jsonObject.put(HAL_HIDL_PACKAGE_NAME, mHalHidlPackageName); 725 CLog.i("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 726 } 727 728 if (mSystraceProcessName != null) { 729 jsonObject.put(SYSTRACE_PROCESS_NAME, mSystraceProcessName); 730 CLog.i("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 731 } 732 733 if (mPassthroughMode) { 734 jsonObject.put(PASSTHROUGH_MODE, mPassthroughMode); 735 CLog.i("Added %s to the Json object", PASSTHROUGH_MODE); 736 } 737 } 738 739 /** 740 * Log a test module execution status to device logcat. 741 * 742 * @param status 743 * @return true if succesful, false otherwise 744 */ printToDeviceLogcatAboutTestModuleStatus(String status)745 private boolean printToDeviceLogcatAboutTestModuleStatus(String status) { 746 try { 747 mDevice.executeShellCommand(String.format( 748 "log -p i -t \"VTS\" \"[Test Module] %s %s\"", mTestModuleName, status)); 749 } catch (DeviceNotAvailableException e) { 750 CLog.w("Device unavailable while trying to write a message to logcat."); 751 return false; 752 } 753 return true; 754 } 755 756 /** 757 * This method prepares a command for the test and runs the python file as 758 * given in the arguments. 759 * 760 * @param listener 761 * @param runUtil 762 * @param mTestCasePath 763 * @param mTestConfigPath 764 * @throws RuntimeException 765 * @throws IllegalArgumentException 766 */ doRunTest(ITestRunListener listener)767 private void doRunTest(ITestRunListener listener) throws RuntimeException, IllegalArgumentException { 768 CLog.i("Device serial number: " + mDevice.getSerialNumber()); 769 770 JSONObject jsonObject = new JSONObject(); 771 File vtsRunnerLogDir = null; 772 try { 773 vtsRunnerLogDir = FileUtil.createTempDir("vts-runner-log"); 774 updateVtsRunnerTestConfig(jsonObject); 775 776 jsonObject.put(LOG_PATH, vtsRunnerLogDir.getAbsolutePath()); 777 CLog.i("Added %s to the Json object", LOG_PATH); 778 } catch(IOException e) { 779 throw new RuntimeException("Failed to read test config json file"); 780 } catch(JSONException e) { 781 throw new RuntimeException("Failed to build updated test config json data"); 782 } 783 784 CLog.i("config json: %s", jsonObject.toString()); 785 786 String jsonFilePath = null; 787 try { 788 File tmpFile = FileUtil.createTempFile( 789 mBuildInfo.getTestTag() + "-config-" + mBuildInfo.getDeviceSerial(), ".json"); 790 jsonFilePath = tmpFile.getAbsolutePath(); 791 CLog.i("config json file path: %s", jsonFilePath); 792 FileWriter fw = new FileWriter(jsonFilePath); 793 fw.write(jsonObject.toString()); 794 fw.close(); 795 } catch(IOException e) { 796 throw new RuntimeException("Failed to create device config json file"); 797 } 798 799 if (mPythonBin == null){ 800 mPythonBin = getPythonBinary(); 801 } 802 String[] baseOpts = {mPythonBin, "-m"}; 803 String[] testModule = {mTestCasePath.replace("/", "."), jsonFilePath}; 804 String[] cmd; 805 cmd = ArrayUtil.buildArray(baseOpts, testModule); 806 807 printToDeviceLogcatAboutTestModuleStatus("BEGIN"); 808 CommandResult commandResult = mRunUtil.runTimedCmd(mTestTimeout, cmd); 809 810 if (commandResult != null) { 811 CommandStatus commandStatus = commandResult.getStatus(); 812 if (commandStatus != CommandStatus.SUCCESS 813 && commandStatus != CommandStatus.TIMED_OUT) { 814 CLog.e("Python process failed"); 815 CLog.e("Python path: %s", mPythonPath); 816 CLog.e("Stderr: %s", commandResult.getStderr()); 817 CLog.e("Stdout: %s", commandResult.getStdout()); 818 printVtsLogs(vtsRunnerLogDir); 819 printToDeviceLogcatAboutTestModuleStatus("ERROR"); 820 throw new RuntimeException("Failed to run VTS test"); 821 } 822 CLog.i("Standard output is: %s", commandResult.getStdout()); 823 CLog.i("Parsing test result: %s", commandResult.getStderr()); 824 printToDeviceLogcatAboutTestModuleStatus("END"); 825 } else { 826 printToDeviceLogcatAboutTestModuleStatus("FRAMEWORK_ERROR"); 827 } 828 829 VtsMultiDeviceTestResultParser parser = 830 new VtsMultiDeviceTestResultParser(listener, mRunName); 831 832 if (mUseStdoutLogs) { 833 if (commandResult.getStdout() == null) { 834 CLog.e("The std:out is null for CommandResult."); 835 throw new RuntimeException("The std:out is null for CommandResult."); 836 } 837 parser.processNewLines(commandResult.getStdout().split("\n")); 838 } else { 839 // parse from test_run_summary.json instead of std:out 840 String jsonData = null; 841 JSONObject object = null; 842 File testRunSummary = getFileTestRunSummary(vtsRunnerLogDir); 843 if (testRunSummary == null) { 844 throw new RuntimeException("Couldn't locate the file : " + 845 TEST_RUN_SUMMARY_FILE_NAME); 846 } 847 try { 848 jsonData = FileUtil.readStringFromFile(testRunSummary); 849 CLog.i("Test Result Summary: %s", jsonData); 850 object = new JSONObject(jsonData); 851 } catch (IOException e) { 852 CLog.e("Error occurred in parsing Json file : %s", testRunSummary.toPath()); 853 } catch (JSONException e) { 854 CLog.e("Error occurred in parsing Json String : %s", jsonData); 855 } 856 if (object == null) { 857 CLog.e("Json object is null."); 858 throw new RuntimeException("Json object is null."); 859 } 860 parser.processJsonFile(object); 861 } 862 printVtsLogs(vtsRunnerLogDir); 863 FileUtil.recursiveDelete(vtsRunnerLogDir); 864 CLog.i("Deleted the runner log dir, %s.", vtsRunnerLogDir); 865 if (jsonFilePath != null) { 866 FileUtil.deleteFile(new File(jsonFilePath)); 867 CLog.i("Deleted the runner json config file, %s.", jsonFilePath); 868 } 869 } 870 871 /** 872 * This method return the file test_run_details.txt which is then used to parse logs. 873 * 874 * @param logDir : The file that needs to be converted 875 * @return the file named test_run_details.txt 876 * @throws IllegalArgumentException 877 */ getFileTestRunSummary(File logDir)878 private File getFileTestRunSummary(File logDir) throws IllegalArgumentException { 879 File[] children; 880 if (logDir == null) { 881 throw new IllegalArgumentException("Argument logDir is null."); 882 } 883 children = logDir.listFiles(); 884 if (children != null) { 885 for (File child : children) { 886 if (!child.isDirectory()) { 887 if (child.getName().equals(TEST_RUN_SUMMARY_FILE_NAME)) { 888 return child; 889 } 890 } else { 891 File file = getFileTestRunSummary(child); 892 if (file != null) { 893 return file; 894 } 895 } 896 } 897 } 898 return null; 899 } 900 901 /** 902 * The method prints all VTS runner log files 903 * 904 * @param logDir the File instance of the base log dir. 905 */ printVtsLogs(File logDir)906 private void printVtsLogs(File logDir) { 907 File[] children; 908 if (logDir == null) { 909 CLog.e("Scan VTS log dir: null\n"); 910 return; 911 } 912 CLog.i("Scan VTS log dir %s\n", logDir.getAbsolutePath()); 913 children = logDir.listFiles(); 914 if (children != null) { 915 for (File child : children) { 916 if (child.isDirectory()) { 917 printVtsLogs(child); 918 } else { 919 CLog.i("VTS log file %s\n", child.getAbsolutePath()); 920 try { 921 if (child.getName().startsWith("vts_agent") && 922 child.getName().endsWith(".log")) { 923 CLog.i("Content: %s\n", FileUtil.readStringFromFile(child)); 924 } else { 925 CLog.i("skip %s\n", child.getName()); 926 } 927 } catch (IOException e) { 928 CLog.e("I/O error\n"); 929 } 930 } 931 } 932 } 933 } 934 935 /** 936 * This method returns whether the OS is Windows. 937 */ isOnWindows()938 private static boolean isOnWindows() { 939 return System.getProperty(OS_NAME).contains(WINDOWS); 940 } 941 942 /** 943 * This method sets the python path. It's based on the 944 * assumption that the environment variable $ANDROID_BUILD_TOP is set. 945 */ setPythonPath()946 private void setPythonPath() { 947 StringBuilder sb = new StringBuilder(); 948 String separator = File.pathSeparator; 949 if (System.getenv(PYTHONPATH) != null) { 950 sb.append(separator); 951 sb.append(System.getenv(PYTHONPATH)); 952 } 953 954 // to get the path for android-vts/testcases/ which keeps the VTS python code under vts. 955 if (mBuildInfo != null) { 956 CompatibilityBuildHelper mBuildHelper = new CompatibilityBuildHelper(mBuildInfo); 957 958 File testDir = null; 959 try { 960 testDir = mBuildHelper.getTestsDir(); 961 } catch (FileNotFoundException e) { 962 /* pass */ 963 } 964 if (testDir != null) { 965 sb.append(separator); 966 mTestCaseDataDir = testDir.getAbsolutePath(); 967 sb.append(mTestCaseDataDir); 968 } else if (mBuildInfo.getFile(VTS) != null) { 969 sb.append(separator); 970 sb.append(mBuildInfo.getFile(VTS).getAbsolutePath()).append("/.."); 971 } 972 } 973 974 // for when one uses PythonVirtualenvPreparer. 975 if (mBuildInfo.getFile(PYTHONPATH) != null) { 976 sb.append(separator); 977 sb.append(mBuildInfo.getFile(PYTHONPATH).getAbsolutePath()); 978 } 979 if (System.getenv("ANDROID_BUILD_TOP") != null) { 980 sb.append(separator); 981 sb.append(System.getenv("ANDROID_BUILD_TOP")).append("/test"); 982 } 983 if (sb.length() == 0) { 984 throw new RuntimeException("Could not find python path on host machine"); 985 } 986 mPythonPath = sb.substring(1); 987 CLog.i("mPythonPath: %s", mPythonPath); 988 } 989 990 /** 991 * This method gets the python binary. 992 */ getPythonBinary()993 private String getPythonBinary() { 994 boolean isWindows = isOnWindows(); 995 String python = (isWindows? "python.exe": "python"); 996 File venvDir = mBuildInfo.getFile(VIRTUAL_ENV_PATH); 997 if (venvDir != null) { 998 String binDir = (isWindows? "Scripts": "bin"); 999 File pythonBinaryFile = new File(venvDir.getAbsolutePath(), 1000 binDir + File.separator + python); 1001 String pythonBinPath = pythonBinaryFile.getAbsolutePath(); 1002 if (pythonBinaryFile.exists()) { 1003 CLog.i("Python path " + pythonBinPath + ".\n"); 1004 return pythonBinPath; 1005 } 1006 CLog.e(python + " doesn't exist under the " + 1007 "created virtualenv dir (" + pythonBinPath + ").\n"); 1008 } else { 1009 CLog.e(VIRTUAL_ENV_PATH + " not available in BuildInfo. " + 1010 "Please use VtsPythonVirtualenvPreparer tartget preparer.\n"); 1011 } 1012 1013 IRunUtil runUtil = (mRunUtil == null ? RunUtil.getDefault() : mRunUtil); 1014 CommandResult c = runUtil.runTimedCmd(1000, 1015 (isWindows ? "where" : "which"), python); 1016 String pythonBin = c.getStdout().trim(); 1017 if (pythonBin.length() == 0) { 1018 throw new RuntimeException("Could not find python binary on host " 1019 + "machine"); 1020 } 1021 return pythonBin; 1022 } 1023 1024 /** 1025 * {@inheritDoc} 1026 */ 1027 @Override setAbi(IAbi abi)1028 public void setAbi(IAbi abi){ 1029 mAbi = abi; 1030 } 1031 } 1032