1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.annotations.VisibleForTesting; 20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.result.ITestInvocationListener; 30 import com.android.tradefed.result.ITestLifeCycleReceiver; 31 import com.android.tradefed.result.TestDescription; 32 import com.android.tradefed.targetprep.VtsCoveragePreparer; 33 import com.android.tradefed.targetprep.VtsPythonVirtualenvPreparer; 34 import com.android.tradefed.util.CommandResult; 35 import com.android.tradefed.util.CommandStatus; 36 import com.android.tradefed.util.FileUtil; 37 import com.android.tradefed.util.JsonUtil; 38 import com.android.tradefed.util.OutputUtil; 39 import com.android.tradefed.util.VtsDashboardUtil; 40 import com.android.tradefed.util.VtsPythonRunnerHelper; 41 import com.android.tradefed.util.VtsVendorConfigFileUtil; 42 43 import org.json.JSONArray; 44 import org.json.JSONException; 45 import org.json.JSONObject; 46 47 import java.io.BufferedWriter; 48 import java.io.File; 49 import java.io.FileNotFoundException; 50 import java.io.FileWriter; 51 import java.io.IOException; 52 import java.io.PrintWriter; 53 import java.nio.file.Paths; 54 import java.util.ArrayList; 55 import java.util.Arrays; 56 import java.util.Collection; 57 import java.util.HashMap; 58 import java.util.List; 59 import java.util.Set; 60 import java.util.TreeMap; 61 import java.util.TreeSet; 62 63 /** 64 * A Test that runs a vts multi device test package (part of Vendor Test Suite, VTS) on given 65 * device.<p> 66 * TODO: Complete unit tests 67 */ 68 @OptionClass(alias = "vtsmultidevicetest") 69 public class VtsMultiDeviceTest 70 implements IDeviceTest, IRemoteTest, ITestFilterReceiver, IRuntimeHintProvider, 71 ITestCollector, IBuildReceiver, IAbiReceiver, IInvocationContextReceiver { 72 static final String ACTS_TEST_MODULE = "ACTS_TEST_MODULE"; 73 static final String ADAPTER_ACTS_PATH = "vts/runners/adapters/acts/acts_adapter"; 74 static final String ANDROIDDEVICE = "AndroidDevice"; 75 static final String BUILD = "build"; 76 static final String BUILD_ID = "build_id"; 77 static final String BUILD_TARGET = "build_target"; 78 static final String COVERAGE_PROPERTY = "ro.vts.coverage"; 79 static final String DATA_FILE_PATH = "data_file_path"; 80 static final String LOG_PATH = "log_path"; 81 static final String LOG_SEVERITY = "log_severity"; 82 static final String NAME = "name"; 83 static final String SERIAL = "serial"; 84 static final String TESTMODULE = "TestModule"; 85 static final String TEST_BED = "test_bed"; 86 static final String TEST_PLAN_REPORT_FILE = "TEST_PLAN_REPORT_FILE"; 87 static final String TEST_SUITE = "test_suite"; 88 static final String TEST_TIMEOUT = "test_timeout"; 89 static final String ABI_NAME = "abi_name"; 90 static final String ABI_BITNESS = "abi_bitness"; 91 static final String SKIP_ON_32BIT_ABI = "skip_on_32bit_abi"; 92 static final String SKIP_ON_64BIT_ABI = "skip_on_64bit_abi"; 93 static final String SHELL_DEFAULT_NOHUP = "shell_default_nohup"; 94 static final String SKIP_IF_THERMAL_THROTTLING = "skip_if_thermal_throttling"; 95 static final String DISABLE_CPU_FREQUENCY_SCALING = "disable_cpu_frequency_scaling"; 96 static final String DISABLE_FRAMEWORK = "DISABLE_FRAMEWORK"; 97 static final String STOP_NATIVE_SERVERS = "STOP_NATIVE_SERVERS"; 98 static final String RUN_32BIT_ON_64BIT_ABI = "run_32bit_on_64bit_abi"; 99 static final String CONFIG_FILE_EXTENSION = ".config"; 100 static final String INCLUDE_FILTER = "include_filter"; 101 static final String EXCLUDE_FILTER = "exclude_filter"; 102 static final String EXCLUDE_OVER_INCLUDE = "exclude_over_include"; 103 static final String BINARY_TEST_SOURCE = "binary_test_source"; 104 static final String BINARY_TEST_WORKING_DIRECTORY = "binary_test_working_directory"; 105 static final String BINARY_TEST_ENVP = "binary_test_envp"; 106 static final String BINARY_TEST_ARGS = "binary_test_args"; 107 static final String BINARY_TEST_LD_LIBRARY_PATH = "binary_test_ld_library_path"; 108 static final String BINARY_TEST_PROFILING_LIBRARY_PATH = "binary_test_profiling_library_path"; 109 @Deprecated static final String BINARY_TEST_DISABLE_FRAMEWORK = "binary_test_disable_framework"; 110 @Deprecated 111 static final String BINARY_TEST_STOP_NATIVE_SERVERS = "binary_test_stop_native_servers"; 112 static final String BINARY_TEST_TYPE_GTEST = "gtest"; 113 static final String BINARY_TEST_TYPE_LLVMFUZZER = "llvmfuzzer"; 114 static final String BINARY_TEST_TYPE_HAL_HIDL_GTEST = "hal_hidl_gtest"; 115 static final String BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST = "hal_hidl_replay_test"; 116 static final String BINARY_TEST_TYPE_HOST_BINARY_TEST = "host_binary_test"; 117 static final String BUG_REPORT_ON_FAILURE = "BUG_REPORT_ON_FAILURE"; 118 static final String COLLECT_TESTS_ONLY = "collect_tests_only"; 119 static final String CONFIG_STR = "CONFIG_STR"; 120 static final String CONFIG_INT = "CONFIG_INT"; 121 static final String CONFIG_BOOL = "CONFIG_BOOL"; 122 static final String LOGCAT_ON_FAILURE = "LOGCAT_ON_FAILURE"; 123 static final String ENABLE_COVERAGE = "enable_coverage"; 124 static final String EXCLUDE_COVERAGE_PATH = "exclude_coverage_path"; 125 static final String ENABLE_LOG_UPLOADING = "enable_log_uploading"; 126 static final String ENABLE_PROFILING = "enable_profiling"; 127 static final String PROFILING_ARG_VALUE = "profiling_arg_value"; 128 static final String ENABLE_SANCOV = "enable_sancov"; 129 static final String GTEST_BATCH_MODE = "gtest_batch_mode"; 130 static final String SAVE_TRACE_FIEL_REMOTE = "save_trace_file_remote"; 131 static final String OUTPUT_COVERAGE_REPORT = "output_coverage_report"; 132 static final String COVERAGE_REPORT_PATH = "coverage_report_path"; 133 static final String GLOBAL_COVERAGE = "global_coverage"; 134 static final String LTP_NUMBER_OF_THREADS = "ltp_number_of_threads"; 135 static final String MAX_RETRY_COUNT = "max_retry_count"; 136 static final String MOBLY_TEST_MODULE = "MOBLY_TEST_MODULE"; 137 static final String NATIVE_SERVER_PROCESS_NAME = "native_server_process_name"; 138 static final String PASSTHROUGH_MODE = "passthrough_mode"; 139 static final String PRECONDITION_HWBINDER_SERVICE = "precondition_hwbinder_service"; 140 static final String PRECONDITION_FEATURE = "precondition_feature"; 141 static final String PRECONDITION_FILE_PATH_PREFIX = "precondition_file_path_prefix"; 142 static final String PRECONDITION_FIRST_API_LEVEL = "precondition_first_api_level"; 143 static final String PRECONDITION_LSHAL = "precondition_lshal"; 144 static final String PRECONDITION_SYSPROP = "precondition_sysprop"; 145 static final String PRECONDITION_VINTF = "precondition_vintf"; 146 static final String ENABLE_SYSTRACE = "enable_systrace"; 147 static final String HAL_HIDL_REPLAY_TEST_TRACE_PATHS = "hal_hidl_replay_test_trace_paths"; 148 static final String HAL_HIDL_PACKAGE_NAME = "hal_hidl_package_name"; 149 static final String REPORT_MESSAGE_FILE_NAME = "report_proto.msg"; 150 static final String RUN_AS_VTS_SELF_TEST = "run_as_vts_self_test"; 151 static final String RUN_AS_COMPLIANCE_TEST = "run_as_compliance_test"; 152 static final String SYSTRACE_PROCESS_NAME = "systrace_process_name"; 153 static final String TEMPLATE_BINARY_TEST_PATH = "vts/testcases/template/binary_test/binary_test"; 154 static final String TEMPLATE_GTEST_BINARY_TEST_PATH = "vts/testcases/template/gtest_binary_test/gtest_binary_test"; 155 static final String TEMPLATE_LLVMFUZZER_TEST_PATH = "vts/testcases/template/llvmfuzzer_test/llvmfuzzer_test"; 156 static final String TEMPLATE_MOBLY_TEST_PATH = "vts/testcases/template/mobly/mobly_test"; 157 static final String TEMPLATE_HAL_HIDL_GTEST_PATH = "vts/testcases/template/hal_hidl_gtest/hal_hidl_gtest"; 158 static final String TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH = "vts/testcases/template/hal_hidl_replay_test/hal_hidl_replay_test"; 159 static final String TEMPLATE_HOST_BINARY_TEST_PATH = "vts/testcases/template/host_binary_test/host_binary_test"; 160 static final String TEST_RUN_SUMMARY_FILE_NAME = "test_run_summary.json"; 161 static final float DEFAULT_TARGET_VERSION = -1; 162 static final String DEFAULT_TESTCASE_CONFIG_PATH = 163 "vts/tools/vts-tradefed/res/default/DefaultTestCase.runner_conf"; 164 165 private ITestDevice mDevice = null; 166 private IAbi mAbi = null; 167 168 @Option(name = "test-timeout", 169 description = "The amount of time (in milliseconds) for a test invocation. " 170 + "If the test cannot finish before timeout, it is interrupted. As some " 171 + "classes generate test cases during setup, they can use the given timeout " 172 + "value for each generated test set.", 173 isTimeVal = true) 174 private long mTestTimeout = 1000 * 60 * 3; 175 176 @Option(name = "max-test-timeout", 177 description = "The maximum amount of time (in milliseconds) for a test invocation. " 178 + "This timeout value doesn't change with number of generated test sets.", 179 isTimeVal = true) 180 private long mMaxTestTimeout = 1000 * 60 * 60 * 3; 181 182 @Option(name = "test-module-name", 183 description = "The name for a test module.") 184 private String mTestModuleName = null; 185 186 @Option(name = "test-case-path", 187 description = "The path for test case.") 188 private String mTestCasePath = null; 189 190 @Option(name = "test-case-path-type", 191 description = "The type of test case path ('module' by default or 'file').") 192 private String mTestCasePathType = null; 193 194 @Option(name = "test-config-path", 195 description = "The path for test case config file.") 196 private String mTestConfigPath = null; 197 198 @Option(name = "precondition-hwbinder-service", 199 description = "The name of a HW binder service needed to run the test.") 200 private String mPreconditionHwBinderServiceName = null; 201 202 @Option(name = "precondition-feature", 203 description = "The name of a `pm`-listable feature needed to run the test.") 204 private String mPreconditionFeature = null; 205 206 @Option(name = "precondition-file-path-prefix", 207 description = "The path prefix of a target-side file needed to run the test." 208 + "Format of tags:" 209 + " <source>: source without tag." 210 + " <tag>::<source>: <tag> specifies bitness of testcase: _32bit or _64bit" 211 + " Note: multiple sources are ANDed" 212 + "Format of each source string:" 213 + " <source>: absolute path of file prefix on device") 214 private Collection<String> mPreconditionFilePathPrefix = new ArrayList<>(); 215 216 @Option(name = "precondition-first-api-level", 217 description = "The lowest first API level required to run the test.") 218 private int mPreconditionFirstApiLevel = 0; 219 220 @Option(name = "precondition-lshal", 221 description = "The name of a `lshal`-listable feature needed to run the test.") 222 private String mPreconditionLshal = null; 223 224 @Option(name = "precondition-sysprop", 225 description = "The name=value for a system property configuration that needs " 226 + "to be met to run the test.") 227 private String mPreconditionSysProp = null; 228 229 @Option(name = "precondition-vintf", 230 description = "The full name of a HAL specified in vendor/manifest.xml and " 231 + "needed to run the test (e.g., android.hardware.graphics.mapper@2.0). " 232 + "this can override precondition-lshal option.") 233 private String mPreconditionVintf = null; 234 235 @Option(name = "use-stdout-logs", 236 description = "Flag that determines whether to use std:out to parse output.") 237 private boolean mUseStdoutLogs = false; 238 239 @Option(name = "enable-dashboard-uploading", 240 description = "Enables the runner's dashboard result uploading feature.") 241 private boolean mEnableDashboardUploading = true; 242 243 @Option(name = "enable-log-uploading", 244 description = "Enables the runner's log uploading feature.") 245 private boolean mEnableLogUploading = false; 246 247 @Option(name = "include-filter", 248 description = "The positive filter of the test names to run.") 249 private Set<String> mIncludeFilters = new TreeSet<>(); 250 251 @Option(name = "exclude-filter", 252 description = "The negative filter of the test names to run.") 253 private Set<String> mExcludeFilters = new TreeSet<>(); 254 255 @Option(name = "exclude-over-include", 256 description = "The negative filter of the test names to run.") 257 private boolean mExcludeOverInclude = false; 258 259 @Option(name = "runtime-hint", description = "The hint about the test's runtime.", 260 isTimeVal = true) 261 private long mRuntimeHint = 60000; // 1 minute 262 263 @Option(name = "enable-profiling", description = "Enable profiling for the tests.") 264 private boolean mEnableProfiling = false; 265 266 @Option(name = "profiling-arg-value", description = "Whether to profile for arg value.") 267 private boolean mProfilingArgValue = false; 268 269 @Option(name = "save-trace-file-remote", 270 description = "Whether to save the trace file in remote storage.") 271 private boolean mSaveTraceFileRemote = false; 272 273 @Option(name = "enable-systrace", description = "Enable systrace for the tests.") 274 private boolean mEnableSystrace = false; 275 276 @Option(name = "enable-coverage", 277 description = "Enable coverage for the tests. In order for coverage to be measured, " + 278 "ro.vts.coverage system must have value \"1\" to indicate the target " + 279 "build is coverage instrumented.") 280 private boolean mEnableCoverage = true; 281 282 @Option(name = "global-coverage", description = "True to measure coverage for entire test, " 283 + "measure coverage for each test case otherwise. Currently, only global " 284 + "coverage is supported for binary tests") 285 private boolean mGlobalCoverage = true; 286 287 @Option(name = "enable-sancov", 288 description = "Enable sanitizer coverage for the tests. In order for coverage to be " 289 + "measured, the device must be a sancov build with its build info and " 290 + "unstripped binaries available to the sancov preparer class.") 291 private boolean mEnableSancov = true; 292 293 @Option(name = "output-coverage-report", description = "Whether to store raw coverage report.") 294 private boolean mOutputCoverageReport = false; 295 296 // Another design option is to parse a string or use enum for host preference on BINDER, 297 // PASSTHROUGH and DEFAULT(which is BINDER). Also in the future, we might want to deal with 298 // the case of target preference on PASSTHROUGH (if host does not specify to use BINDER mode). 299 @Option(name = "passthrough-mode", description = "Set getStub to use passthrough mode. " 300 + "Value true means use passthrough mode if available; false for binderized mode if " 301 + "available. Default is false") 302 private boolean mPassthroughMode = false; 303 304 @Option(name = "ltp-number-of-threads", 305 description = "Number of threads to run the LTP test cases. " 306 + "0 means using number of avaiable CPU threads.") 307 private int mLtpNumberOfThreads = -1; 308 309 @Option(name = "shell-default-nohup", 310 description = "Whether to by default use nohup for shell commands.") 311 private boolean mShellDefaultNohup = false; 312 313 @Option(name = "skip-on-32bit-abi", description = "Whether to skip tests on 32bit ABI.") 314 private boolean mSkipOn32BitAbi = false; 315 316 @Option(name = "skip-on-64bit-abi", description = "Whether to skip tests on 64bit ABI.") 317 private boolean mSkipOn64BitAbi = false; 318 319 @Option(name = "skip-if-thermal-throttling", 320 description = "Whether to skip tests if target device suffers from thermal throttling.") 321 private boolean mSkipIfThermalThrottling = false; 322 323 @Option(name = "disable-cpu-frequency-scaling", 324 description = "Whether to disable cpu frequency scaling for test.") 325 private boolean mDisableCpuFrequencyScaling = true; 326 327 @Option(name = "run-32bit-on-64bit-abi", 328 description = "Whether to run 32bit tests on 64bit ABI.") 329 private boolean mRun32bBitOn64BitAbi = false; 330 331 @Option(name = "binary-test-source", 332 description = "Binary test source paths relative to vts testcase directory on host." 333 + "Format of tags:" 334 + " <source>: source without tag." 335 + " <tag>::<source>: source with tag. Can be used to separate 32bit and 64" 336 + " bit tests with same file name." 337 + " <tag1>::<source1>, <tag1>::<source2>, <tag2>::<source3>: multiple" 338 + " sources connected by comma. White spaces in-between" 339 + " will be ignored." 340 + "Format of each source string:" 341 + " <source file>: push file and create test case." 342 + " Source is relative path of source file under vts's testcases" 343 + " folder. Source file will be pushed to working directory" 344 + " discarding original directory structure, and test cases will" 345 + " be created using the pushed file." 346 + " <source file>-><destination file>: push file and create test case." 347 + " Destination path is absolute path on device. Test cases will" 348 + " be created using the pushed file." 349 + " <source file>->: push file only." 350 + " Push the source file to its' tag's corresponding" 351 + " working directory. Test case will not be created on" 352 + " this file. This is equivalent to but simpler than specifying a" 353 + " working directory for the tag and use VtsFilePusher to push the" 354 + " file to the directory." 355 + " -><destination file>: create test case only." 356 + " Destination is absolute device side path." 357 + " Note: each path in source string can be a directory. However, the" 358 + " default binary test runner and gtest binary test runner does not" 359 + " support creating test cases from a directory. You will need to" 360 + " override the binary test runner's CreateTestCase method in python." 361 + " If you wish to push a source file to a specific destination and not" 362 + " create a test case from it, please use VtsFilePusher.") 363 private Collection<String> mBinaryTestSource = new ArrayList<>(); 364 365 @Option(name = "binary-test-working-directory", description = "Working directories for binary " 366 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 367 + "However, each tag should only has one working directory. This option is optional for " 368 + "binary tests. If not specified, different directories will be used for files with " 369 + "different tags.") 370 private Collection<String> mBinaryTestWorkingDirectory = new ArrayList<>(); 371 372 @Option(name = "binary-test-envp", description = "Additional environment path for binary " 373 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 374 + "There can be multiple instances of binary-test-envp for a same tag, which will " 375 + "later automatically be combined.") 376 private Collection<String> mBinaryTestEnvp = new ArrayList<>(); 377 378 @Option(name = "binary-test-args", description = "Additional args or flags for binary " 379 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 380 + "There can be multiple instances of binary-test-args for a same tag, which will " 381 + "later automatically be combined.") 382 private Collection<String> mBinaryTestArgs = new ArrayList<>(); 383 384 @Option(name = "binary-test-ld-library-path", description = "LD_LIBRARY_PATH for binary " 385 + "tests. Tags can be added to the front of each instance using '::' as delimiter. " 386 + "Multiple directories can be added under a same tag using ':' as delimiter. " 387 + "There can be multiple instances of ld-library-path for a same tag, which will " 388 + "later automatically be combined using ':' as delimiter. Paths without a tag " 389 + "will only used for binaries without tag. This option is optional for binary tests.") 390 private Collection<String> mBinaryTestLdLibraryPath = new ArrayList<>(); 391 392 @Option(name = "binary-test-profiling-library-path", description = "Path to lookup and load " 393 + "profiling libraries for tests with profiling enabled. Tags can be added to the " 394 + "front of each directory using '::' as delimiter. Only one directory could be " 395 + "specified for the same tag. This option is optional for binary tests. If not " 396 + "specified, default directories will be used for files with different tags.") 397 private Collection<String> mBinaryTestProfilingLibraryPath = new ArrayList<>(); 398 399 @Deprecated 400 @Option(name = "binary-test-disable-framework", 401 description = "Adb stop/start before/after test.") 402 private boolean mBinaryTestDisableFramework = false; 403 404 @Deprecated 405 @Option(name = "binary-test-stop-native-servers", 406 description = "Set to stop all properly configured native servers during the testing.") 407 private boolean mBinaryTestStopNativeServers = false; 408 409 @Option(name = "disable-framework", description = "Adb stop/start before/after test.") 410 private boolean mDisableFramework = false; 411 412 @Option(name = "stop-native-servers", 413 description = "Set to stop all properly configured native servers during the testing.") 414 private boolean mStopNativeServers = false; 415 416 @Option(name = "bug-report-on-failure", 417 description = "To catch bugreport zip file at the end of failed test cases. " 418 + "If set to true, a report will be caught through adh shell command at the " 419 + "end of each failed test cases.") 420 private boolean mBugReportOnFailure = false; 421 422 @Option(name = "logcat-on-failure", 423 description = "To catch logcat from each buffers at the end of failed test cases. " 424 + "If set to true, a report will be caught through adh shell command at the " 425 + "end of each failed test cases.") 426 private boolean mLogcatOnFailure = true; 427 428 @Option(name = "native-server-process-name", 429 description = "Name of a native server process. The runner checks to make sure " 430 + "each specified native server process is not running after the framework stop.") 431 private Collection<String> mNativeServerProcessName = new ArrayList<>(); 432 433 @Option(name = "binary-test-type", description = "Binary test type. Only specify this when " 434 + "running an extended binary test without a python test file. Available options: gtest") 435 private String mBinaryTestType = ""; 436 437 @Option(name = "hal-hidl-replay-test-trace-path", description = "The path of a trace file to replay.") 438 private Collection<String> mHalHidlReplayTestTracePaths = new ArrayList<>(); 439 440 @Option(name = "hal-hidl-package-name", description = "The name of a target HIDL HAL package " 441 + "e.g., 'android.hardware.light@2.0'.") 442 private String mHalHidlPackageName = null; 443 444 @Option(name = "systrace-process-name", description = "Process name for systrace.") 445 private String mSystraceProcessName = null; 446 447 @Option(name = "collect-tests-only", 448 description = "Only invoke setUpClass, generate*, and tearDownClass to collect list " 449 + "of applicable test cases. All collected tests pass without being executed.") 450 private boolean mCollectTestsOnly = false; 451 452 @Option(name = "gtest-batch-mode", description = "Run Gtest binaries in batch mode.") 453 private boolean mGtestBatchMode = false; 454 455 @Option(name = "log-severity", 456 description = "Set the log severity level." 457 + "Note, this is a legacy option and does not affect how log files are saved." 458 + "By setting it to INFO, it will only make python DEBUG log not showing on " 459 + "console even if TradeFed log display level is set to DEBUG." 460 + "Therefore, it is not recommemded to set or modify this value in the current" 461 + "implementation.") 462 private String mLogSeverity = "DEBUG"; 463 464 @Option(name = "run-as-vts-self-test", 465 description = "Run the module as vts-selftest. " 466 + "When the value is set to true, only setUpClass and tearDownClass function " 467 + "of the module will be called to ensure the framework is free of bug. " 468 + "Note that exception in tearDownClass will not be reported as failure.") 469 private boolean mRunAsVtsSelfTest = false; 470 471 @Option(name = "exclude-coverage-path", 472 description = "The coverage path that should be excluded in results. " 473 + "Used only when enable-coverage is true.") 474 private Collection<String> mExcludeCoveragePath = new ArrayList<>(); 475 476 @Option(name = "mobly-test-module", 477 description = "Mobly test module name. " 478 + "If this value is specified, VTS will use mobly test template " 479 + "with the configurations." 480 + "Multiple values can be added by repeatly using this option.") 481 private Collection<String> mMoblyTestModule = new ArrayList<>(); 482 483 @Option(name = "acts-test-module", 484 description = "Acts test module name. " 485 + "If this value is specified, VTS will use acts test adapter " 486 + "with the configurations." 487 + "Multiple values can be added by repeatly using this option.") 488 private String mActsTestModule = null; 489 490 @Option(name = "config-str", 491 description = "Key-value map of custom config string. " 492 + "The map will be passed directly to python runner and test module. " 493 + "Only one value per key is stored." 494 + "If the value for the same key is set multiple times, only the last value is " 495 + "used.") 496 private TreeMap<String, String> mConfigStr = new TreeMap<>(); 497 498 @Option(name = "config-int", 499 description = "Key-value map of custom config integer. " 500 + "The map will be passed directly to python runner and test module. " 501 + "Only one value per key is stored." 502 + "If the value for the same key is set multiple times, only the last value is " 503 + "used.") 504 private TreeMap<String, Integer> mConfigInt = new TreeMap<>(); 505 506 @Option(name = "config-bool", 507 description = "Key-value map of custom config boolean. " 508 + "The map will be passed directly to python runner and test module. " 509 + "Only one value per key is stored." 510 + "If the value for the same key is set multiple times, only the last value is " 511 + "used.") 512 private TreeMap<String, Boolean> mConfigBool = new TreeMap<>(); 513 514 @Option(name = "max-retry-count", 515 description = "The max number of retries. Currerntly done by VTS Python runner in " 516 + "a test case granularity.") 517 private int mMaxRetryCount = 0; 518 519 private IBuildInfo mBuildInfo = null; 520 private String mRunName = null; 521 // the path to android-vts/testcases 522 private String mTestCaseDir = "./"; 523 524 private VtsVendorConfigFileUtil configReader = null; 525 private IInvocationContext mInvocationContext = null; 526 private OutputUtil mOutputUtil = null; 527 protected CompatibilityBuildHelper mBuildHelper = null; 528 529 /** 530 * {@inheritDoc} 531 */ 532 @Override setInvocationContext(IInvocationContext context)533 public void setInvocationContext(IInvocationContext context) { 534 mInvocationContext = context; 535 setDevice(context.getDevices().get(0)); 536 setBuild(context.getBuildInfos().get(0)); 537 } 538 539 /** 540 * @return the mInvocationContext 541 */ getInvocationContext()542 public IInvocationContext getInvocationContext() { 543 return mInvocationContext; 544 } 545 546 /** 547 * {@inheritDoc} 548 */ 549 @Override setDevice(ITestDevice device)550 public void setDevice(ITestDevice device) { 551 mDevice = device; 552 } 553 554 /** 555 * {@inheritDoc} 556 */ 557 @Override getDevice()558 public ITestDevice getDevice() { 559 return mDevice; 560 } 561 setTestCasePath(String path)562 public void setTestCasePath(String path){ 563 mTestCasePath = path; 564 } 565 setTestConfigPath(String path)566 public void setTestConfigPath(String path){ 567 mTestConfigPath = path; 568 } 569 570 /** 571 * {@inheritDoc} 572 */ 573 @Override addIncludeFilter(String filter)574 public void addIncludeFilter(String filter) { 575 mIncludeFilters.add(cleanFilter(filter)); 576 } 577 578 /** 579 * {@inheritDoc} 580 */ 581 @Override addAllIncludeFilters(Set<String> filters)582 public void addAllIncludeFilters(Set<String> filters) { 583 for (String filter : filters) { 584 mIncludeFilters.add(cleanFilter(filter)); 585 } 586 } 587 588 /** 589 * {@inheritDoc} 590 */ 591 @Override clearIncludeFilters()592 public void clearIncludeFilters() { 593 mIncludeFilters.clear(); 594 } 595 596 /** {@inheritDoc} */ 597 @Override getIncludeFilters()598 public Set<String> getIncludeFilters() { 599 return mIncludeFilters; 600 } 601 602 /** 603 * {@inheritDoc} 604 */ 605 @Override addExcludeFilter(String filter)606 public void addExcludeFilter(String filter) { 607 mExcludeFilters.add(cleanFilter(filter)); 608 } 609 610 /** 611 * {@inheritDoc} 612 */ 613 @Override addAllExcludeFilters(Set<String> filters)614 public void addAllExcludeFilters(Set<String> filters) { 615 for (String filter : filters) { 616 mExcludeFilters.add(cleanFilter(filter)); 617 } 618 } 619 620 /** {@inheritDoc} */ 621 @Override clearExcludeFilters()622 public void clearExcludeFilters() { 623 mExcludeFilters.clear(); 624 } 625 626 /** {@inheritDoc} */ 627 @Override getExcludeFilters()628 public Set<String> getExcludeFilters() { 629 return mExcludeFilters; 630 } 631 632 /** 633 * Conforms filters using a {@link TestDescription} format 634 * to be recognized by the GTest executable. 635 */ cleanFilter(String filter)636 private String cleanFilter(String filter) { 637 return filter.replace('#', '.'); 638 } 639 640 /** 641 * {@inheritDoc} 642 */ 643 @Override getRuntimeHint()644 public long getRuntimeHint() { 645 return mRuntimeHint; 646 } 647 648 /** 649 * {@inheritDoc} 650 */ 651 @Override setCollectTestsOnly(boolean shouldCollectTest)652 public void setCollectTestsOnly(boolean shouldCollectTest) { 653 mCollectTestsOnly = shouldCollectTest; 654 } 655 656 /** 657 * Generate a device json object from ITestDevice object. 658 * 659 * @param device device object 660 * @throws RuntimeException 661 * @throws JSONException 662 */ generateJsonDeviceItem(ITestDevice device)663 private JSONObject generateJsonDeviceItem(ITestDevice device) throws JSONException { 664 JSONObject deviceItemObject = new JSONObject(); 665 deviceItemObject.put(SERIAL, device.getSerialNumber()); 666 try { 667 deviceItemObject.put("product_type", device.getProductType()); 668 deviceItemObject.put("product_variant", device.getProductVariant()); 669 deviceItemObject.put("build_alias", device.getBuildAlias()); 670 deviceItemObject.put("build_id", device.getBuildId()); 671 deviceItemObject.put("build_flavor", device.getBuildFlavor()); 672 } catch (DeviceNotAvailableException e) { 673 CLog.e("Device %s not available.", device.getSerialNumber()); 674 throw new RuntimeException("Failed to get device information"); 675 } 676 return deviceItemObject; 677 } 678 679 /** 680 * {@inheritDoc} 681 */ 682 @SuppressWarnings("deprecation") 683 @Override run(ITestInvocationListener listener)684 public void run(ITestInvocationListener listener) 685 throws IllegalArgumentException, DeviceNotAvailableException { 686 mOutputUtil = new OutputUtil(listener); 687 mOutputUtil.setTestModuleName(mTestModuleName); 688 if (mAbi != null) { 689 mOutputUtil.setAbiName(mAbi.getName()); 690 } 691 692 if (mTestCasePath == null) { 693 if (!mBinaryTestSource.isEmpty()) { 694 String template; 695 switch (mBinaryTestType) { 696 case BINARY_TEST_TYPE_GTEST: 697 template = TEMPLATE_GTEST_BINARY_TEST_PATH; 698 break; 699 case BINARY_TEST_TYPE_HAL_HIDL_GTEST: 700 template = TEMPLATE_HAL_HIDL_GTEST_PATH; 701 break; 702 case BINARY_TEST_TYPE_HOST_BINARY_TEST: 703 template = TEMPLATE_HOST_BINARY_TEST_PATH; 704 break; 705 default: 706 template = TEMPLATE_BINARY_TEST_PATH; 707 } 708 CLog.d("Using default test case template at %s.", template); 709 setTestCasePath(template); 710 if (mEnableCoverage && !mGlobalCoverage) { 711 CLog.e("Only global coverage is supported for test type %s.", mBinaryTestType); 712 throw new RuntimeException("Failed to produce VTS runner test config"); 713 } 714 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST)) { 715 setTestCasePath(TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH); 716 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_LLVMFUZZER)) { 717 // Fuzz test don't need test-case-path. 718 setTestCasePath(TEMPLATE_LLVMFUZZER_TEST_PATH); 719 } else if (!mMoblyTestModule.isEmpty()) { 720 setTestCasePath(TEMPLATE_MOBLY_TEST_PATH); 721 } else if (mActsTestModule != null) { 722 setTestCasePath(ADAPTER_ACTS_PATH); 723 } else { 724 throw new IllegalArgumentException("test-case-path is not set."); 725 } 726 } 727 728 doRunTest(listener); 729 } 730 731 /** 732 * {@inheritDoc} 733 */ 734 @Override setBuild(IBuildInfo buildInfo)735 public void setBuild(IBuildInfo buildInfo) { 736 mBuildInfo = buildInfo; 737 mBuildHelper = new CompatibilityBuildHelper(mBuildInfo); 738 } 739 740 /** 741 * Populate a jsonObject with default fields. 742 * This method uses deepMergeJsonObjects method from JsonUtil to merge the default config file with the target 743 * config file. Field already defined in target config file will not be overwritten. Also, JSONArray will not be 744 * deep merged. 745 * 746 * @param jsonObject the target json object to populate 747 * @param testCaseDataDir data file path 748 * @throws IOException 749 * @throws JSONException 750 */ populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir)751 private void populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir) 752 throws IOException, JSONException { 753 CLog.d("Populating default fields to json object from %s", DEFAULT_TESTCASE_CONFIG_PATH); 754 String content = 755 FileUtil.readStringFromFile(new File(mTestCaseDir, DEFAULT_TESTCASE_CONFIG_PATH)); 756 JSONObject defaultJsonObject = new JSONObject(content); 757 758 JsonUtil.deepMergeJsonObjects(jsonObject, defaultJsonObject); 759 } 760 761 /** 762 * Derive mRunName from module name or test paths. 763 * 764 * @return the derived mRunName. 765 * @throws RuntimeException if mTestModuleName, mTestConfigPath, and mTestCasePath are null. 766 */ deriveRunName()767 private String deriveRunName() throws RuntimeException { 768 if (mRunName != null) { 769 return mRunName; 770 } 771 772 if (mTestModuleName != null) { 773 mRunName = mTestModuleName; 774 } else { 775 CLog.w("--test-module-name not set (not recommended); deriving automatically"); 776 if (mTestConfigPath != null) { 777 mRunName = new File(mTestConfigPath).getName(); 778 mRunName = mRunName.replace(CONFIG_FILE_EXTENSION, ""); 779 } else if (mTestCasePath != null) { 780 mRunName = new File(mTestCasePath).getName(); 781 } else { 782 throw new RuntimeException( 783 "Failed to derive test module name; use --test-module-name option"); 784 } 785 } 786 return mRunName; 787 } 788 789 /** 790 * This method reads the provided VTS runner test json config, adds or updates some of its 791 * fields (e.g., to add build info and device serial IDs), and returns the updated json object. 792 * This method calls populateDefaultJsonFields to populate the config JSONObject if the config file is missing 793 * or some required fields is missing from the JSONObject. If test name is not specified, this method 794 * will use the config file's file name file without extension as test name. If config file is missing, this method 795 * will use the test case's class name as test name. 796 * 797 * @param log_path the path of a directory to store the VTS runner logs. 798 * @return the updated JSONObject as the new test config. 799 */ updateVtsRunnerTestConfig(JSONObject jsonObject)800 protected void updateVtsRunnerTestConfig(JSONObject jsonObject) 801 throws IOException, JSONException { 802 configReader = new VtsVendorConfigFileUtil(); 803 if (configReader.LoadVendorConfig(mBuildInfo)) { 804 JSONObject vendorConfigJson = configReader.GetVendorConfigJson(); 805 if (vendorConfigJson != null) { 806 JsonUtil.deepMergeJsonObjects(jsonObject, vendorConfigJson); 807 } 808 } 809 810 CLog.d("Load original test config %s %s", mTestCaseDir, mTestConfigPath); 811 String content = null; 812 813 if (mTestConfigPath != null) { 814 content = FileUtil.readStringFromFile( 815 new File(Paths.get(mTestCaseDir, mTestConfigPath).toString())); 816 CLog.d("Loaded original test config %s", content); 817 if (content != null) { 818 JsonUtil.deepMergeJsonObjects(jsonObject, new JSONObject(content)); 819 } 820 } 821 822 populateDefaultJsonFields(jsonObject, mTestCaseDir); 823 CLog.d("Built a Json object using the loaded original test config"); 824 825 JSONArray deviceArray = new JSONArray(); 826 827 boolean coverageBuild = false; 828 boolean sancovBuild = false; 829 830 boolean first_device = true; 831 for (ITestDevice device : mInvocationContext.getDevices()) { 832 JSONObject deviceJson = generateJsonDeviceItem(device); 833 try { 834 String coverageProperty = device.getProperty(COVERAGE_PROPERTY); 835 boolean enable_coverage_for_device = 836 coverageProperty != null && coverageProperty.equals("1"); 837 if (first_device) { 838 coverageBuild = enable_coverage_for_device; 839 first_device = false; 840 } else { 841 if (coverageBuild && (!enable_coverage_for_device)) { 842 CLog.e("Device %s is not coverage build while others are.", 843 device.getSerialNumber()); 844 throw new RuntimeException("Device build not the same."); 845 } 846 } 847 } catch (DeviceNotAvailableException e) { 848 CLog.e("Device %s not available.", device.getSerialNumber()); 849 throw new RuntimeException("Failed to get device information"); 850 } 851 852 File sancovDir = 853 mBuildInfo.getFile(VtsCoveragePreparer.getSancovResourceDirKey(device)); 854 if (sancovDir != null) { 855 deviceJson.put("sancov_resources_path", sancovDir.getAbsolutePath()); 856 sancovBuild = true; 857 } 858 File gcovDir = mBuildInfo.getFile(VtsCoveragePreparer.getGcovResourceDirKey(device)); 859 if (gcovDir != null) { 860 deviceJson.put("gcov_resources_path", gcovDir.getAbsolutePath()); 861 coverageBuild = true; 862 } 863 deviceArray.put(deviceJson); 864 } 865 866 JSONArray testBedArray = (JSONArray) jsonObject.get(TEST_BED); 867 if (testBedArray.length() == 0) { 868 JSONObject testBedItemObject = new JSONObject(); 869 String testName = deriveRunName(); 870 CLog.d("Setting test module name as %s", testName); 871 testBedItemObject.put(NAME, testName); 872 testBedItemObject.put(ANDROIDDEVICE, deviceArray); 873 testBedArray.put(testBedItemObject); 874 } else if (testBedArray.length() == 1) { 875 JSONObject testBedItemObject = (JSONObject) testBedArray.get(0); 876 JSONArray androidDeviceArray = (JSONArray) testBedItemObject.get(ANDROIDDEVICE); 877 int length; 878 length = (androidDeviceArray.length() > deviceArray.length()) 879 ? androidDeviceArray.length() 880 : deviceArray.length(); 881 for (int index = 0; index < length; index++) { 882 if (index < androidDeviceArray.length()) { 883 if (index < deviceArray.length()) { 884 JsonUtil.deepMergeJsonObjects((JSONObject) androidDeviceArray.get(index), 885 (JSONObject) deviceArray.get(index)); 886 } 887 } else if (index < deviceArray.length()) { 888 androidDeviceArray.put(index, deviceArray.get(index)); 889 } 890 } 891 } else { 892 CLog.e("Multi-device not yet supported: %d devices requested", 893 testBedArray.length()); 894 throw new RuntimeException("Failed to produce VTS runner test config"); 895 } 896 jsonObject.put(DATA_FILE_PATH, mTestCaseDir); 897 CLog.d("Added %s = %s to the Json object", DATA_FILE_PATH, mTestCaseDir); 898 899 JSONObject build = new JSONObject(); 900 build.put(BUILD_ID, mBuildInfo.getBuildId()); 901 build.put(BUILD_TARGET, mBuildInfo.getBuildTargetName()); 902 jsonObject.put(BUILD, build); 903 CLog.d("Added %s to the Json object", BUILD); 904 905 JSONObject suite = new JSONObject(); 906 suite.put(NAME, mBuildInfo.getTestTag()); 907 suite.put(INCLUDE_FILTER, new JSONArray(mIncludeFilters)); 908 CLog.d("Added include filter to test suite: %s", mIncludeFilters); 909 suite.put(EXCLUDE_FILTER, new JSONArray(mExcludeFilters)); 910 CLog.d("Added exclude filter to test suite: %s", mExcludeFilters); 911 912 String coverageReportPath = mBuildInfo.getBuildAttributes().get("coverage_report_path"); 913 if (coverageReportPath != null) { 914 jsonObject.put(OUTPUT_COVERAGE_REPORT, true); 915 CLog.d("Added %s to the Json object", OUTPUT_COVERAGE_REPORT); 916 jsonObject.put(COVERAGE_REPORT_PATH, coverageReportPath); 917 CLog.d("Added %s to the Json object", COVERAGE_REPORT_PATH); 918 } 919 920 if (mExcludeOverInclude) { 921 jsonObject.put(EXCLUDE_OVER_INCLUDE, mExcludeOverInclude); 922 CLog.d("Added %s to the Json object", EXCLUDE_OVER_INCLUDE); 923 } 924 925 jsonObject.put(TEST_SUITE, suite); 926 CLog.d("Added %s to the Json object", TEST_SUITE); 927 928 jsonObject.put(TEST_TIMEOUT, mTestTimeout); 929 CLog.i("Added %s to the Json object: %d", TEST_TIMEOUT, mTestTimeout); 930 931 if (!mLogSeverity.isEmpty()) { 932 String logSeverity = mLogSeverity.toUpperCase(); 933 ArrayList<String> severityList = 934 new ArrayList<String>(Arrays.asList("ERROR", "WARNING", "INFO", "DEBUG")); 935 if (!severityList.contains(logSeverity)) { 936 CLog.w("Unsupported log severity %s, use default log_severity:INFO instead.", 937 logSeverity); 938 logSeverity = "INFO"; 939 } 940 jsonObject.put(LOG_SEVERITY, logSeverity); 941 CLog.d("Added %s to the Json object: %s", LOG_SEVERITY, logSeverity); 942 } 943 944 if (mShellDefaultNohup) { 945 jsonObject.put(SHELL_DEFAULT_NOHUP, mShellDefaultNohup); 946 CLog.d("Added %s to the Json object", SHELL_DEFAULT_NOHUP); 947 } 948 949 if (mAbi != null) { 950 jsonObject.put(ABI_NAME, mAbi.getName()); 951 CLog.d("Added %s to the Json object", ABI_NAME); 952 jsonObject.put(ABI_BITNESS, mAbi.getBitness()); 953 CLog.d("Added %s to the Json object", ABI_BITNESS); 954 } 955 956 if (mSkipOn32BitAbi) { 957 jsonObject.put(SKIP_ON_32BIT_ABI, mSkipOn32BitAbi); 958 CLog.d("Added %s to the Json object", SKIP_ON_32BIT_ABI); 959 } 960 961 if (mSkipOn64BitAbi) { 962 jsonObject.put(SKIP_ON_64BIT_ABI, mSkipOn64BitAbi); 963 CLog.d("Added %s to the Json object", SKIP_ON_64BIT_ABI); 964 } else if (mRun32bBitOn64BitAbi) { 965 jsonObject.put(RUN_32BIT_ON_64BIT_ABI, mRun32bBitOn64BitAbi); 966 CLog.d("Added %s to the Json object", RUN_32BIT_ON_64BIT_ABI); 967 } 968 969 if (mSkipIfThermalThrottling) { 970 jsonObject.put(SKIP_IF_THERMAL_THROTTLING, mSkipIfThermalThrottling); 971 CLog.d("Added %s to the Json object", SKIP_IF_THERMAL_THROTTLING); 972 } 973 974 jsonObject.put(DISABLE_CPU_FREQUENCY_SCALING, mDisableCpuFrequencyScaling); 975 CLog.d("Added %s to the Json object, value: %s", DISABLE_CPU_FREQUENCY_SCALING, 976 mDisableCpuFrequencyScaling); 977 978 if (!mBinaryTestSource.isEmpty()) { 979 jsonObject.put(BINARY_TEST_SOURCE, new JSONArray(mBinaryTestSource)); 980 CLog.d("Added %s to the Json object", BINARY_TEST_SOURCE); 981 } 982 983 if (!mBinaryTestWorkingDirectory.isEmpty()) { 984 jsonObject.put(BINARY_TEST_WORKING_DIRECTORY, 985 new JSONArray(mBinaryTestWorkingDirectory)); 986 CLog.d("Added %s to the Json object", BINARY_TEST_WORKING_DIRECTORY); 987 } 988 989 if (!mBinaryTestEnvp.isEmpty()) { 990 jsonObject.put(BINARY_TEST_ENVP, new JSONArray(mBinaryTestEnvp)); 991 CLog.d("Added %s to the Json object", BINARY_TEST_ENVP); 992 } 993 994 if (!mBinaryTestArgs.isEmpty()) { 995 jsonObject.put(BINARY_TEST_ARGS, new JSONArray(mBinaryTestArgs)); 996 CLog.d("Added %s to the Json object", BINARY_TEST_ARGS); 997 } 998 999 if (!mBinaryTestLdLibraryPath.isEmpty()) { 1000 jsonObject.put(BINARY_TEST_LD_LIBRARY_PATH, 1001 new JSONArray(mBinaryTestLdLibraryPath)); 1002 CLog.d("Added %s to the Json object", BINARY_TEST_LD_LIBRARY_PATH); 1003 } 1004 1005 if (mBugReportOnFailure) { 1006 jsonObject.put(BUG_REPORT_ON_FAILURE, mBugReportOnFailure); 1007 CLog.d("Added %s to the Json object", BUG_REPORT_ON_FAILURE); 1008 } 1009 1010 if (!mLogcatOnFailure) { 1011 jsonObject.put(LOGCAT_ON_FAILURE, mLogcatOnFailure); 1012 CLog.d("Added %s to the Json object", LOGCAT_ON_FAILURE); 1013 } 1014 1015 if (mEnableProfiling) { 1016 jsonObject.put(ENABLE_PROFILING, mEnableProfiling); 1017 CLog.d("Added %s to the Json object", ENABLE_PROFILING); 1018 } 1019 1020 if (mProfilingArgValue) { 1021 jsonObject.put(PROFILING_ARG_VALUE, mProfilingArgValue); 1022 CLog.d("Added %s to the Json object", PROFILING_ARG_VALUE); 1023 } 1024 1025 if (mSaveTraceFileRemote) { 1026 jsonObject.put(SAVE_TRACE_FIEL_REMOTE, mSaveTraceFileRemote); 1027 CLog.d("Added %s to the Json object", SAVE_TRACE_FIEL_REMOTE); 1028 } 1029 1030 if (mEnableSystrace) { 1031 jsonObject.put(ENABLE_SYSTRACE, mEnableSystrace); 1032 CLog.d("Added %s to the Json object", ENABLE_SYSTRACE); 1033 } 1034 1035 if (mEnableCoverage) { 1036 jsonObject.put(GLOBAL_COVERAGE, mGlobalCoverage); 1037 if (!mExcludeCoveragePath.isEmpty()) { 1038 jsonObject.put(EXCLUDE_COVERAGE_PATH, new JSONArray(mExcludeCoveragePath)); 1039 CLog.d("Added %s to the Json object", EXCLUDE_COVERAGE_PATH); 1040 } 1041 if (coverageBuild) { 1042 jsonObject.put(ENABLE_COVERAGE, mEnableCoverage); 1043 CLog.d("Added %s to the Json object", ENABLE_COVERAGE); 1044 } else { 1045 CLog.d("Device build has coverage disabled"); 1046 } 1047 } 1048 1049 if (mEnableSancov) { 1050 if (sancovBuild) { 1051 jsonObject.put(ENABLE_SANCOV, mEnableSancov); 1052 CLog.d("Added %s to the Json object", ENABLE_SANCOV); 1053 } else { 1054 CLog.d("Device build has sancov disabled"); 1055 } 1056 } 1057 1058 if (mPreconditionHwBinderServiceName != null) { 1059 jsonObject.put(PRECONDITION_HWBINDER_SERVICE, mPreconditionHwBinderServiceName); 1060 CLog.d("Added %s to the Json object", PRECONDITION_HWBINDER_SERVICE); 1061 } 1062 1063 if (mPreconditionFeature != null) { 1064 jsonObject.put(PRECONDITION_FEATURE, mPreconditionFeature); 1065 CLog.d("Added %s to the Json object", PRECONDITION_FEATURE); 1066 } 1067 1068 if (!mPreconditionFilePathPrefix.isEmpty()) { 1069 jsonObject.put( 1070 PRECONDITION_FILE_PATH_PREFIX, new JSONArray(mPreconditionFilePathPrefix)); 1071 CLog.d("Added %s to the Json object", PRECONDITION_FILE_PATH_PREFIX); 1072 } 1073 1074 if (mPreconditionFirstApiLevel != 0) { 1075 jsonObject.put(PRECONDITION_FIRST_API_LEVEL, mPreconditionFirstApiLevel); 1076 CLog.d("Added %s to the Json object", PRECONDITION_FIRST_API_LEVEL); 1077 } 1078 1079 if (mPreconditionLshal != null) { 1080 jsonObject.put(PRECONDITION_LSHAL, mPreconditionLshal); 1081 CLog.d("Added %s to the Json object", PRECONDITION_LSHAL); 1082 } 1083 1084 if (mPreconditionVintf != null) { 1085 jsonObject.put(PRECONDITION_VINTF, mPreconditionVintf); 1086 CLog.d("Added %s to the Json object", PRECONDITION_VINTF); 1087 } 1088 1089 if (mPreconditionSysProp != null) { 1090 jsonObject.put(PRECONDITION_SYSPROP, mPreconditionSysProp); 1091 CLog.d("Added %s to the Json object", PRECONDITION_SYSPROP); 1092 } 1093 1094 if (!mBinaryTestProfilingLibraryPath.isEmpty()) { 1095 jsonObject.put(BINARY_TEST_PROFILING_LIBRARY_PATH, 1096 new JSONArray(mBinaryTestProfilingLibraryPath)); 1097 CLog.d("Added %s to the Json object", BINARY_TEST_PROFILING_LIBRARY_PATH); 1098 } 1099 1100 if (mDisableFramework) { 1101 jsonObject.put(DISABLE_FRAMEWORK, mDisableFramework); 1102 CLog.d("Added %s to the Json object", DISABLE_FRAMEWORK); 1103 } 1104 1105 if (mStopNativeServers) { 1106 jsonObject.put(STOP_NATIVE_SERVERS, mStopNativeServers); 1107 CLog.d("Added %s to the Json object", STOP_NATIVE_SERVERS); 1108 } 1109 1110 if (mBinaryTestDisableFramework) { 1111 jsonObject.put(BINARY_TEST_DISABLE_FRAMEWORK, mBinaryTestDisableFramework); 1112 CLog.d("Added %s to the Json object", BINARY_TEST_DISABLE_FRAMEWORK); 1113 } 1114 1115 if (mBinaryTestStopNativeServers) { 1116 jsonObject.put(BINARY_TEST_STOP_NATIVE_SERVERS, mBinaryTestStopNativeServers); 1117 CLog.d("Added %s to the Json object", BINARY_TEST_STOP_NATIVE_SERVERS); 1118 } 1119 1120 if (!mNativeServerProcessName.isEmpty()) { 1121 jsonObject.put(NATIVE_SERVER_PROCESS_NAME, new JSONArray(mNativeServerProcessName)); 1122 CLog.d("Added %s to the Json object", NATIVE_SERVER_PROCESS_NAME); 1123 } 1124 1125 if (!mHalHidlReplayTestTracePaths.isEmpty()) { 1126 jsonObject.put(HAL_HIDL_REPLAY_TEST_TRACE_PATHS, 1127 new JSONArray(mHalHidlReplayTestTracePaths)); 1128 CLog.d("Added %s to the Json object", HAL_HIDL_REPLAY_TEST_TRACE_PATHS); 1129 } 1130 1131 if (mHalHidlPackageName != null) { 1132 jsonObject.put(HAL_HIDL_PACKAGE_NAME, mHalHidlPackageName); 1133 CLog.d("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 1134 } 1135 1136 if (mSystraceProcessName != null) { 1137 jsonObject.put(SYSTRACE_PROCESS_NAME, mSystraceProcessName); 1138 CLog.d("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 1139 } 1140 1141 if (mPassthroughMode) { 1142 jsonObject.put(PASSTHROUGH_MODE, mPassthroughMode); 1143 CLog.d("Added %s to the Json object", PASSTHROUGH_MODE); 1144 } 1145 1146 if (mCollectTestsOnly) { 1147 jsonObject.put(COLLECT_TESTS_ONLY, mCollectTestsOnly); 1148 CLog.d("Added %s to the Json object", COLLECT_TESTS_ONLY); 1149 } 1150 1151 if (mGtestBatchMode) { 1152 jsonObject.put(GTEST_BATCH_MODE, mGtestBatchMode); 1153 CLog.d("Added %s to the Json object", GTEST_BATCH_MODE); 1154 } 1155 1156 if (mLtpNumberOfThreads >= 0) { 1157 jsonObject.put(LTP_NUMBER_OF_THREADS, mLtpNumberOfThreads); 1158 CLog.d("Added %s to the Json object", LTP_NUMBER_OF_THREADS); 1159 } 1160 1161 if (mRunAsVtsSelfTest) { 1162 jsonObject.put(RUN_AS_VTS_SELF_TEST, mRunAsVtsSelfTest); 1163 CLog.d("Added %s to the Json object", RUN_AS_VTS_SELF_TEST); 1164 } 1165 1166 if ("vts".equals(mBuildInfo.getTestTag())) { 1167 jsonObject.put(RUN_AS_COMPLIANCE_TEST, true); 1168 CLog.d("Added %s to the Json object", RUN_AS_COMPLIANCE_TEST); 1169 } 1170 1171 if (!mMoblyTestModule.isEmpty()) { 1172 jsonObject.put(MOBLY_TEST_MODULE, new JSONArray(mMoblyTestModule)); 1173 CLog.d("Added %s to the Json object", MOBLY_TEST_MODULE); 1174 } 1175 1176 if (mActsTestModule != null) { 1177 jsonObject.put(ACTS_TEST_MODULE, mActsTestModule); 1178 CLog.d("Added %s to the Json object", ACTS_TEST_MODULE); 1179 } 1180 1181 if (mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV) != null) { 1182 jsonObject.put(VtsPythonVirtualenvPreparer.VIRTUAL_ENV, 1183 mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV).getAbsolutePath()); 1184 } 1185 1186 if (mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3) != null) { 1187 jsonObject.put(VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3, 1188 mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3) 1189 .getAbsolutePath()); 1190 } 1191 1192 if (!mConfigStr.isEmpty()) { 1193 jsonObject.put(CONFIG_STR, new JSONObject(mConfigStr)); 1194 CLog.d("Added %s to the Json object", CONFIG_STR); 1195 } 1196 1197 if (!mConfigInt.isEmpty()) { 1198 jsonObject.put(CONFIG_INT, new JSONObject(mConfigInt)); 1199 CLog.d("Added %s to the Json object", CONFIG_INT); 1200 } 1201 1202 if (!mConfigBool.isEmpty()) { 1203 jsonObject.put(CONFIG_BOOL, new JSONObject(mConfigBool)); 1204 CLog.d("Added %s to the Json object", CONFIG_BOOL); 1205 } 1206 1207 if (mEnableLogUploading) { 1208 jsonObject.put(ENABLE_LOG_UPLOADING, "true"); 1209 CLog.d("Added %s to the Json object with value: true)", ENABLE_LOG_UPLOADING); 1210 } 1211 1212 if (mMaxRetryCount > 0) { 1213 jsonObject.put(MAX_RETRY_COUNT, mMaxRetryCount); 1214 CLog.d("Added %s to the Json object", MAX_RETRY_COUNT); 1215 } 1216 } 1217 1218 /** 1219 * Log a test module execution status to device logcat. 1220 * 1221 * @param status 1222 * @return true if succesful, false otherwise 1223 */ 1224 @VisibleForTesting printToDeviceLogcatAboutTestModuleStatus(String status)1225 protected void printToDeviceLogcatAboutTestModuleStatus(String status) 1226 throws DeviceNotAvailableException { 1227 mDevice.executeShellCommand(String.format( 1228 "log -p i -t \"VTS\" \"[Test Module] %s %s\"", mTestModuleName, status)); 1229 } 1230 1231 /** 1232 * Create vts python test runner test config json file. 1233 * 1234 * @param status 1235 * @throws RuntimeException 1236 * @return test config json file absolute path string 1237 */ 1238 @VisibleForTesting createVtsRunnerTestConfigJsonFile(File vtsRunnerLogDir)1239 protected String createVtsRunnerTestConfigJsonFile(File vtsRunnerLogDir) { 1240 JSONObject jsonObject = new JSONObject(); 1241 try { 1242 updateVtsRunnerTestConfig(jsonObject); 1243 1244 jsonObject.put(LOG_PATH, vtsRunnerLogDir.getAbsolutePath()); 1245 CLog.d("Added %s to the Json object", LOG_PATH); 1246 } catch (IOException e) { 1247 throw new RuntimeException("Failed to read test config json file"); 1248 } catch (JSONException e) { 1249 throw new RuntimeException("Failed to build updated test config json data"); 1250 } 1251 1252 CLog.d("VTS python test config json: %s", jsonObject.toString()); 1253 1254 String jsonFilePath = null; 1255 try { 1256 File tmpFile = FileUtil.createTempFile( 1257 mBuildInfo.getTestTag() + "-config-" + mBuildInfo.getDeviceSerial(), ".json", 1258 vtsRunnerLogDir); 1259 jsonFilePath = tmpFile.getAbsolutePath(); 1260 CLog.d("VTS test config json file path: %s", jsonFilePath); 1261 FileUtil.writeToFile(jsonObject.toString(), tmpFile); 1262 } catch (IOException e) { 1263 throw new RuntimeException("Failed to create vts test config json file"); 1264 } 1265 return jsonFilePath; 1266 } 1267 AddTestModuleKeys(String test_module_name, long test_module_timestamp)1268 private boolean AddTestModuleKeys(String test_module_name, long test_module_timestamp) { 1269 if (test_module_name.length() == 0 || test_module_timestamp == -1) { 1270 CLog.e(String.format("Test module keys (%s,%d) are invalid.", test_module_name, 1271 test_module_timestamp)); 1272 return false; 1273 } 1274 File reportFile = mBuildInfo.getFile(TEST_PLAN_REPORT_FILE); 1275 1276 if (reportFile != null) { 1277 try (FileWriter fw = new FileWriter(reportFile.getAbsoluteFile(), true); 1278 BufferedWriter bw = new BufferedWriter(fw); 1279 PrintWriter out = new PrintWriter(bw)) { 1280 out.println(String.format("%s %s", test_module_name, test_module_timestamp)); 1281 } catch (IOException e) { 1282 CLog.e(String.format( 1283 "Can't write to the test plan result file, %s", TEST_PLAN_REPORT_FILE)); 1284 return false; 1285 } 1286 } else { 1287 CLog.w("No test plan report file configured."); 1288 } 1289 return true; 1290 } 1291 1292 /** 1293 * This method prepares a command for the test and runs the python file as 1294 * given in the arguments. 1295 * 1296 * @param listener 1297 * @throws RuntimeException 1298 * @throws IllegalArgumentException 1299 */ doRunTest(ITestLifeCycleReceiver listener)1300 private void doRunTest(ITestLifeCycleReceiver listener) 1301 throws IllegalArgumentException, DeviceNotAvailableException { 1302 long methodStartTime = System.currentTimeMillis(); 1303 CLog.d("Device serial number: " + mDevice.getSerialNumber()); 1304 1305 setTestCaseDir(); 1306 1307 VtsMultiDeviceTestResultParser parser = 1308 new VtsMultiDeviceTestResultParser(listener, deriveRunName()); 1309 1310 File vtsRunnerLogDir = null; 1311 try { 1312 vtsRunnerLogDir = FileUtil.createTempDir("vts-runner-log"); 1313 } catch(IOException e) { 1314 throw new RuntimeException("Failed to creat temp vts-runner-log directory"); 1315 } 1316 1317 long timeout = mMaxTestTimeout; 1318 if (mMaxTestTimeout < mTestTimeout) { 1319 // The Python runner will receive 2 interrupts. 1320 // Delay the 2nd one to avoid interrupting the runner's teardown procedure. 1321 timeout = mTestTimeout + VtsPythonRunnerHelper.TEST_ABORT_TIMEOUT_MSECS; 1322 CLog.w("max-test-timeout is less than test-timeout. Set max timeout to %dms.", timeout); 1323 } 1324 1325 try { 1326 String jsonFilePath = createVtsRunnerTestConfigJsonFile(vtsRunnerLogDir); 1327 1328 VtsPythonRunnerHelper vtsPythonRunnerHelper = 1329 createVtsPythonRunnerHelper(new File(mTestCaseDir)); 1330 1331 List<String> cmd = new ArrayList<>(); 1332 cmd.add("python"); 1333 if (mTestCasePathType != null && mTestCasePathType.toLowerCase().equals("file")) { 1334 String testScript = mTestCasePath; 1335 if (!testScript.endsWith(".py")) { 1336 testScript += ".py"; 1337 } 1338 cmd.add(testScript); 1339 } else { 1340 cmd.add("-m"); 1341 cmd.add(mTestCasePath.replace("/", ".")); 1342 } 1343 cmd.add(jsonFilePath); 1344 1345 printToDeviceLogcatAboutTestModuleStatus("BEGIN"); 1346 1347 CommandResult commandResult = new CommandResult(); 1348 String interruptMessage = vtsPythonRunnerHelper.runPythonRunner( 1349 cmd.toArray(new String[0]), commandResult, timeout); 1350 1351 List<String> errorMsgs = new ArrayList<>(); 1352 if (commandResult != null) { 1353 CommandStatus commandStatus = commandResult.getStatus(); 1354 if (commandStatus != CommandStatus.SUCCESS 1355 && commandStatus != CommandStatus.TIMED_OUT) { 1356 errorMsgs.add("Python process failed"); 1357 errorMsgs.add("Command stdout: " + commandResult.getStdout()); 1358 errorMsgs.add("Command stderr: " + commandResult.getStderr()); 1359 errorMsgs.add("Command status: " + commandStatus); 1360 } 1361 } 1362 1363 if (mUseStdoutLogs) { 1364 if (commandResult.getStdout() == null) { 1365 errorMsgs.add("The stdout is null for CommandResult."); 1366 } 1367 parser.processNewLines(commandResult.getStdout().split("\n")); 1368 } else { 1369 // parse from test_run_summary.json instead of stdout 1370 File testRunSummary = getFileTestRunSummary(vtsRunnerLogDir); 1371 if (testRunSummary == null) { 1372 errorMsgs.add("Couldn't locate the file : " + TEST_RUN_SUMMARY_FILE_NAME); 1373 } else { 1374 JSONObject object = null; 1375 try { 1376 String jsonData = FileUtil.readStringFromFile(testRunSummary); 1377 CLog.d("Test Result Summary: %s", jsonData); 1378 object = new JSONObject(jsonData); 1379 parser.processJsonFile(object); 1380 } catch (IOException | JSONException e) { 1381 errorMsgs.add( 1382 "Error occurred in parsing Json file " + testRunSummary.toPath()); 1383 CLog.e(e); 1384 } 1385 try { 1386 JSONObject planObject = object.getJSONObject(TESTMODULE); 1387 String test_module_name = planObject.getString("Name"); 1388 long test_module_timestamp = planObject.getLong("Timestamp"); 1389 AddTestModuleKeys(test_module_name, test_module_timestamp); 1390 } catch (JSONException e) { 1391 // Do not report this as part of errorMsgs. These are optional metadata 1392 CLog.e(e); 1393 } 1394 } 1395 } 1396 if (errorMsgs.size() > 0) { 1397 CLog.e(String.join(".\n", errorMsgs)); 1398 listener.testRunFailed(String.join(".\n", errorMsgs)); 1399 listener.testRunEnded(System.currentTimeMillis() - methodStartTime, 1400 new HashMap<String, Metric>()); 1401 } 1402 1403 printToDeviceLogcatAboutTestModuleStatus("END"); 1404 if (interruptMessage != null) { 1405 throw new RuntimeException(interruptMessage); 1406 } 1407 } finally { 1408 try { 1409 mOutputUtil.ZipVtsRunnerOutputDir(vtsRunnerLogDir); 1410 1411 if (mEnableDashboardUploading) { 1412 File reportMsg = FileUtil.findFile(vtsRunnerLogDir, REPORT_MESSAGE_FILE_NAME); 1413 CLog.d("Report message path: %s", reportMsg); 1414 if (reportMsg == null) { 1415 CLog.e("Cannot find report message proto file."); 1416 } else if (reportMsg.length() > 0) { 1417 CLog.i("Uploading report message. File size: %s", reportMsg.length()); 1418 VtsDashboardUtil dashboardUtil = new VtsDashboardUtil(configReader); 1419 dashboardUtil.Upload(reportMsg.getAbsolutePath()); 1420 } 1421 } 1422 } finally { 1423 CLog.d("Deleted the runner log dir, %s.", vtsRunnerLogDir); 1424 FileUtil.recursiveDelete(vtsRunnerLogDir); 1425 } 1426 // If the framework was disabled in python, make sure we re-enable it no matter what. 1427 // The python side never re-enable the framework. 1428 if (mBinaryTestDisableFramework || mStopNativeServers) { 1429 for (ITestDevice device : mInvocationContext.getDevices()) { 1430 device.executeShellCommand("start"); 1431 } 1432 } 1433 } 1434 for (ITestDevice device : mInvocationContext.getDevices()) { 1435 device.waitForDeviceAvailable(); 1436 } 1437 } 1438 1439 /** 1440 * This method return the file test_run_summary.json which is then used to parse logs. 1441 * 1442 * @param logDir : The file that needs to be converted 1443 * @return the file named test_run_summary.json 1444 * @throws IllegalArgumentException 1445 */ getFileTestRunSummary(File logDir)1446 private File getFileTestRunSummary(File logDir) throws IllegalArgumentException { 1447 File[] children; 1448 if (logDir == null) { 1449 throw new IllegalArgumentException("Argument logDir is null."); 1450 } 1451 children = logDir.listFiles(); 1452 if (children != null) { 1453 for (File child : children) { 1454 if (!child.isDirectory()) { 1455 if (child.getName().equals(TEST_RUN_SUMMARY_FILE_NAME)) { 1456 return child; 1457 } 1458 } else { 1459 File file = getFileTestRunSummary(child); 1460 if (file != null) { 1461 return file; 1462 } 1463 } 1464 } 1465 } 1466 return null; 1467 } 1468 1469 /** 1470 * Set the path for android-vts/testcases/ which keeps the VTS python code under vts. 1471 */ setTestCaseDir()1472 private void setTestCaseDir() { 1473 try { 1474 mTestCaseDir = mBuildHelper.getTestsDir().getAbsolutePath(); 1475 } catch (FileNotFoundException e) { 1476 CLog.e("Cannot get testcase dir. Tests may not run correctly."); 1477 } 1478 } 1479 1480 /** 1481 * {@inheritDoc} 1482 */ 1483 @Override setAbi(IAbi abi)1484 public void setAbi(IAbi abi){ 1485 mAbi = abi; 1486 } 1487 1488 /** 1489 * Creates a {@link VtsPythonRunnerHelper}. 1490 */ 1491 @VisibleForTesting createVtsPythonRunnerHelper(File workingDir)1492 protected VtsPythonRunnerHelper createVtsPythonRunnerHelper(File workingDir) { 1493 return new VtsPythonRunnerHelper(mBuildInfo, workingDir); 1494 } 1495 } 1496