1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.presubmit; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.compatibility.common.tradefed.targetprep.ApkInstaller; 23 import com.android.compatibility.common.tradefed.targetprep.PreconditionPreparer; 24 import com.android.compatibility.common.tradefed.testtype.JarHostTest; 25 import com.android.tradefed.build.FolderBuildInfo; 26 import com.android.tradefed.config.ConfigurationDescriptor; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.ConfigurationFactory; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.invoker.shard.token.TokenProperty; 33 import com.android.tradefed.targetprep.ITargetPreparer; 34 import com.android.tradefed.testtype.AndroidJUnitTest; 35 import com.android.tradefed.testtype.HostTest; 36 import com.android.tradefed.testtype.IRemoteTest; 37 import com.android.tradefed.testtype.ITestFilterReceiver; 38 import com.android.tradefed.testtype.suite.ITestSuite; 39 import com.android.tradefed.testtype.suite.params.ModuleParameters; 40 41 import org.junit.Assert; 42 import org.junit.Test; 43 import org.junit.runner.RunWith; 44 import org.junit.runners.JUnit4; 45 46 import java.io.File; 47 import java.io.FilenameFilter; 48 import java.util.ArrayList; 49 import java.util.Arrays; 50 import java.util.HashMap; 51 import java.util.HashSet; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Set; 55 56 /** 57 * Test that configuration in CTS can load and have expected properties. 58 */ 59 @RunWith(JUnit4.class) 60 public class CtsConfigLoadingTest { 61 62 private static final String METADATA_COMPONENT = "component"; 63 private static final Set<String> KNOWN_COMPONENTS = 64 new HashSet<>( 65 Arrays.asList( 66 // modifications to the list below must be reviewed 67 "abuse", 68 "art", 69 "auth", 70 "auto", 71 "autofill", 72 "backup", 73 "bionic", 74 "bluetooth", 75 "camera", 76 "contentcapture", 77 "deviceinfo", 78 "deqp", 79 "devtools", 80 "framework", 81 "graphics", 82 "hdmi", 83 "inputmethod", 84 "libcore", 85 "location", 86 "media", 87 "metrics", 88 "misc", 89 "mocking", 90 "networking", 91 "neuralnetworks", 92 "print", 93 "renderscript", 94 "security", 95 "statsd", 96 "systems", 97 "sysui", 98 "telecom", 99 "tv", 100 "uitoolkit", 101 "vr", 102 "webview", 103 "wifi")); 104 private static final Set<String> KNOWN_MISC_MODULES = 105 new HashSet<>( 106 Arrays.asList( 107 // Modifications to the list below must be approved by someone in 108 // test/suite_harness/OWNERS. 109 "CtsSliceTestCases.config", 110 "CtsSampleDeviceTestCases.config", 111 "CtsUsbTests.config", 112 "CtsGpuToolsHostTestCases.config", 113 "CtsEdiHostTestCases.config", 114 "CtsClassLoaderFactoryPathClassLoaderTestCases.config", 115 "CtsSampleHostTestCases.config", 116 "CtsHardwareTestCases.config", 117 "CtsMonkeyTestCases.config", 118 "CtsAndroidAppTestCases.config", 119 "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config", 120 "CtsAppComponentFactoryTestCases.config", 121 "CtsSeccompHostTestCases.config")); 122 123 /** 124 * List of the officially supported runners in CTS, they meet all the interfaces criteria as 125 * well as support sharding very well. Any new addition should go through a review. 126 */ 127 private static final Set<String> SUPPORTED_CTS_TEST_TYPE = new HashSet<>(Arrays.asList( 128 // Cts runners 129 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 130 "com.android.compatibility.testtype.DalvikTest", 131 "com.android.compatibility.testtype.LibcoreTest", 132 "com.drawelements.deqp.runner.DeqpTestRunner", 133 // Tradefed runners 134 "com.android.tradefed.testtype.AndroidJUnitTest", 135 "com.android.tradefed.testtype.HostTest", 136 "com.android.tradefed.testtype.GTest" 137 )); 138 139 /** 140 * In Most cases we impose the usage of the AndroidJUnitRunner because it supports all the 141 * features required (filtering, sharding, etc.). We do not typically expect people to need a 142 * different runner. 143 */ 144 private static final Set<String> ALLOWED_INSTRUMENTATION_RUNNER_NAME = new HashSet<>(); 145 static { 146 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("android.support.test.runner.AndroidJUnitRunner"); 147 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("androidx.test.runner.AndroidJUnitRunner"); 148 } 149 private static final Set<String> RUNNER_EXCEPTION = new HashSet<>(); 150 static { 151 // Used for a bunch of system-api cts tests 152 RUNNER_EXCEPTION.add("repackaged.android.test.InstrumentationTestRunner"); 153 // Used by a UiRendering scenario where an activity is persisted between tests 154 RUNNER_EXCEPTION.add("android.uirendering.cts.runner.UiRenderingRunner"); 155 } 156 157 /** 158 * Families of module parameterization that MUST be specified explicitly in the module 159 * AndroidTest.xml. 160 */ 161 private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>(); 162 163 static { 164 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY); 165 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY); 166 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.SECONDARY_USER_FAMILY); 167 } 168 169 /** 170 * Whitelist to start enforcing metadata on modules. No additional entry will be allowed! This 171 * is meant to burn down the remaining modules definition. 172 */ 173 private static final Set<String> WHITELIST_MODULE_PARAMETERS = new HashSet<>(); 174 175 static { 176 WHITELIST_MODULE_PARAMETERS.add("CtsAccessibilityServiceTestCases.config"); 177 WHITELIST_MODULE_PARAMETERS.add("CtsActivityManagerBackgroundActivityTestCases.config"); 178 WHITELIST_MODULE_PARAMETERS.add("CtsAppOpsTestCases.config"); 179 WHITELIST_MODULE_PARAMETERS.add("CtsCarrierApiTestCases.config"); 180 WHITELIST_MODULE_PARAMETERS.add("CtsContentCaptureServiceTestCases.config"); 181 WHITELIST_MODULE_PARAMETERS.add("CtsDeqpTestCases.config"); 182 WHITELIST_MODULE_PARAMETERS.add("CtsHiddenApiKillswitchDebugClassTestCases.config"); 183 WHITELIST_MODULE_PARAMETERS.add("CtsHiddenApiKillswitchWhitelistTestCases.config"); 184 WHITELIST_MODULE_PARAMETERS.add("CtsHiddenApiKillswitchWildcardTestCases.config"); 185 WHITELIST_MODULE_PARAMETERS.add("CtsLocationTestCases.config"); 186 WHITELIST_MODULE_PARAMETERS.add("CtsLocation2TestCases.config"); 187 WHITELIST_MODULE_PARAMETERS.add("CtsMediaTestCases.config"); 188 WHITELIST_MODULE_PARAMETERS.add("CtsMediaV2TestCases.config"); 189 WHITELIST_MODULE_PARAMETERS.add("CtsOpenGlPerfTestCases.config"); 190 WHITELIST_MODULE_PARAMETERS.add("CtsOsTestCases.config"); 191 WHITELIST_MODULE_PARAMETERS.add("CtsPermission2TestCases.config"); 192 WHITELIST_MODULE_PARAMETERS.add("CtsPermissionTestCases.config"); 193 WHITELIST_MODULE_PARAMETERS.add("CtsProviderUiTestCases.config"); 194 WHITELIST_MODULE_PARAMETERS.add("CtsRsBlasTestCases.config"); 195 WHITELIST_MODULE_PARAMETERS.add("CtsSkQPTestCases.config"); 196 WHITELIST_MODULE_PARAMETERS.add("CtsWrapNoWrapTestCases.config"); 197 WHITELIST_MODULE_PARAMETERS.add("CtsWrapWrapDebugMallocDebugTestCases.config"); 198 } 199 200 /** 201 * Test that configuration shipped in Tradefed can be parsed. 202 * -> Exclude deprecated ApkInstaller. 203 * -> Check if host-side tests are non empty. 204 */ 205 @Test testConfigurationLoad()206 public void testConfigurationLoad() throws Exception { 207 String ctsRoot = System.getProperty("CTS_ROOT"); 208 File testcases = new File(ctsRoot, "/android-cts/testcases/"); 209 if (!testcases.exists()) { 210 fail(String.format("%s does not exists", testcases)); 211 return; 212 } 213 File[] listConfig = testcases.listFiles(new FilenameFilter() { 214 @Override 215 public boolean accept(File dir, String name) { 216 if (name.endsWith(".config")) { 217 return true; 218 } 219 return false; 220 } 221 }); 222 assertTrue(listConfig.length > 0); 223 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 224 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 225 stubFolder.setRootDir(new File(ctsRoot)); 226 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, "CTS"); 227 stubFolder.addBuildAttribute("ROOT_DIR", ctsRoot); 228 TestInformation stubTestInfo = TestInformation.newBuilder().build(); 229 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(ctsRoot)); 230 231 List<String> missingMandatoryParameters = new ArrayList<>(); 232 // We expect to be able to load every single config in testcases/ 233 for (File config : listConfig) { 234 IConfiguration c = ConfigurationFactory.getInstance() 235 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 236 // Ensure the deprecated ApkInstaller is not used anymore. 237 for (ITargetPreparer prep : c.getTargetPreparers()) { 238 if (prep.getClass().isAssignableFrom(ApkInstaller.class)) { 239 throw new ConfigurationException( 240 String.format("%s: Use com.android.tradefed.targetprep.suite." 241 + "SuiteApkInstaller instead of com.android.compatibility." 242 + "common.tradefed.targetprep.ApkInstaller, options will be " 243 + "the same.", config)); 244 } 245 if (prep.getClass().isAssignableFrom(PreconditionPreparer.class)) { 246 throw new ConfigurationException( 247 String.format( 248 "%s: includes a PreconditionPreparer (%s) which is not allowed" 249 + " in modules.", 250 config.getName(), prep.getClass())); 251 } 252 } 253 // We can ensure that Host side tests are not empty. 254 for (IRemoteTest test : c.getTests()) { 255 // Check that all the tests runners are well supported. 256 if (!SUPPORTED_CTS_TEST_TYPE.contains(test.getClass().getCanonicalName())) { 257 throw new ConfigurationException( 258 String.format( 259 "testtype %s is not officially supported by CTS. " 260 + "The supported ones are: %s", 261 test.getClass().getCanonicalName(), SUPPORTED_CTS_TEST_TYPE)); 262 } 263 if (test instanceof HostTest) { 264 HostTest hostTest = (HostTest) test; 265 // We inject a made up folder so that it can find the tests. 266 hostTest.setBuild(stubFolder); 267 hostTest.setTestInformation(stubTestInfo); 268 int testCount = hostTest.countTestCases(); 269 if (testCount == 0) { 270 throw new ConfigurationException( 271 String.format("%s: %s reports 0 test cases.", 272 config.getName(), test)); 273 } 274 } 275 // Tests are expected to implement that interface. 276 if (!(test instanceof ITestFilterReceiver)) { 277 throw new IllegalArgumentException(String.format( 278 "Test in module %s must implement ITestFilterReceiver.", 279 config.getName())); 280 } 281 // Ensure that the device runner is the AJUR one if explicitly specified. 282 if (test instanceof AndroidJUnitTest) { 283 AndroidJUnitTest instru = (AndroidJUnitTest) test; 284 if (instru.getRunnerName() != null && 285 !ALLOWED_INSTRUMENTATION_RUNNER_NAME.contains(instru.getRunnerName())) { 286 // Some runner are exempt 287 if (!RUNNER_EXCEPTION.contains(instru.getRunnerName())) { 288 throw new ConfigurationException( 289 String.format("%s: uses '%s' instead of on of '%s' that are " 290 + "expected", config.getName(), instru.getRunnerName(), 291 ALLOWED_INSTRUMENTATION_RUNNER_NAME)); 292 } 293 } 294 } 295 } 296 ConfigurationDescriptor cd = c.getConfigurationDescription(); 297 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 298 List<String> component = cd.getMetaData(METADATA_COMPONENT); 299 Assert.assertNotNull(String.format("Missing module metadata field \"component\", " 300 + "please add the following line to your AndroidTest.xml:\n" 301 + "<option name=\"config-descriptor:metadata\" key=\"component\" " 302 + "value=\"...\" />\nwhere \"value\" must be one of: %s\n" 303 + "config: %s", KNOWN_COMPONENTS, config), 304 component); 305 Assert.assertEquals(String.format("Module config contains more than one \"component\" " 306 + "metadata field: %s\nconfig: %s", component, config), 307 1, component.size()); 308 String cmp = component.get(0); 309 Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata " 310 + "field \"%s\", supported ones are: %s\nconfig: %s", 311 cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp)); 312 313 if ("misc".equals(cmp)) { 314 String configFileName = config.getName(); 315 Assert.assertTrue( 316 String.format( 317 "Adding new module %s to \"misc\" component is restricted, " 318 + "please pick a component that your module fits in", 319 configFileName), 320 KNOWN_MISC_MODULES.contains(configFileName)); 321 } 322 323 // Check that specified parameters are expected 324 boolean res = 325 checkModuleParameters( 326 config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 327 if (!res) { 328 missingMandatoryParameters.add(config.getName()); 329 } 330 // Check that specified tokens are expected 331 checkTokens(config.getName(), cd.getMetaData(ITestSuite.TOKEN_KEY)); 332 333 // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" /> 334 Assert.assertTrue(String.format( 335 "Module config %s does not contains " 336 + "'<option name=\"test-suite-tag\" value=\"cts\" />'", config.getName()), 337 cd.getSuiteTags().contains("cts")); 338 339 // Check not-shardable: JarHostTest cannot create empty shards so it should never need 340 // to be not-shardable. 341 if (cd.isNotShardable()) { 342 for (IRemoteTest test : c.getTests()) { 343 if (test.getClass().isAssignableFrom(JarHostTest.class)) { 344 throw new ConfigurationException( 345 String.format("config: %s. JarHostTest does not need the " 346 + "not-shardable option.", config.getName())); 347 } 348 } 349 } 350 // Ensure options have been set 351 c.validateOptions(); 352 } 353 354 // Exempt the whitelist 355 missingMandatoryParameters.removeAll(WHITELIST_MODULE_PARAMETERS); 356 // Ensure the mandatory fields are filled 357 if (!missingMandatoryParameters.isEmpty()) { 358 String msg = 359 String.format( 360 "The following %s modules are missing some of the mandatory " 361 + "parameters [instant_app, not_instant_app, " 362 + "multi_abi, not_multi_abi, " 363 + "secondary_user, not_secondary_user]: '%s'", 364 missingMandatoryParameters.size(), missingMandatoryParameters); 365 throw new ConfigurationException(msg); 366 } 367 } 368 369 /** Test that all parameter metadata can be resolved. */ checkModuleParameters(String configName, List<String> parameters)370 private boolean checkModuleParameters(String configName, List<String> parameters) 371 throws ConfigurationException { 372 if (parameters == null) { 373 return false; 374 } 375 Map<String, Boolean> families = createFamilyCheckMap(); 376 for (String param : parameters) { 377 try { 378 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase()); 379 if (families.containsKey(p.getFamily())) { 380 families.put(p.getFamily(), true); 381 } 382 } catch (IllegalArgumentException e) { 383 throw new ConfigurationException( 384 String.format("Config: %s includes an unknown parameter '%s'.", 385 configName, param)); 386 } 387 } 388 if (families.containsValue(false)) { 389 return false; 390 } 391 return true; 392 } 393 394 /** Test that all tokens can be resolved. */ checkTokens(String configName, List<String> tokens)395 private void checkTokens(String configName, List<String> tokens) throws ConfigurationException { 396 if (tokens == null) { 397 return; 398 } 399 for (String token : tokens) { 400 try { 401 TokenProperty.valueOf(token.toUpperCase()); 402 } catch (IllegalArgumentException e) { 403 throw new ConfigurationException( 404 String.format( 405 "Config: %s includes an unknown token '%s'.", configName, token)); 406 } 407 } 408 } 409 createFamilyCheckMap()410 private Map<String, Boolean> createFamilyCheckMap() { 411 Map<String, Boolean> families = new HashMap<>(); 412 for (String family : MANDATORY_PARAMETERS_FAMILY) { 413 families.put(family, false); 414 } 415 return families; 416 } 417 } 418