1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.presubmit; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.compatibility.common.tradefed.targetprep.ApkInstaller; 23 import com.android.compatibility.common.tradefed.targetprep.PreconditionPreparer; 24 import com.android.compatibility.common.tradefed.testtype.JarHostTest; 25 import com.android.tradefed.build.FolderBuildInfo; 26 import com.android.tradefed.config.ConfigurationDescriptor; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.ConfigurationFactory; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.config.IDeviceConfiguration; 31 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 32 import com.android.tradefed.invoker.InvocationContext; 33 import com.android.tradefed.invoker.TestInformation; 34 import com.android.tradefed.invoker.shard.token.TokenProperty; 35 import com.android.tradefed.targetprep.DeviceSetup; 36 import com.android.tradefed.targetprep.ITargetPreparer; 37 import com.android.tradefed.targetprep.PythonVirtualenvPreparer; 38 import com.android.tradefed.testtype.AndroidJUnitTest; 39 import com.android.tradefed.testtype.GTest; 40 import com.android.tradefed.testtype.HostTest; 41 import com.android.tradefed.testtype.IRemoteTest; 42 import com.android.tradefed.testtype.ITestFilterReceiver; 43 import com.android.tradefed.testtype.suite.ITestSuite; 44 import com.android.tradefed.testtype.suite.params.ModuleParameters; 45 import com.android.tradefed.util.FileUtil; 46 47 import org.junit.Assert; 48 import org.junit.Test; 49 import org.junit.runner.RunWith; 50 import org.junit.runners.JUnit4; 51 52 import java.io.File; 53 import java.io.IOException; 54 import java.util.ArrayList; 55 import java.util.Arrays; 56 import java.util.HashMap; 57 import java.util.HashSet; 58 import java.util.List; 59 import java.util.Map; 60 import java.util.Set; 61 import java.util.regex.Pattern; 62 63 /** 64 * Test that configuration in CTS can load and have expected properties. 65 */ 66 @RunWith(JUnit4.class) 67 public class CtsConfigLoadingTest { 68 69 private static final Pattern TODO_BUG_PATTERN = Pattern.compile(".*TODO\\(b/[0-9]+\\).*", Pattern.DOTALL); 70 private static final String METADATA_COMPONENT = "component"; 71 private static final Set<String> KNOWN_COMPONENTS = 72 new HashSet<>( 73 Arrays.asList( 74 // modifications to the list below must be reviewed 75 "abuse", 76 "art", 77 "auth", 78 "auto", 79 "autofill", 80 "backup", 81 "bionic", 82 "bluetooth", 83 "camera", 84 "contentcapture", 85 "deviceinfo", 86 "deqp", 87 "devtools", 88 "framework", 89 "graphics", 90 "hdmi", 91 "inputmethod", 92 "libcore", 93 "libnativehelper", 94 "location", 95 "media", 96 "metrics", 97 "misc", 98 "mocking", 99 "networking", 100 "neuralnetworks", 101 "print", 102 "renderscript", 103 "security", 104 "statsd", 105 "systems", 106 "sysui", 107 "telecom", 108 "tv", 109 "uitoolkit", 110 "uwb", 111 "vr", 112 "webview", 113 "wifi")); 114 private static final Set<String> KNOWN_MISC_MODULES = 115 new HashSet<>( 116 Arrays.asList( 117 // Modifications to the list below must be approved by someone in 118 // test/suite_harness/OWNERS. 119 "CtsSliceTestCases.config", 120 "CtsSampleDeviceTestCases.config", 121 "CtsUsbTests.config", 122 "CtsGpuToolsHostTestCases.config", 123 "CtsEdiHostTestCases.config", 124 "CtsClassLoaderFactoryPathClassLoaderTestCases.config", 125 "CtsSampleHostTestCases.config", 126 "CtsHardwareTestCases.config", 127 "CtsMonkeyTestCases.config", 128 "CtsAndroidAppTestCases.config", 129 "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config", 130 "CtsAppComponentFactoryTestCases.config", 131 "CtsSeccompHostTestCases.config")); 132 133 /** 134 * List of the officially supported runners in CTS, they meet all the interfaces criteria as 135 * well as support sharding very well. Any new addition should go through a review. 136 */ 137 private static final Set<String> SUPPORTED_CTS_TEST_TYPE = new HashSet<>(Arrays.asList( 138 // Cts runners 139 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 140 "com.android.compatibility.testtype.DalvikTest", 141 "com.android.compatibility.testtype.LibcoreTest", 142 "com.drawelements.deqp.runner.DeqpTestRunner", 143 // Tradefed runners 144 "com.android.tradefed.testtype.AndroidJUnitTest", 145 "com.android.tradefed.testtype.ArtRunTest", 146 "com.android.tradefed.testtype.HostTest", 147 "com.android.tradefed.testtype.GTest", 148 "com.android.tradefed.testtype.mobly.MoblyBinaryHostTest" 149 )); 150 151 /** 152 * In Most cases we impose the usage of the AndroidJUnitRunner because it supports all the 153 * features required (filtering, sharding, etc.). We do not typically expect people to need a 154 * different runner. 155 */ 156 private static final Set<String> ALLOWED_INSTRUMENTATION_RUNNER_NAME = new HashSet<>(); 157 static { 158 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("android.support.test.runner.AndroidJUnitRunner"); 159 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("androidx.test.runner.AndroidJUnitRunner"); 160 } 161 private static final Set<String> RUNNER_EXCEPTION = new HashSet<>(); 162 static { 163 // Used for a bunch of system-api cts tests 164 RUNNER_EXCEPTION.add("repackaged.android.test.InstrumentationTestRunner"); 165 // Used by a UiRendering scenario where an activity is persisted between tests 166 RUNNER_EXCEPTION.add("android.uirendering.cts.runner.UiRenderingRunner"); 167 } 168 169 /** 170 * Families of module parameterization that MUST be specified explicitly in the module 171 * AndroidTest.xml. 172 */ 173 private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>(); 174 175 static { 176 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY); 177 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY); 178 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.SECONDARY_USER_FAMILY); 179 } 180 181 /** 182 * AllowList to start enforcing metadata on modules. No additional entry will be allowed! This 183 * is meant to burn down the remaining modules definition. 184 */ 185 private static final Set<String> ALLOWLIST_MODULE_PARAMETERS = new HashSet<>(); 186 187 static { 188 } 189 190 /** 191 * Test that configuration shipped in Tradefed can be parsed. 192 * -> Exclude deprecated ApkInstaller. 193 * -> Check if host-side tests are non empty. 194 */ 195 @Test testConfigurationLoad()196 public void testConfigurationLoad() throws Exception { 197 String ctsRoot = System.getProperty("CTS_ROOT"); 198 File testcases = new File(ctsRoot, "/android-cts/testcases/"); 199 if (!testcases.exists()) { 200 fail(String.format("%s does not exists", testcases)); 201 return; 202 } 203 Set<File> listConfigs = FileUtil.findFilesObject(testcases, ".*\\.config"); 204 assertTrue(listConfigs.size() > 0); 205 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 206 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 207 stubFolder.setRootDir(new File(ctsRoot)); 208 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, "CTS"); 209 stubFolder.addBuildAttribute("ROOT_DIR", ctsRoot); 210 TestInformation stubTestInfo = TestInformation.newBuilder() 211 .setInvocationContext(new InvocationContext()).build(); 212 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(ctsRoot)); 213 214 List<String> missingMandatoryParameters = new ArrayList<>(); 215 // We expect to be able to load every single config in testcases/ 216 for (File config : listConfigs) { 217 IConfiguration c = ConfigurationFactory.getInstance() 218 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 219 if (c.getDeviceConfig().size() > 2) { 220 throw new ConfigurationException(String.format("%s declares more than 2 devices.", config)); 221 } 222 int deviceCount = 0; 223 for (IDeviceConfiguration dConfig : c.getDeviceConfig()) { 224 // Ensure the deprecated ApkInstaller is not used anymore. 225 for (ITargetPreparer prep : dConfig.getTargetPreparers()) { 226 if (prep.getClass().isAssignableFrom(ApkInstaller.class)) { 227 throw new ConfigurationException( 228 String.format("%s: Use com.android.tradefed.targetprep.suite." 229 + "SuiteApkInstaller instead of com.android.compatibility." 230 + "common.tradefed.targetprep.ApkInstaller, options will be " 231 + "the same.", config)); 232 } 233 if (prep.getClass().isAssignableFrom(PreconditionPreparer.class)) { 234 throw new ConfigurationException( 235 String.format( 236 "%s: includes a PreconditionPreparer (%s) which is not " 237 + "allowed in modules.", 238 config.getName(), prep.getClass())); 239 } 240 if (prep.getClass().isAssignableFrom(DeviceSetup.class)) { 241 DeviceSetup deviceSetup = (DeviceSetup) prep; 242 if (!deviceSetup.isForceSkipSystemProps()) { 243 throw new ConfigurationException( 244 String.format("%s: %s needs to be configured with " 245 + "<option name=\"force-skip-system-props\" " 246 + "value=\"true\" /> in CTS.", 247 config.getName(), prep.getClass())); 248 } 249 } 250 if (prep.getClass().isAssignableFrom(PythonVirtualenvPreparer.class)) { 251 // Ensure each modules has a tracking bug to be imported. 252 checkPythonModules(config, deviceCount); 253 } 254 } 255 deviceCount++; 256 } 257 // We can ensure that Host side tests are not empty. 258 for (IRemoteTest test : c.getTests()) { 259 // Check that all the tests runners are well supported. 260 if (!SUPPORTED_CTS_TEST_TYPE.contains(test.getClass().getCanonicalName())) { 261 throw new ConfigurationException( 262 String.format( 263 "testtype %s is not officially supported by CTS. " 264 + "The supported ones are: %s", 265 test.getClass().getCanonicalName(), SUPPORTED_CTS_TEST_TYPE)); 266 } 267 if (test instanceof HostTest) { 268 HostTest hostTest = (HostTest) test; 269 // We inject a made up folder so that it can find the tests. 270 hostTest.setBuild(stubFolder); 271 hostTest.setTestInformation(stubTestInfo); 272 int testCount = hostTest.countTestCases(); 273 if (testCount == 0) { 274 throw new ConfigurationException( 275 String.format("%s: %s reports 0 test cases.", 276 config.getName(), test)); 277 } 278 } 279 if (test instanceof GTest) { 280 if (((GTest) test).isRebootBeforeTestEnabled()) { 281 throw new ConfigurationException(String.format( 282 "%s: instead of reboot-before-test use a RebootTargetPreparer " 283 + "which is more optimized during sharding.", config.getName())); 284 } 285 } 286 // Tests are expected to implement that interface. 287 if (!(test instanceof ITestFilterReceiver)) { 288 throw new IllegalArgumentException(String.format( 289 "Test in module %s must implement ITestFilterReceiver.", 290 config.getName())); 291 } 292 // Ensure that the device runner is the AJUR one if explicitly specified. 293 if (test instanceof AndroidJUnitTest) { 294 AndroidJUnitTest instru = (AndroidJUnitTest) test; 295 if (instru.getRunnerName() != null && 296 !ALLOWED_INSTRUMENTATION_RUNNER_NAME.contains(instru.getRunnerName())) { 297 // Some runner are exempt 298 if (!RUNNER_EXCEPTION.contains(instru.getRunnerName())) { 299 throw new ConfigurationException( 300 String.format("%s: uses '%s' instead of on of '%s' that are " 301 + "expected", config.getName(), instru.getRunnerName(), 302 ALLOWED_INSTRUMENTATION_RUNNER_NAME)); 303 } 304 } 305 } 306 } 307 308 ConfigurationDescriptor cd = c.getConfigurationDescription(); 309 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 310 List<String> component = cd.getMetaData(METADATA_COMPONENT); 311 Assert.assertNotNull(String.format("Missing module metadata field \"component\", " 312 + "please add the following line to your AndroidTest.xml:\n" 313 + "<option name=\"config-descriptor:metadata\" key=\"component\" " 314 + "value=\"...\" />\nwhere \"value\" must be one of: %s\n" 315 + "config: %s", KNOWN_COMPONENTS, config), 316 component); 317 Assert.assertEquals(String.format("Module config contains more than one \"component\" " 318 + "metadata field: %s\nconfig: %s", component, config), 319 1, component.size()); 320 String cmp = component.get(0); 321 Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata " 322 + "field \"%s\", supported ones are: %s\nconfig: %s", 323 cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp)); 324 325 if ("misc".equals(cmp)) { 326 String configFileName = config.getName(); 327 Assert.assertTrue( 328 String.format( 329 "Adding new module %s to \"misc\" component is restricted, " 330 + "please pick a component that your module fits in", 331 configFileName), 332 KNOWN_MISC_MODULES.contains(configFileName)); 333 } 334 335 // Check that specified parameters are expected 336 boolean res = 337 checkModuleParameters( 338 config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 339 if (!res) { 340 missingMandatoryParameters.add(config.getName()); 341 } 342 // Check that specified tokens are expected 343 checkTokens(config.getName(), cd.getMetaData(ITestSuite.TOKEN_KEY)); 344 345 // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" /> 346 Assert.assertTrue(String.format( 347 "Module config %s does not contains " 348 + "'<option name=\"test-suite-tag\" value=\"cts\" />'", config.getName()), 349 cd.getSuiteTags().contains("cts")); 350 351 // Check not-shardable: JarHostTest cannot create empty shards so it should never need 352 // to be not-shardable. 353 if (cd.isNotShardable()) { 354 for (IRemoteTest test : c.getTests()) { 355 if (test.getClass().isAssignableFrom(JarHostTest.class)) { 356 throw new ConfigurationException( 357 String.format("config: %s. JarHostTest does not need the " 358 + "not-shardable option.", config.getName())); 359 } 360 } 361 } 362 // Ensure options have been set 363 c.validateOptions(); 364 } 365 366 // Exempt the allow list 367 missingMandatoryParameters.removeAll(ALLOWLIST_MODULE_PARAMETERS); 368 // Ensure the mandatory fields are filled 369 if (!missingMandatoryParameters.isEmpty()) { 370 String msg = 371 String.format( 372 "The following %s modules are missing some of the mandatory " 373 + "parameters [instant_app, not_instant_app, " 374 + "multi_abi, not_multi_abi, " 375 + "secondary_user, not_secondary_user]: '%s'", 376 missingMandatoryParameters.size(), missingMandatoryParameters); 377 throw new ConfigurationException(msg); 378 } 379 } 380 381 /** Test that all parameter metadata can be resolved. */ checkModuleParameters(String configName, List<String> parameters)382 private boolean checkModuleParameters(String configName, List<String> parameters) 383 throws ConfigurationException { 384 if (parameters == null) { 385 return false; 386 } 387 Map<String, Boolean> families = createFamilyCheckMap(); 388 for (String param : parameters) { 389 try { 390 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase()); 391 if (families.containsKey(p.getFamily())) { 392 families.put(p.getFamily(), true); 393 } 394 } catch (IllegalArgumentException e) { 395 throw new ConfigurationException( 396 String.format("Config: %s includes an unknown parameter '%s'.", 397 configName, param)); 398 } 399 } 400 if (families.containsValue(false)) { 401 return false; 402 } 403 return true; 404 } 405 406 /** Test that all tokens can be resolved. */ checkTokens(String configName, List<String> tokens)407 private void checkTokens(String configName, List<String> tokens) throws ConfigurationException { 408 if (tokens == null) { 409 return; 410 } 411 for (String token : tokens) { 412 try { 413 TokenProperty.valueOf(token.toUpperCase()); 414 } catch (IllegalArgumentException e) { 415 throw new ConfigurationException( 416 String.format( 417 "Config: %s includes an unknown token '%s'.", configName, token)); 418 } 419 } 420 } 421 createFamilyCheckMap()422 private Map<String, Boolean> createFamilyCheckMap() { 423 Map<String, Boolean> families = new HashMap<>(); 424 for (String family : MANDATORY_PARAMETERS_FAMILY) { 425 families.put(family, false); 426 } 427 return families; 428 } 429 430 /** 431 * For each usage of python virtualenv preparer, make sure we have tracking bugs to import as 432 * source the python libs. 433 */ checkPythonModules(File config, int deviceCount)434 private void checkPythonModules(File config, int deviceCount) 435 throws IOException, ConfigurationException { 436 if (deviceCount != 0) { 437 throw new ConfigurationException( 438 String.format("%s: PythonVirtualenvPreparer should only be declared for " 439 + "the first <device> tag in the config", config.getName())); 440 } 441 if (!TODO_BUG_PATTERN.matcher(FileUtil.readStringFromFile(config)).matches()) { 442 throw new ConfigurationException( 443 String.format("%s: Contains some virtualenv python lib usage but no " 444 + "tracking bug to import them as source.", config.getName())); 445 } 446 } 447 } 448