1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.presubmit; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.compatibility.common.tradefed.targetprep.ApkInstaller; 23 import com.android.compatibility.common.tradefed.targetprep.PreconditionPreparer; 24 import com.android.compatibility.common.tradefed.testtype.JarHostTest; 25 import com.android.tradefed.build.FolderBuildInfo; 26 import com.android.tradefed.config.ConfigurationDescriptor; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.ConfigurationFactory; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.invoker.shard.token.TokenProperty; 31 import com.android.tradefed.targetprep.ITargetPreparer; 32 import com.android.tradefed.testtype.AndroidJUnitTest; 33 import com.android.tradefed.testtype.HostTest; 34 import com.android.tradefed.testtype.IRemoteTest; 35 import com.android.tradefed.testtype.ITestFilterReceiver; 36 import com.android.tradefed.testtype.suite.ITestSuite; 37 import com.android.tradefed.testtype.suite.params.ModuleParameters; 38 39 import org.junit.Assert; 40 import org.junit.Test; 41 import org.junit.runner.RunWith; 42 import org.junit.runners.JUnit4; 43 44 import java.io.File; 45 import java.io.FilenameFilter; 46 import java.util.ArrayList; 47 import java.util.Arrays; 48 import java.util.HashMap; 49 import java.util.HashSet; 50 import java.util.List; 51 import java.util.Map; 52 import java.util.Set; 53 54 /** 55 * Test that configuration in CTS can load and have expected properties. 56 */ 57 @RunWith(JUnit4.class) 58 public class CtsConfigLoadingTest { 59 60 private static final String METADATA_COMPONENT = "component"; 61 private static final Set<String> KNOWN_COMPONENTS = new HashSet<>(Arrays.asList( 62 // modifications to the list below must be reviewed 63 "abuse", 64 "art", 65 "auth", 66 "auto", 67 "autofill", 68 "backup", 69 "bionic", 70 "bluetooth", 71 "camera", 72 "contentcapture", 73 "deviceinfo", 74 "deqp", 75 "devtools", 76 "framework", 77 "graphics", 78 "inputmethod", 79 "libcore", 80 "location", 81 "media", 82 "metrics", 83 "misc", 84 "mocking", 85 "networking", 86 "neuralnetworks", 87 "print", 88 "renderscript", 89 "security", 90 "statsd", 91 "systems", 92 "sysui", 93 "telecom", 94 "tv", 95 "uitoolkit", 96 "vr", 97 "webview" 98 )); 99 private static final Set<String> KNOWN_MISC_MODULES = 100 new HashSet<>( 101 Arrays.asList( 102 // Modifications to the list below must be approved by someone in 103 // test/suite_harness/OWNERS. 104 "CtsSliceTestCases.config", 105 "CtsSampleDeviceTestCases.config", 106 "CtsUsbTests.config", 107 "CtsGpuToolsHostTestCases.config", 108 "CtsEdiHostTestCases.config", 109 "CtsClassLoaderFactoryPathClassLoaderTestCases.config", 110 "CtsSampleHostTestCases.config", 111 "CtsHardwareTestCases.config", 112 "CtsMonkeyTestCases.config", 113 "CtsAndroidAppTestCases.config", 114 "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config", 115 "CtsAppComponentFactoryTestCases.config", 116 "CtsSeccompHostTestCases.config")); 117 118 /** 119 * List of the officially supported runners in CTS, they meet all the interfaces criteria as 120 * well as support sharding very well. Any new addition should go through a review. 121 */ 122 private static final Set<String> SUPPORTED_CTS_TEST_TYPE = new HashSet<>(Arrays.asList( 123 // Cts runners 124 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 125 "com.android.compatibility.testtype.DalvikTest", 126 "com.android.compatibility.testtype.LibcoreTest", 127 "com.drawelements.deqp.runner.DeqpTestRunner", 128 // Tradefed runners 129 "com.android.tradefed.testtype.AndroidJUnitTest", 130 "com.android.tradefed.testtype.HostTest", 131 "com.android.tradefed.testtype.GTest" 132 )); 133 134 /** 135 * In Most cases we impose the usage of the AndroidJUnitRunner because it supports all the 136 * features required (filtering, sharding, etc.). We do not typically expect people to need a 137 * different runner. 138 */ 139 private static final Set<String> ALLOWED_INSTRUMENTATION_RUNNER_NAME = new HashSet<>(); 140 static { 141 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("android.support.test.runner.AndroidJUnitRunner"); 142 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("androidx.test.runner.AndroidJUnitRunner"); 143 } 144 private static final Set<String> RUNNER_EXCEPTION = new HashSet<>(); 145 static { 146 // Used for a bunch of system-api cts tests 147 RUNNER_EXCEPTION.add("repackaged.android.test.InstrumentationTestRunner"); 148 // Used by a UiRendering scenario where an activity is persisted between tests 149 RUNNER_EXCEPTION.add("android.uirendering.cts.runner.UiRenderingRunner"); 150 } 151 152 /** 153 * Families of module parameterization that MUST be specified explicitly in the module 154 * AndroidTest.xml. 155 */ 156 private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>(); 157 158 static { 159 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY); 160 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY); 161 } 162 163 /** 164 * Whitelist to start enforcing metadata on modules. No additional entry will be allowed! This 165 * is meant to burn down the remaining modules definition. 166 */ 167 private static final Set<String> WHITELIST_MODULE_PARAMETERS = new HashSet<>(); 168 169 static { 170 WHITELIST_MODULE_PARAMETERS.add("CtsWidgetTestCases.config"); 171 WHITELIST_MODULE_PARAMETERS.add("CtsSimpleCpuTestCases.config"); 172 WHITELIST_MODULE_PARAMETERS.add("CtsAppSecurityHostTestCases.config"); 173 WHITELIST_MODULE_PARAMETERS.add("CtsMediaStressTestCases.config"); 174 WHITELIST_MODULE_PARAMETERS.add("CtsTelephonySdk28TestCases.config"); 175 WHITELIST_MODULE_PARAMETERS.add("CtsHardwareTestCases.config"); 176 WHITELIST_MODULE_PARAMETERS.add("CtsGestureTestCases.config"); 177 WHITELIST_MODULE_PARAMETERS.add("CtsNetTestCases.config"); 178 WHITELIST_MODULE_PARAMETERS.add("CtsPermissionTestCasesSdk28.config"); 179 } 180 181 /** 182 * Test that configuration shipped in Tradefed can be parsed. 183 * -> Exclude deprecated ApkInstaller. 184 * -> Check if host-side tests are non empty. 185 */ 186 @Test testConfigurationLoad()187 public void testConfigurationLoad() throws Exception { 188 String ctsRoot = System.getProperty("CTS_ROOT"); 189 File testcases = new File(ctsRoot, "/android-cts/testcases/"); 190 if (!testcases.exists()) { 191 fail(String.format("%s does not exists", testcases)); 192 return; 193 } 194 File[] listConfig = testcases.listFiles(new FilenameFilter() { 195 @Override 196 public boolean accept(File dir, String name) { 197 if (name.endsWith(".config")) { 198 return true; 199 } 200 return false; 201 } 202 }); 203 assertTrue(listConfig.length > 0); 204 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 205 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 206 stubFolder.setRootDir(new File(ctsRoot)); 207 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, "CTS"); 208 stubFolder.addBuildAttribute("ROOT_DIR", ctsRoot); 209 List<String> missingMandatoryParameters = new ArrayList<>(); 210 // We expect to be able to load every single config in testcases/ 211 for (File config : listConfig) { 212 IConfiguration c = ConfigurationFactory.getInstance() 213 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 214 // Ensure the deprecated ApkInstaller is not used anymore. 215 for (ITargetPreparer prep : c.getTargetPreparers()) { 216 if (prep.getClass().isAssignableFrom(ApkInstaller.class)) { 217 throw new ConfigurationException( 218 String.format("%s: Use com.android.tradefed.targetprep.suite." 219 + "SuiteApkInstaller instead of com.android.compatibility." 220 + "common.tradefed.targetprep.ApkInstaller, options will be " 221 + "the same.", config)); 222 } 223 if (prep.getClass().isAssignableFrom(PreconditionPreparer.class)) { 224 throw new ConfigurationException( 225 String.format( 226 "%s: includes a PreconditionPreparer (%s) which is not allowed" 227 + " in modules.", 228 config.getName(), prep.getClass())); 229 } 230 } 231 // We can ensure that Host side tests are not empty. 232 for (IRemoteTest test : c.getTests()) { 233 // Check that all the tests runners are well supported. 234 if (!SUPPORTED_CTS_TEST_TYPE.contains(test.getClass().getCanonicalName())) { 235 throw new ConfigurationException( 236 String.format("testtype %s is not officially supported by CTS.", 237 test.getClass().getCanonicalName())); 238 } 239 if (test instanceof HostTest) { 240 HostTest hostTest = (HostTest) test; 241 // We inject a made up folder so that it can find the tests. 242 hostTest.setBuild(stubFolder); 243 int testCount = hostTest.countTestCases(); 244 if (testCount == 0) { 245 throw new ConfigurationException( 246 String.format("%s: %s reports 0 test cases.", 247 config.getName(), test)); 248 } 249 } 250 // Tests are expected to implement that interface. 251 if (!(test instanceof ITestFilterReceiver)) { 252 throw new IllegalArgumentException(String.format( 253 "Test in module %s must implement ITestFilterReceiver.", 254 config.getName())); 255 } 256 // Ensure that the device runner is the AJUR one if explicitly specified. 257 if (test instanceof AndroidJUnitTest) { 258 AndroidJUnitTest instru = (AndroidJUnitTest) test; 259 if (instru.getRunnerName() != null && 260 !ALLOWED_INSTRUMENTATION_RUNNER_NAME.contains(instru.getRunnerName())) { 261 // Some runner are exempt 262 if (!RUNNER_EXCEPTION.contains(instru.getRunnerName())) { 263 throw new ConfigurationException( 264 String.format("%s: uses '%s' instead of on of '%s' that are " 265 + "expected", config.getName(), instru.getRunnerName(), 266 ALLOWED_INSTRUMENTATION_RUNNER_NAME)); 267 } 268 } 269 } 270 } 271 ConfigurationDescriptor cd = c.getConfigurationDescription(); 272 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 273 List<String> component = cd.getMetaData(METADATA_COMPONENT); 274 Assert.assertNotNull(String.format("Missing module metadata field \"component\", " 275 + "please add the following line to your AndroidTest.xml:\n" 276 + "<option name=\"config-descriptor:metadata\" key=\"component\" " 277 + "value=\"...\" />\nwhere \"value\" must be one of: %s\n" 278 + "config: %s", KNOWN_COMPONENTS, config), 279 component); 280 Assert.assertEquals(String.format("Module config contains more than one \"component\" " 281 + "metadata field: %s\nconfig: %s", component, config), 282 1, component.size()); 283 String cmp = component.get(0); 284 Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata " 285 + "field \"%s\", supported ones are: %s\nconfig: %s", 286 cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp)); 287 288 if ("misc".equals(cmp)) { 289 String configFileName = config.getName(); 290 Assert.assertTrue( 291 String.format( 292 "Adding new module %s to \"misc\" component is restricted, " 293 + "please pick a component that your module fits in", 294 configFileName), 295 KNOWN_MISC_MODULES.contains(configFileName)); 296 } 297 298 // Check that specified parameters are expected 299 boolean res = 300 checkModuleParameters( 301 config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 302 if (!res) { 303 missingMandatoryParameters.add(config.getName()); 304 } 305 // Check that specified tokens are expected 306 checkTokens(config.getName(), cd.getMetaData(ITestSuite.TOKEN_KEY)); 307 308 // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" /> 309 Assert.assertTrue(String.format( 310 "Module config %s does not contains " 311 + "'<option name=\"test-suite-tag\" value=\"cts\" />'", config.getName()), 312 cd.getSuiteTags().contains("cts")); 313 314 // Check not-shardable: JarHostTest cannot create empty shards so it should never need 315 // to be not-shardable. 316 if (cd.isNotShardable()) { 317 for (IRemoteTest test : c.getTests()) { 318 if (test.getClass().isAssignableFrom(JarHostTest.class)) { 319 throw new ConfigurationException( 320 String.format("config: %s. JarHostTest does not need the " 321 + "not-shardable option.", config.getName())); 322 } 323 } 324 } 325 // Ensure options have been set 326 c.validateOptions(); 327 } 328 329 // Exempt the whitelist 330 missingMandatoryParameters.removeAll(WHITELIST_MODULE_PARAMETERS); 331 // Ensure the mandatory fields are filled 332 if (!missingMandatoryParameters.isEmpty()) { 333 String msg = 334 String.format( 335 "The following %s modules are missing some of the mandatory " 336 + "parameters [instant_app, not_instant_app, " 337 + "multi_abi, not_multi_abi]: '%s'", 338 missingMandatoryParameters.size(), missingMandatoryParameters); 339 throw new ConfigurationException(msg); 340 } 341 } 342 343 /** Test that all parameter metadata can be resolved. */ checkModuleParameters(String configName, List<String> parameters)344 private boolean checkModuleParameters(String configName, List<String> parameters) 345 throws ConfigurationException { 346 if (parameters == null) { 347 return false; 348 } 349 Map<String, Boolean> families = createFamilyCheckMap(); 350 for (String param : parameters) { 351 try { 352 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase()); 353 if (families.containsKey(p.getFamily())) { 354 families.put(p.getFamily(), true); 355 } 356 } catch (IllegalArgumentException e) { 357 throw new ConfigurationException( 358 String.format("Config: %s includes an unknown parameter '%s'.", 359 configName, param)); 360 } 361 } 362 if (families.containsValue(false)) { 363 return false; 364 } 365 return true; 366 } 367 368 /** Test that all tokens can be resolved. */ checkTokens(String configName, List<String> tokens)369 private void checkTokens(String configName, List<String> tokens) throws ConfigurationException { 370 if (tokens == null) { 371 return; 372 } 373 for (String token : tokens) { 374 try { 375 TokenProperty.valueOf(token.toUpperCase()); 376 } catch (IllegalArgumentException e) { 377 throw new ConfigurationException( 378 String.format( 379 "Config: %s includes an unknown token '%s'.", configName, token)); 380 } 381 } 382 } 383 createFamilyCheckMap()384 private Map<String, Boolean> createFamilyCheckMap() { 385 Map<String, Boolean> families = new HashMap<>(); 386 for (String family : MANDATORY_PARAMETERS_FAMILY) { 387 families.put(family, false); 388 } 389 return families; 390 } 391 } 392