1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.loading; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.tradefed.build.FolderBuildInfo; 23 import com.android.tradefed.config.ConfigurationDescriptor; 24 import com.android.tradefed.config.ConfigurationException; 25 import com.android.tradefed.config.ConfigurationFactory; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.config.IDeviceConfiguration; 28 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 29 import com.android.tradefed.invoker.InvocationContext; 30 import com.android.tradefed.invoker.TestInformation; 31 import com.android.tradefed.targetprep.ITargetPreparer; 32 import com.android.tradefed.targetprep.RootTargetPreparer; 33 import com.android.tradefed.testtype.suite.ITestSuite; 34 import com.android.tradefed.testtype.suite.TestSuiteInfo; 35 import com.android.tradefed.testtype.suite.params.ModuleParameters; 36 import com.android.tradefed.util.FileUtil; 37 38 import com.google.common.base.Strings; 39 40 import org.junit.Assert; 41 import org.junit.Test; 42 import org.junit.runner.RunWith; 43 import org.junit.runners.JUnit4; 44 45 import java.io.File; 46 import java.util.ArrayList; 47 import java.util.Arrays; 48 import java.util.HashMap; 49 import java.util.HashSet; 50 import java.util.List; 51 import java.util.Map; 52 import java.util.Set; 53 54 /** 55 * Test CTS specific config requirements. 56 */ 57 @RunWith(JUnit4.class) 58 public class CtsConfigLoadingTest { 59 60 private static final String METADATA_COMPONENT = "component"; 61 private static final Set<String> KNOWN_COMPONENTS = 62 new HashSet<>( 63 Arrays.asList( 64 // modifications to the list below must be reviewed 65 "adservices", 66 "art", 67 "auth", 68 "auto", 69 "autofill", 70 "backup", 71 "bionic", 72 "bluetooth", 73 "camera", 74 "contentcapture", 75 "deviceinfo", 76 "deqp", 77 "devtools", 78 "framework", 79 "graphics", 80 "hdmi", 81 "inputmethod", 82 "libcore", 83 "location", 84 "media", 85 "metrics", 86 "misc", 87 "mocking", 88 "networking", 89 "neuralnetworks", 90 "nfc", 91 "packagemanager", 92 "permissions", 93 "print", 94 "renderscript", 95 "security", 96 "statsd", 97 "systems", 98 "sysui", 99 "telecom", 100 "threadnetwork", 101 "tv", 102 "uitoolkit", 103 "uwb", 104 "vr", 105 "webview", 106 "wifi")); 107 private static final Set<String> KNOWN_MISC_MODULES = 108 new HashSet<>( 109 Arrays.asList( 110 // Modifications to the list below must be approved by someone in 111 // test/suite_harness/OWNERS. 112 "CtsSliceTestCases.config", 113 "CtsSampleDeviceTestCases.config", 114 "CtsUsbTests.config", 115 "CtsGpuToolsHostTestCases.config", 116 "CtsEdiHostTestCases.config", 117 "CtsClassLoaderFactoryPathClassLoaderTestCases.config", 118 "CtsSampleHostTestCases.config", 119 "CtsHardwareTestCases.config", 120 "CtsAndroidAppTestCases.config", 121 "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config", 122 "CtsAppComponentFactoryTestCases.config", 123 "CtsSeccompHostTestCases.config")); 124 125 126 /** 127 * Families of module parameterization that MUST be specified explicitly in the module 128 * AndroidTest.xml. 129 */ 130 private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>(); 131 132 static { 133 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY); 134 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY); 135 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.SECONDARY_USER_FAMILY); 136 } 137 138 /** 139 * AllowList to start enforcing metadata on modules. No additional entry will be allowed! This 140 * is meant to burn down the remaining modules definition. 141 */ 142 private static final Set<String> ALLOWLIST_MODULE_PARAMETERS = new HashSet<>(); 143 144 static { 145 } 146 147 /** 148 * Test that configuration shipped in Tradefed can be parsed. 149 * -> Exclude deprecated ApkInstaller. 150 * -> Check if host-side tests are non empty. 151 */ 152 @Test testConfigurationLoad()153 public void testConfigurationLoad() throws Exception { 154 String rootVar = String.format("%s_ROOT", getSuiteName().toUpperCase()); 155 String suiteRoot = System.getProperty(rootVar); 156 if (Strings.isNullOrEmpty(suiteRoot)) { 157 fail(String.format("Should run within a suite context: %s doesn't exist", rootVar)); 158 } 159 File testcases = 160 new File( 161 suiteRoot, 162 String.format("/android-%s/testcases/", getSuiteName().toLowerCase())); 163 if (!testcases.exists()) { 164 fail(String.format("%s does not exists", testcases)); 165 return; 166 } 167 Set<File> listConfigs = FileUtil.findFilesObject(testcases, ".*\\.config"); 168 assertTrue(listConfigs.size() > 0); 169 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 170 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 171 stubFolder.setRootDir(new File(suiteRoot)); 172 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, getSuiteName().toUpperCase()); 173 stubFolder.addBuildAttribute("ROOT_DIR", suiteRoot); 174 TestInformation stubTestInfo = TestInformation.newBuilder() 175 .setInvocationContext(new InvocationContext()).build(); 176 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(suiteRoot)); 177 178 List<String> missingMandatoryParameters = new ArrayList<>(); 179 // We expect to be able to load every single config in testcases/ 180 for (File config : listConfigs) { 181 IConfiguration c = ConfigurationFactory.getInstance() 182 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 183 184 ConfigurationDescriptor cd = c.getConfigurationDescription(); 185 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 186 List<String> component = cd.getMetaData(METADATA_COMPONENT); 187 Assert.assertNotNull(String.format("Missing module metadata field \"component\", " 188 + "please add the following line to your AndroidTest.xml:\n" 189 + "<option name=\"config-descriptor:metadata\" key=\"component\" " 190 + "value=\"...\" />\nwhere \"value\" must be one of: %s\n" 191 + "config: %s", KNOWN_COMPONENTS, config), 192 component); 193 Assert.assertEquals(String.format("Module config contains more than one \"component\" " 194 + "metadata field: %s\nconfig: %s", component, config), 195 1, component.size()); 196 String cmp = component.get(0); 197 Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata " 198 + "field \"%s\", supported ones are: %s\nconfig: %s", 199 cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp)); 200 201 if ("misc".equals(cmp)) { 202 String configFileName = config.getName(); 203 Assert.assertTrue( 204 String.format( 205 "Adding new module %s to \"misc\" component is restricted, " 206 + "please pick a component that your module fits in", 207 configFileName), 208 KNOWN_MISC_MODULES.contains(configFileName)); 209 } 210 211 // Check that specified parameters are expected 212 boolean res = 213 checkModuleParameters( 214 config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 215 if (!res) { 216 missingMandatoryParameters.add(config.getName()); 217 } 218 219 String suiteName = getSuiteName().toLowerCase(); 220 // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" /> 221 Assert.assertTrue( 222 String.format( 223 "Module config %s does not contains " 224 + "'<option name=\"test-suite-tag\" value=\"%s\" />'", 225 config.getName(), suiteName), 226 cd.getSuiteTags().contains(suiteName)); 227 // Prevent any RootTargetPreparer in CTS as it's not expected to work 228 // https://source.android.com/docs/compatibility/cts/development#writing-cts-tests 229 for (IDeviceConfiguration dConfig : c.getDeviceConfig()) { 230 // Ensure we do not actively apply root in CTS 231 for (ITargetPreparer prep : dConfig.getTargetPreparers()) { 232 if (prep.getClass().isAssignableFrom(RootTargetPreparer.class) && 233 ((RootTargetPreparer) prep).shouldForceRoot() 234 && ((RootTargetPreparer) prep).shouldThrowOnFailure()) { 235 throw new ConfigurationException( 236 String.format("%s: Usage of root in CTS is forbidden.", config)); 237 } 238 } 239 } 240 241 // Ensure options have been set 242 c.validateOptions(); 243 } 244 245 // Exempt the allow list 246 missingMandatoryParameters.removeAll(ALLOWLIST_MODULE_PARAMETERS); 247 // Ensure the mandatory fields are filled 248 if (!missingMandatoryParameters.isEmpty()) { 249 String msg = 250 String.format( 251 "The following %s modules are missing some of the mandatory " 252 + "parameters [instant_app, not_instant_app, " 253 + "multi_abi, not_multi_abi, " 254 + "secondary_user, not_secondary_user]: '%s'", 255 missingMandatoryParameters.size(), missingMandatoryParameters); 256 throw new ConfigurationException(msg); 257 } 258 } 259 260 /** Test that all parameter metadata can be resolved. */ checkModuleParameters(String configName, List<String> parameters)261 private boolean checkModuleParameters(String configName, List<String> parameters) 262 throws ConfigurationException { 263 if (parameters == null) { 264 return false; 265 } 266 Map<String, Boolean> families = createFamilyCheckMap(); 267 for (String param : parameters) { 268 try { 269 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase()); 270 if (families.containsKey(p.getFamily())) { 271 families.put(p.getFamily(), true); 272 } 273 } catch (IllegalArgumentException e) { 274 throw new ConfigurationException( 275 String.format("Config: %s includes an unknown parameter '%s'.", 276 configName, param)); 277 } 278 } 279 if (families.containsValue(false)) { 280 return false; 281 } 282 return true; 283 } 284 createFamilyCheckMap()285 private Map<String, Boolean> createFamilyCheckMap() { 286 Map<String, Boolean> families = new HashMap<>(); 287 for (String family : MANDATORY_PARAMETERS_FAMILY) { 288 families.put(family, false); 289 } 290 return families; 291 } 292 getSuiteName()293 private String getSuiteName() { 294 return TestSuiteInfo.getInstance().getName(); 295 } 296 } 297