1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.loading; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.tradefed.build.FolderBuildInfo; 23 import com.android.tradefed.config.ConfigurationDescriptor; 24 import com.android.tradefed.config.ConfigurationException; 25 import com.android.tradefed.config.ConfigurationFactory; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 28 import com.android.tradefed.invoker.InvocationContext; 29 import com.android.tradefed.invoker.TestInformation; 30 import com.android.tradefed.testtype.suite.ITestSuite; 31 import com.android.tradefed.testtype.suite.TestSuiteInfo; 32 import com.android.tradefed.testtype.suite.params.ModuleParameters; 33 import com.android.tradefed.util.FileUtil; 34 35 import com.google.common.base.Strings; 36 37 import org.junit.Assert; 38 import org.junit.Test; 39 import org.junit.runner.RunWith; 40 import org.junit.runners.JUnit4; 41 42 import java.io.File; 43 import java.util.ArrayList; 44 import java.util.Arrays; 45 import java.util.HashMap; 46 import java.util.HashSet; 47 import java.util.List; 48 import java.util.Map; 49 import java.util.Set; 50 51 /** 52 * Test CTS specific config requirements. 53 */ 54 @RunWith(JUnit4.class) 55 public class CtsConfigLoadingTest { 56 57 private static final String METADATA_COMPONENT = "component"; 58 private static final Set<String> KNOWN_COMPONENTS = 59 new HashSet<>( 60 Arrays.asList( 61 // modifications to the list below must be reviewed 62 "abuse", 63 "adservices", 64 "art", 65 "auth", 66 "auto", 67 "autofill", 68 "backup", 69 "bionic", 70 "bluetooth", 71 "camera", 72 "contentcapture", 73 "deviceinfo", 74 "deqp", 75 "devtools", 76 "framework", 77 "graphics", 78 "hdmi", 79 "inputmethod", 80 "libcore", 81 "libnativehelper", 82 "location", 83 "media", 84 "metrics", 85 "misc", 86 "mocking", 87 "networking", 88 "neuralnetworks", 89 "packagemanager", 90 "permissions", 91 "print", 92 "renderscript", 93 "security", 94 "statsd", 95 "systems", 96 "sysui", 97 "telecom", 98 "tv", 99 "uitoolkit", 100 "uwb", 101 "vr", 102 "webview", 103 "wifi")); 104 private static final Set<String> KNOWN_MISC_MODULES = 105 new HashSet<>( 106 Arrays.asList( 107 // Modifications to the list below must be approved by someone in 108 // test/suite_harness/OWNERS. 109 "CtsSliceTestCases.config", 110 "CtsSampleDeviceTestCases.config", 111 "CtsUsbTests.config", 112 "CtsGpuToolsHostTestCases.config", 113 "CtsEdiHostTestCases.config", 114 "CtsClassLoaderFactoryPathClassLoaderTestCases.config", 115 "CtsSampleHostTestCases.config", 116 "CtsHardwareTestCases.config", 117 "CtsAndroidAppTestCases.config", 118 "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config", 119 "CtsAppComponentFactoryTestCases.config", 120 "CtsSeccompHostTestCases.config")); 121 122 123 /** 124 * Families of module parameterization that MUST be specified explicitly in the module 125 * AndroidTest.xml. 126 */ 127 private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>(); 128 129 static { 130 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY); 131 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY); 132 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.SECONDARY_USER_FAMILY); 133 } 134 135 /** 136 * AllowList to start enforcing metadata on modules. No additional entry will be allowed! This 137 * is meant to burn down the remaining modules definition. 138 */ 139 private static final Set<String> ALLOWLIST_MODULE_PARAMETERS = new HashSet<>(); 140 141 static { 142 } 143 144 /** 145 * Test that configuration shipped in Tradefed can be parsed. 146 * -> Exclude deprecated ApkInstaller. 147 * -> Check if host-side tests are non empty. 148 */ 149 @Test testConfigurationLoad()150 public void testConfigurationLoad() throws Exception { 151 String rootVar = String.format("%s_ROOT", getSuiteName().toUpperCase()); 152 String suiteRoot = System.getProperty(rootVar); 153 if (Strings.isNullOrEmpty(suiteRoot)) { 154 fail(String.format("Should run within a suite context: %s doesn't exist", rootVar)); 155 } 156 File testcases = new File(suiteRoot, String.format("/android-%s/testcases/", getSuiteName().toLowerCase())); 157 if (!testcases.exists()) { 158 fail(String.format("%s does not exists", testcases)); 159 return; 160 } 161 Set<File> listConfigs = FileUtil.findFilesObject(testcases, ".*\\.config"); 162 assertTrue(listConfigs.size() > 0); 163 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 164 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 165 stubFolder.setRootDir(new File(suiteRoot)); 166 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, getSuiteName().toUpperCase()); 167 stubFolder.addBuildAttribute("ROOT_DIR", suiteRoot); 168 TestInformation stubTestInfo = TestInformation.newBuilder() 169 .setInvocationContext(new InvocationContext()).build(); 170 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(suiteRoot)); 171 172 List<String> missingMandatoryParameters = new ArrayList<>(); 173 // We expect to be able to load every single config in testcases/ 174 for (File config : listConfigs) { 175 IConfiguration c = ConfigurationFactory.getInstance() 176 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 177 178 ConfigurationDescriptor cd = c.getConfigurationDescription(); 179 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 180 List<String> component = cd.getMetaData(METADATA_COMPONENT); 181 Assert.assertNotNull(String.format("Missing module metadata field \"component\", " 182 + "please add the following line to your AndroidTest.xml:\n" 183 + "<option name=\"config-descriptor:metadata\" key=\"component\" " 184 + "value=\"...\" />\nwhere \"value\" must be one of: %s\n" 185 + "config: %s", KNOWN_COMPONENTS, config), 186 component); 187 Assert.assertEquals(String.format("Module config contains more than one \"component\" " 188 + "metadata field: %s\nconfig: %s", component, config), 189 1, component.size()); 190 String cmp = component.get(0); 191 Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata " 192 + "field \"%s\", supported ones are: %s\nconfig: %s", 193 cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp)); 194 195 if ("misc".equals(cmp)) { 196 String configFileName = config.getName(); 197 Assert.assertTrue( 198 String.format( 199 "Adding new module %s to \"misc\" component is restricted, " 200 + "please pick a component that your module fits in", 201 configFileName), 202 KNOWN_MISC_MODULES.contains(configFileName)); 203 } 204 205 // Check that specified parameters are expected 206 boolean res = 207 checkModuleParameters( 208 config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 209 if (!res) { 210 missingMandatoryParameters.add(config.getName()); 211 } 212 213 String suiteName = getSuiteName().toLowerCase(); 214 // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" /> 215 Assert.assertTrue(String.format( 216 "Module config %s does not contains " 217 + "'<option name=\"test-suite-tag\" value=\"%s\" />'", config.getName(), suiteName), 218 cd.getSuiteTags().contains(suiteName)); 219 220 // Ensure options have been set 221 c.validateOptions(); 222 } 223 224 // Exempt the allow list 225 missingMandatoryParameters.removeAll(ALLOWLIST_MODULE_PARAMETERS); 226 // Ensure the mandatory fields are filled 227 if (!missingMandatoryParameters.isEmpty()) { 228 String msg = 229 String.format( 230 "The following %s modules are missing some of the mandatory " 231 + "parameters [instant_app, not_instant_app, " 232 + "multi_abi, not_multi_abi, " 233 + "secondary_user, not_secondary_user]: '%s'", 234 missingMandatoryParameters.size(), missingMandatoryParameters); 235 throw new ConfigurationException(msg); 236 } 237 } 238 239 /** Test that all parameter metadata can be resolved. */ checkModuleParameters(String configName, List<String> parameters)240 private boolean checkModuleParameters(String configName, List<String> parameters) 241 throws ConfigurationException { 242 if (parameters == null) { 243 return false; 244 } 245 Map<String, Boolean> families = createFamilyCheckMap(); 246 for (String param : parameters) { 247 try { 248 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase()); 249 if (families.containsKey(p.getFamily())) { 250 families.put(p.getFamily(), true); 251 } 252 } catch (IllegalArgumentException e) { 253 throw new ConfigurationException( 254 String.format("Config: %s includes an unknown parameter '%s'.", 255 configName, param)); 256 } 257 } 258 if (families.containsValue(false)) { 259 return false; 260 } 261 return true; 262 } 263 createFamilyCheckMap()264 private Map<String, Boolean> createFamilyCheckMap() { 265 Map<String, Boolean> families = new HashMap<>(); 266 for (String family : MANDATORY_PARAMETERS_FAMILY) { 267 families.put(family, false); 268 } 269 return families; 270 } 271 getSuiteName()272 private String getSuiteName() { 273 return TestSuiteInfo.getInstance().getName(); 274 } 275 } 276