1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.loading; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.compatibility.common.tradefed.targetprep.ApkInstaller; 23 import com.android.compatibility.common.tradefed.targetprep.PreconditionPreparer; 24 import com.android.compatibility.common.tradefed.testtype.JarHostTest; 25 import com.android.tradefed.build.FolderBuildInfo; 26 import com.android.tradefed.config.ConfigurationDescriptor; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.ConfigurationFactory; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.config.IDeviceConfiguration; 31 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 32 import com.android.tradefed.invoker.InvocationContext; 33 import com.android.tradefed.invoker.TestInformation; 34 import com.android.tradefed.invoker.shard.token.TokenProperty; 35 import com.android.tradefed.targetprep.DeviceSetup; 36 import com.android.tradefed.targetprep.ITargetPreparer; 37 import com.android.tradefed.targetprep.PythonVirtualenvPreparer; 38 import com.android.tradefed.testtype.AndroidJUnitTest; 39 import com.android.tradefed.testtype.GTest; 40 import com.android.tradefed.testtype.HostTest; 41 import com.android.tradefed.testtype.IRemoteTest; 42 import com.android.tradefed.testtype.ITestFilterReceiver; 43 import com.android.tradefed.testtype.suite.ITestSuite; 44 import com.android.tradefed.testtype.suite.TestSuiteInfo; 45 import com.android.tradefed.util.FileUtil; 46 47 import com.google.common.base.Strings; 48 49 import org.junit.Assert; 50 import org.junit.Test; 51 import org.junit.runner.RunWith; 52 import org.junit.runners.JUnit4; 53 54 import java.io.File; 55 import java.io.IOException; 56 import java.util.Arrays; 57 import java.util.HashSet; 58 import java.util.List; 59 import java.util.Set; 60 import java.util.regex.Pattern; 61 62 /** 63 * Test that configuration in *TS can load and have expected properties. 64 */ 65 @RunWith(JUnit4.class) 66 public class CommonConfigLoadingTest { 67 68 private static final Pattern TODO_BUG_PATTERN = Pattern.compile(".*TODO\\(b/[0-9]+\\).*", Pattern.DOTALL); 69 70 /** 71 * List of the officially supported runners in CTS, they meet all the interfaces criteria as 72 * well as support sharding very well. Any new addition should go through a review. 73 */ 74 private static final Set<String> SUPPORTED_SUITE_TEST_TYPE = new HashSet<>(Arrays.asList( 75 // Suite runners 76 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 77 "com.android.compatibility.testtype.DalvikTest", 78 "com.android.compatibility.testtype.LibcoreTest", 79 "com.drawelements.deqp.runner.DeqpTestRunner", 80 // Tradefed runners 81 "com.android.tradefed.testtype.AndroidJUnitTest", 82 "com.android.tradefed.testtype.ArtRunTest", 83 "com.android.tradefed.testtype.HostTest", 84 "com.android.tradefed.testtype.GTest", 85 "com.android.tradefed.testtype.mobly.MoblyBinaryHostTest", 86 "com.android.tradefed.testtype.pandora.PtsBotTest", 87 // VTS specific runners 88 "com.android.tradefed.testtype.binary.KernelTargetTest", 89 "com.android.tradefed.testtype.python.PythonBinaryHostTest", 90 "com.android.tradefed.testtype.binary.ExecutableTargetTest", 91 "com.android.tradefed.testtype.binary.ExecutableHostTest", 92 "com.android.tradefed.testtype.rust.RustBinaryTest" 93 )); 94 95 /** 96 * In Most cases we impose the usage of the AndroidJUnitRunner because it supports all the 97 * features required (filtering, sharding, etc.). We do not typically expect people to need a 98 * different runner. 99 */ 100 private static final Set<String> ALLOWED_INSTRUMENTATION_RUNNER_NAME = new HashSet<>(); 101 static { 102 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("android.support.test.runner.AndroidJUnitRunner"); 103 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("androidx.test.runner.AndroidJUnitRunner"); 104 } 105 private static final Set<String> RUNNER_EXCEPTION = new HashSet<>(); 106 static { 107 // Used for a bunch of system-api cts tests 108 RUNNER_EXCEPTION.add("repackaged.android.test.InstrumentationTestRunner"); 109 // Used by a UiRendering scenario where an activity is persisted between tests 110 RUNNER_EXCEPTION.add("android.uirendering.cts.runner.UiRenderingRunner"); 111 // Used to avoid crashing runner on -eng build due to Log.wtf() - b/216648699 112 RUNNER_EXCEPTION.add("com.android.server.uwb.CustomTestRunner"); 113 RUNNER_EXCEPTION.add("com.android.server.wifi.CustomTestRunner"); 114 // HealthConnect APK use Hilt for dependency injection. For test setup it needs 115 // to replace the main Application class with Test Application so Hilt can swap 116 // dependencies for testing. 117 RUNNER_EXCEPTION.add("com.android.healthconnect.controller.tests.HiltTestRunner"); 118 } 119 120 /** 121 * Test that configuration shipped in Tradefed can be parsed. 122 * -> Exclude deprecated ApkInstaller. 123 * -> Check if host-side tests are non empty. 124 */ 125 @Test testConfigurationLoad()126 public void testConfigurationLoad() throws Exception { 127 String rootVar = String.format("%s_ROOT", getSuiteName().toUpperCase()); 128 String suiteRoot = System.getProperty(rootVar); 129 if (Strings.isNullOrEmpty(suiteRoot)) { 130 fail(String.format("Should run within a suite context: %s doesn't exist", rootVar)); 131 } 132 File testcases = new File(suiteRoot, String.format("/android-%s/testcases/", getSuiteName().toLowerCase())); 133 if (!testcases.exists()) { 134 fail(String.format("%s does not exist", testcases)); 135 return; 136 } 137 Set<File> listConfigs = FileUtil.findFilesObject(testcases, ".*\\.config"); 138 assertTrue(listConfigs.size() > 0); 139 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 140 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 141 stubFolder.setRootDir(new File(suiteRoot)); 142 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, getSuiteName().toUpperCase()); 143 stubFolder.addBuildAttribute("ROOT_DIR", suiteRoot); 144 TestInformation stubTestInfo = TestInformation.newBuilder() 145 .setInvocationContext(new InvocationContext()).build(); 146 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(suiteRoot)); 147 148 // We expect to be able to load every single config in testcases/ 149 for (File config : listConfigs) { 150 IConfiguration c = ConfigurationFactory.getInstance() 151 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 152 if (c.getDeviceConfig().size() > 2) { 153 throw new ConfigurationException(String.format("%s declares more than 2 devices.", config)); 154 } 155 int deviceCount = 0; 156 for (IDeviceConfiguration dConfig : c.getDeviceConfig()) { 157 // Ensure the deprecated ApkInstaller is not used anymore. 158 for (ITargetPreparer prep : dConfig.getTargetPreparers()) { 159 if (prep.getClass().isAssignableFrom(ApkInstaller.class)) { 160 throw new ConfigurationException( 161 String.format("%s: Use com.android.tradefed.targetprep.suite." 162 + "SuiteApkInstaller instead of com.android.compatibility." 163 + "common.tradefed.targetprep.ApkInstaller, options will be " 164 + "the same.", config)); 165 } 166 if (prep.getClass().isAssignableFrom(PreconditionPreparer.class)) { 167 throw new ConfigurationException( 168 String.format( 169 "%s: includes a PreconditionPreparer (%s) which is not " 170 + "allowed in modules.", 171 config.getName(), prep.getClass())); 172 } 173 if (prep.getClass().isAssignableFrom(DeviceSetup.class)) { 174 DeviceSetup deviceSetup = (DeviceSetup) prep; 175 if (!deviceSetup.isForceSkipSystemProps()) { 176 throw new ConfigurationException( 177 String.format("%s: %s needs to be configured with " 178 + "<option name=\"force-skip-system-props\" " 179 + "value=\"true\" /> in *TS.", 180 config.getName(), prep.getClass())); 181 } 182 } 183 if (prep.getClass().isAssignableFrom(PythonVirtualenvPreparer.class)) { 184 // Ensure each modules has a tracking bug to be imported. 185 checkPythonModules(config, deviceCount); 186 } 187 } 188 deviceCount++; 189 } 190 // We can ensure that Host side tests are not empty. 191 for (IRemoteTest test : c.getTests()) { 192 // Check that all the tests runners are well supported. 193 if (!SUPPORTED_SUITE_TEST_TYPE.contains(test.getClass().getCanonicalName())) { 194 throw new ConfigurationException( 195 String.format( 196 "testtype %s is not officially supported by *TS. " 197 + "The supported ones are: %s", 198 test.getClass().getCanonicalName(), SUPPORTED_SUITE_TEST_TYPE)); 199 } 200 if (test instanceof HostTest) { 201 HostTest hostTest = (HostTest) test; 202 // We inject a made up folder so that it can find the tests. 203 hostTest.setBuild(stubFolder); 204 hostTest.setTestInformation(stubTestInfo); 205 int testCount = hostTest.countTestCases(); 206 if (testCount == 0) { 207 throw new ConfigurationException( 208 String.format("%s: %s reports 0 test cases.", 209 config.getName(), test)); 210 } 211 } 212 if (test instanceof GTest) { 213 if (((GTest) test).isRebootBeforeTestEnabled()) { 214 throw new ConfigurationException(String.format( 215 "%s: instead of reboot-before-test use a RebootTargetPreparer " 216 + "which is more optimized during sharding.", config.getName())); 217 } 218 } 219 // Tests are expected to implement that interface. 220 if (!(test instanceof ITestFilterReceiver)) { 221 throw new IllegalArgumentException(String.format( 222 "Test in module %s must implement ITestFilterReceiver.", 223 config.getName())); 224 } 225 // Ensure that the device runner is the AJUR one if explicitly specified. 226 if (test instanceof AndroidJUnitTest) { 227 AndroidJUnitTest instru = (AndroidJUnitTest) test; 228 if (instru.getRunnerName() != null && 229 !ALLOWED_INSTRUMENTATION_RUNNER_NAME.contains(instru.getRunnerName())) { 230 // Some runner are exempt 231 if (!RUNNER_EXCEPTION.contains(instru.getRunnerName())) { 232 throw new ConfigurationException( 233 String.format("%s: uses '%s' instead of on of '%s' that are " 234 + "expected", config.getName(), instru.getRunnerName(), 235 ALLOWED_INSTRUMENTATION_RUNNER_NAME)); 236 } 237 } 238 } 239 } 240 241 ConfigurationDescriptor cd = c.getConfigurationDescription(); 242 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 243 244 // Check that specified tokens are expected 245 checkTokens(config.getName(), cd.getMetaData(ITestSuite.TOKEN_KEY)); 246 247 // Check not-shardable: JarHostTest cannot create empty shards so it should never need 248 // to be not-shardable. 249 if (cd.isNotShardable()) { 250 for (IRemoteTest test : c.getTests()) { 251 if (test.getClass().isAssignableFrom(JarHostTest.class)) { 252 throw new ConfigurationException( 253 String.format("config: %s. JarHostTest does not need the " 254 + "not-shardable option.", config.getName())); 255 } 256 } 257 } 258 // Ensure options have been set 259 c.validateOptions(); 260 } 261 } 262 263 /** Test that all tokens can be resolved. */ checkTokens(String configName, List<String> tokens)264 private void checkTokens(String configName, List<String> tokens) throws ConfigurationException { 265 if (tokens == null) { 266 return; 267 } 268 for (String token : tokens) { 269 try { 270 TokenProperty.valueOf(token.toUpperCase()); 271 } catch (IllegalArgumentException e) { 272 throw new ConfigurationException( 273 String.format( 274 "Config: %s includes an unknown token '%s'.", configName, token)); 275 } 276 } 277 } 278 279 /** 280 * For each usage of python virtualenv preparer, make sure we have tracking bugs to import as 281 * source the python libs. 282 */ checkPythonModules(File config, int deviceCount)283 private void checkPythonModules(File config, int deviceCount) 284 throws IOException, ConfigurationException { 285 if (deviceCount != 0) { 286 throw new ConfigurationException( 287 String.format("%s: PythonVirtualenvPreparer should only be declared for " 288 + "the first <device> tag in the config", config.getName())); 289 } 290 if (!TODO_BUG_PATTERN.matcher(FileUtil.readStringFromFile(config)).matches()) { 291 throw new ConfigurationException( 292 String.format("%s: Contains some virtualenv python lib usage but no " 293 + "tracking bug to import them as source.", config.getName())); 294 } 295 } 296 getSuiteName()297 private String getSuiteName() { 298 return TestSuiteInfo.getInstance().getName(); 299 } 300 } 301