1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.config.ConfigurationDescriptor; 20 import com.android.tradefed.config.ConfigurationException; 21 import com.android.tradefed.config.ConfigurationFactory; 22 import com.android.tradefed.config.ConfigurationUtil; 23 import com.android.tradefed.config.IConfiguration; 24 import com.android.tradefed.config.IConfigurationFactory; 25 import com.android.tradefed.config.IDeviceConfiguration; 26 import com.android.tradefed.config.OptionDef; 27 import com.android.tradefed.config.OptionSetter; 28 import com.android.tradefed.device.DeviceFoldableState; 29 import com.android.tradefed.device.metric.IMetricCollector; 30 import com.android.tradefed.error.HarnessRuntimeException; 31 import com.android.tradefed.invoker.IInvocationContext; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.postprocessor.IPostProcessor; 34 import com.android.tradefed.result.error.InfraErrorIdentifier; 35 import com.android.tradefed.targetprep.ITargetPreparer; 36 import com.android.tradefed.testtype.IAbi; 37 import com.android.tradefed.testtype.IAbiReceiver; 38 import com.android.tradefed.testtype.IRemoteTest; 39 import com.android.tradefed.testtype.ITestFileFilterReceiver; 40 import com.android.tradefed.testtype.ITestFilterReceiver; 41 import com.android.tradefed.testtype.suite.params.FoldableExpandingHandler; 42 import com.android.tradefed.testtype.suite.params.IModuleParameterHandler; 43 import com.android.tradefed.testtype.suite.params.MainlineModuleHandler; 44 import com.android.tradefed.testtype.suite.params.ModuleParameters; 45 import com.android.tradefed.testtype.suite.params.ModuleParametersHelper; 46 import com.android.tradefed.testtype.suite.params.NegativeHandler; 47 import com.android.tradefed.testtype.suite.params.NotMultiAbiHandler; 48 import com.android.tradefed.util.AbiUtils; 49 import com.android.tradefed.util.FileUtil; 50 51 import com.google.common.base.Strings; 52 import com.google.common.net.UrlEscapers; 53 54 import java.io.File; 55 import java.io.FilenameFilter; 56 import java.io.IOException; 57 import java.io.PrintWriter; 58 import java.util.ArrayList; 59 import java.util.Arrays; 60 import java.util.Collection; 61 import java.util.Collections; 62 import java.util.HashMap; 63 import java.util.HashSet; 64 import java.util.LinkedHashMap; 65 import java.util.LinkedHashSet; 66 import java.util.List; 67 import java.util.Map; 68 import java.util.Map.Entry; 69 import java.util.Set; 70 import java.util.regex.Matcher; 71 import java.util.regex.Pattern; 72 import java.util.stream.Collectors; 73 74 /** 75 * Retrieves Compatibility test module definitions from the repository. TODO: Add the expansion of 76 * suite when loading a module. 77 */ 78 public class SuiteModuleLoader { 79 80 public static final String CONFIG_EXT = ".config"; 81 private Map<String, List<OptionDef>> mTestOrPreparerOptions = new HashMap<>(); 82 private Map<String, List<OptionDef>> mModuleOptions = new HashMap<>(); 83 private boolean mIncludeAll; 84 private Map<String, LinkedHashSet<SuiteTestFilter>> mIncludeFilters = new HashMap<>(); 85 private Map<String, LinkedHashSet<SuiteTestFilter>> mExcludeFilters = new HashMap<>(); 86 private IConfigurationFactory mConfigFactory = ConfigurationFactory.getInstance(); 87 private IInvocationContext mContext; 88 89 private boolean mAllowParameterizedModules = false; 90 private boolean mAllowMainlineParameterizedModules = false; 91 private boolean mOptimizeMainlineTest = false; 92 private boolean mIgnoreNonPreloadedMainlineModule = false; 93 private boolean mAllowOptionalParameterizedModules = false; 94 private boolean mLoadConfigsWithIncludeFilters = false; 95 private ModuleParameters mForcedModuleParameter = null; 96 private Set<ModuleParameters> mExcludedModuleParameters = new HashSet<>(); 97 private Set<DeviceFoldableState> mFoldableStates = new LinkedHashSet<>(); 98 // Check the mainline parameter configured in a test config must end with .apk, .apks, or .apex. 99 private static final Set<String> MAINLINE_PARAMETERS_TO_VALIDATE = 100 new HashSet<>(Arrays.asList(".apk", ".apks", ".apex")); 101 102 /** 103 * Ctor for the SuiteModuleLoader. 104 * 105 * @param includeFilters The formatted and parsed include filters. 106 * @param excludeFilters The formatted and parsed exclude filters. 107 * @param testArgs the list of test ({@link IRemoteTest}) arguments. 108 * @param moduleArgs the list of module arguments. 109 */ SuiteModuleLoader( Map<String, LinkedHashSet<SuiteTestFilter>> includeFilters, Map<String, LinkedHashSet<SuiteTestFilter>> excludeFilters, List<String> testArgs, List<String> moduleArgs)110 public SuiteModuleLoader( 111 Map<String, LinkedHashSet<SuiteTestFilter>> includeFilters, 112 Map<String, LinkedHashSet<SuiteTestFilter>> excludeFilters, 113 List<String> testArgs, 114 List<String> moduleArgs) { 115 mIncludeAll = includeFilters.isEmpty(); 116 mIncludeFilters = includeFilters; 117 mExcludeFilters = excludeFilters; 118 119 parseArgs(testArgs, mTestOrPreparerOptions); 120 parseArgs(moduleArgs, mModuleOptions); 121 } 122 setInvocationContext(IInvocationContext context)123 public final void setInvocationContext(IInvocationContext context) { 124 mContext = context; 125 } 126 127 /** Sets whether or not to allow parameterized modules. */ setParameterizedModules(boolean allowed)128 public final void setParameterizedModules(boolean allowed) { 129 mAllowParameterizedModules = allowed; 130 } 131 132 /** Sets whether or not to allow parameterized mainline modules. */ setMainlineParameterizedModules(boolean allowed)133 public final void setMainlineParameterizedModules(boolean allowed) { 134 mAllowMainlineParameterizedModules = allowed; 135 } 136 137 /** Sets whether or not to optimize mainline test. */ setOptimizeMainlineTest(boolean allowed)138 public final void setOptimizeMainlineTest(boolean allowed) { 139 mOptimizeMainlineTest = allowed; 140 } 141 142 /** Sets whether or not to ignore installing the module if it is not preloaded. */ setIgnoreNonPreloadedMainlineModule(boolean ignore)143 public final void setIgnoreNonPreloadedMainlineModule(boolean ignore) { 144 mIgnoreNonPreloadedMainlineModule = ignore; 145 } 146 147 /** Sets whether or not to allow optional parameterized modules. */ setOptionalParameterizedModules(boolean allowed)148 public final void setOptionalParameterizedModules(boolean allowed) { 149 mAllowOptionalParameterizedModules = allowed; 150 } 151 152 /** Sets whether or not to load test config based on the given include-filter. */ setLoadConfigsWithIncludeFilters(boolean allowed)153 public final void setLoadConfigsWithIncludeFilters(boolean allowed) { 154 mLoadConfigsWithIncludeFilters = allowed; 155 } 156 157 /** Sets the only {@link ModuleParameters} type that should be run. */ setModuleParameter(ModuleParameters param)158 public final void setModuleParameter(ModuleParameters param) { 159 mForcedModuleParameter = param; 160 } 161 162 /** Sets the set of {@link ModuleParameters} that should not be considered at all. */ setExcludedModuleParameters(Set<ModuleParameters> excludedParams)163 public final void setExcludedModuleParameters(Set<ModuleParameters> excludedParams) { 164 mExcludedModuleParameters = excludedParams; 165 } 166 167 /** Sets the set of {@link DeviceFoldableState} that should be run. */ setFoldableStates(Set<DeviceFoldableState> foldableStates)168 public final void setFoldableStates(Set<DeviceFoldableState> foldableStates) { 169 mFoldableStates = foldableStates; 170 } 171 172 /** Main loading of configurations, looking into the specified files */ loadConfigsFromSpecifiedPaths( List<File> listConfigFiles, Set<IAbi> abis, String suiteTag)173 public LinkedHashMap<String, IConfiguration> loadConfigsFromSpecifiedPaths( 174 List<File> listConfigFiles, Set<IAbi> abis, String suiteTag) { 175 LinkedHashMap<String, IConfiguration> toRun = new LinkedHashMap<>(); 176 for (File configFile : listConfigFiles) { 177 Map<String, IConfiguration> loadedConfigs = 178 loadOneConfig( 179 configFile.getName(), 180 configFile.getAbsolutePath(), 181 abis, 182 suiteTag); 183 // store the module dir path for each config 184 for (IConfiguration loadedConfig : loadedConfigs.values()) { 185 loadedConfig 186 .getConfigurationDescription() 187 .addMetadata( 188 ConfigurationDescriptor.MODULE_DIR_PATH_KEY, 189 configFile.getParentFile().getAbsolutePath()); 190 } 191 toRun.putAll(loadedConfigs); 192 } 193 return toRun; 194 } 195 196 /** Main loading of configurations, looking into a folder */ loadConfigsFromDirectory( List<File> testsDirs, Set<IAbi> abis, String suitePrefix, String suiteTag, List<String> patterns)197 public LinkedHashMap<String, IConfiguration> loadConfigsFromDirectory( 198 List<File> testsDirs, 199 Set<IAbi> abis, 200 String suitePrefix, 201 String suiteTag, 202 List<String> patterns) { 203 LinkedHashMap<String, IConfiguration> toRun = new LinkedHashMap<>(); 204 List<File> listConfigFiles = new ArrayList<>(); 205 listConfigFiles.addAll( 206 ConfigurationUtil.getConfigNamesFileFromDirs(suitePrefix, testsDirs, patterns)); 207 if (mLoadConfigsWithIncludeFilters && !mIncludeFilters.isEmpty()) { 208 CLog.i("Loading test configs based on the given include-filter."); 209 Set<String> filteredConfigNames = new HashSet<>(); 210 for (LinkedHashSet<SuiteTestFilter> entry : mIncludeFilters.values()) { 211 for (SuiteTestFilter file : entry) { 212 // Collect the test config name based on the given include filter. 213 filteredConfigNames.add(String.format("%s.config", file.getBaseName())); 214 } 215 } 216 // Filter the test configs out based on the collected test config names. 217 List<File> filteredConfigs = 218 listConfigFiles.stream() 219 .filter(f -> filteredConfigNames.contains(f.getName())) 220 .collect(Collectors.toList()); 221 listConfigFiles.clear(); 222 listConfigFiles.addAll(filteredConfigs); 223 } 224 // Ensure stable initial order of configurations. 225 Collections.sort(listConfigFiles); 226 toRun.putAll(loadConfigsFromSpecifiedPaths(listConfigFiles, abis, suiteTag)); 227 return toRun; 228 } 229 230 /** 231 * Main loading of configurations, looking into the resources on the classpath. (TF configs for 232 * example). 233 */ loadConfigsFromJars( Set<IAbi> abis, String suitePrefix, String suiteTag)234 public LinkedHashMap<String, IConfiguration> loadConfigsFromJars( 235 Set<IAbi> abis, String suitePrefix, String suiteTag) { 236 LinkedHashMap<String, IConfiguration> toRun = new LinkedHashMap<>(); 237 238 IConfigurationFactory configFactory = ConfigurationFactory.getInstance(); 239 List<String> configs = configFactory.getConfigList(suitePrefix, false); 240 // Sort configs to ensure they are always evaluated and added in the same order. 241 Collections.sort(configs); 242 toRun.putAll(loadTfConfigsFromSpecifiedPaths(configs, abis, suiteTag)); 243 return toRun; 244 } 245 246 /** Main loading of configurations, looking into the specified resources on the classpath. */ loadTfConfigsFromSpecifiedPaths( List<String> configs, Set<IAbi> abis, String suiteTag)247 public LinkedHashMap<String, IConfiguration> loadTfConfigsFromSpecifiedPaths( 248 List<String> configs, Set<IAbi> abis, String suiteTag) { 249 LinkedHashMap<String, IConfiguration> toRun = new LinkedHashMap<>(); 250 for (String configName : configs) { 251 toRun.putAll(loadOneConfig(configName, configName, abis, suiteTag)); 252 } 253 return toRun; 254 } 255 256 /** 257 * Pass the filters to the {@link IRemoteTest}. Default behavior is to ignore if the IRemoteTest 258 * does not implements {@link ITestFileFilterReceiver}. This can be overriden to create a more 259 * restrictive behavior. 260 * 261 * @param test The {@link IRemoteTest} that is being considered. 262 * @param abi The Abi we are currently working on. 263 * @param moduleId The id of the module (usually abi + module name). 264 * @param includeFilters The formatted and parsed include filters. 265 * @param excludeFilters The formatted and parsed exclude filters. 266 */ addFiltersToTest( IRemoteTest test, IAbi abi, String moduleId, Map<String, LinkedHashSet<SuiteTestFilter>> includeFilters, Map<String, LinkedHashSet<SuiteTestFilter>> excludeFilters)267 public void addFiltersToTest( 268 IRemoteTest test, 269 IAbi abi, 270 String moduleId, 271 Map<String, LinkedHashSet<SuiteTestFilter>> includeFilters, 272 Map<String, LinkedHashSet<SuiteTestFilter>> excludeFilters) { 273 274 if (!(test instanceof ITestFilterReceiver)) { 275 CLog.e( 276 "Test (%s) in module %s does not implement ITestFilterReceiver.", 277 test.getClass().getName(), moduleId); 278 return; 279 } 280 LinkedHashSet<SuiteTestFilter> mdIncludes = getFilterList(includeFilters, moduleId); 281 LinkedHashSet<SuiteTestFilter> mdExcludes = getFilterList(excludeFilters, moduleId); 282 if (!mdIncludes.isEmpty()) { 283 addTestIncludes((ITestFilterReceiver) test, mdIncludes, moduleId); 284 } 285 if (!mdExcludes.isEmpty()) { 286 addTestExcludes((ITestFilterReceiver) test, mdExcludes, moduleId); 287 } 288 } 289 290 /** 291 * Load a single config location (file or on TF classpath). It can results in several {@link 292 * IConfiguration}. If a single configuration get expanded in different ways. 293 * 294 * @param configName The actual config name only. (no path) 295 * @param configFullName The fully qualified config name. (with path, if any). 296 * @param abis The set of all abis that needs to run. 297 * @param suiteTag the Tag of the suite aimed to be run. 298 * @return A map of loaded configuration. 299 */ loadOneConfig( String configName, String configFullName, Set<IAbi> abis, String suiteTag)300 private LinkedHashMap<String, IConfiguration> loadOneConfig( 301 String configName, 302 String configFullName, 303 Set<IAbi> abis, 304 String suiteTag) { 305 LinkedHashMap<String, IConfiguration> toRun = new LinkedHashMap<>(); 306 final String name = configName.replace(CONFIG_EXT, ""); 307 final String[] pathArg = new String[] {configFullName}; 308 try { 309 boolean primaryAbi = true; 310 boolean shouldCreateMultiAbi = true; 311 // If a particular parameter was requested to be run, find it. 312 Set<IModuleParameterHandler> mForcedParameters = null; 313 Set<Class<?>> mForcedParameterClasses = null; 314 if (mForcedModuleParameter != null) { 315 mForcedParameters = new HashSet<>(); 316 Map<ModuleParameters, IModuleParameterHandler> moduleParameters = 317 ModuleParametersHelper.resolveParam( 318 mForcedModuleParameter, mAllowOptionalParameterizedModules); 319 mForcedParameterClasses = new HashSet<>(); 320 for (IModuleParameterHandler parameter : moduleParameters.values()) { 321 if (parameter instanceof FoldableExpandingHandler) { 322 for (IModuleParameterHandler fParam : 323 ((FoldableExpandingHandler) parameter) 324 .expandHandler(mFoldableStates)) { 325 mForcedParameterClasses.add(fParam.getClass()); 326 } 327 } else { 328 mForcedParameterClasses.add(parameter.getClass()); 329 } 330 } 331 } 332 333 // Invokes parser to process the test module config file 334 // Need to generate a different config for each ABI as we cannot guarantee the 335 // configs are idempotent. This however means we parse the same file multiple times 336 for (IAbi abi : abis) { 337 // Filter non-primary abi no matter what if not_multi_abi specified 338 if (!shouldCreateMultiAbi && !primaryAbi) { 339 continue; 340 } 341 String baseId = AbiUtils.createId(abi.getName(), name); 342 IConfiguration config = null; 343 try { 344 config = mConfigFactory.createConfigurationFromArgs(pathArg); 345 } catch (ConfigurationException e) { 346 // If the module should not have been running in the first place, give it a 347 // pass on the configuration failure. 348 if (!shouldRunModule(baseId)) { 349 primaryAbi = false; 350 // If the module should not run tests based on the state of filters, 351 // skip this name/abi combination. 352 continue; 353 } 354 throw e; 355 } 356 357 // If a suiteTag is used, we load with it. 358 if (!Strings.isNullOrEmpty(suiteTag) 359 && !config.getConfigurationDescription() 360 .getSuiteTags() 361 .contains(suiteTag)) { 362 // Do not print here, it could leave several hundred lines of logs. 363 continue; 364 } 365 366 boolean skipCreatingBaseConfig = false; 367 List<IModuleParameterHandler> params = null; 368 List<String> mainlineParams = new ArrayList<>(); 369 try { 370 params = getModuleParameters(name, config); 371 mainlineParams = getMainlineModuleParameters(config); 372 } catch (ConfigurationException e) { 373 // If the module should not have been running in the first place, give it a 374 // pass on the configuration failure. 375 if (!shouldRunModule(baseId)) { 376 primaryAbi = false; 377 // If the module should not run tests based on the state of filters, 378 // skip this name/abi combination. 379 continue; 380 } 381 throw e; 382 } 383 // Use the not_multi_abi metadata even if not in parameterized mode. 384 shouldCreateMultiAbi = shouldCreateMultiAbiForBase(params); 385 // Handle parameterized modules if enabled. 386 if (mAllowParameterizedModules) { 387 388 if (params.isEmpty() 389 && mForcedParameters != null 390 // If we have multiple forced parameters, NegativeHandler isn't a valid 391 // option 392 && !(mForcedParameters.size() != 1 393 || (mForcedParameters.iterator().next() 394 instanceof NegativeHandler))) { 395 // If the AndroidTest.xml doesn't specify any parameter but we forced a 396 // parameter like 'instant' to execute. In this case we don't create the 397 // standard module. 398 continue; 399 } 400 401 // If we find any parameterized combination. 402 for (IModuleParameterHandler param : params) { 403 if (param instanceof NegativeHandler) { 404 if (mForcedParameters != null 405 && !mForcedParameterClasses.contains(param.getClass())) { 406 skipCreatingBaseConfig = true; 407 } 408 continue; 409 } 410 if (mForcedParameters != null) { 411 // When a particular parameter is forced, only create it not the others 412 if (mForcedParameterClasses.contains(param.getClass())) { 413 skipCreatingBaseConfig = true; 414 } else { 415 continue; 416 } 417 } 418 // Only create primary abi of parameterized modules 419 if (!primaryAbi) { 420 continue; 421 } 422 String fullId = 423 String.format("%s[%s]", baseId, param.getParameterIdentifier()); 424 String nameWithParam = 425 String.format("%s[%s]", name, param.getParameterIdentifier()); 426 if (shouldRunParameterized( 427 baseId, fullId, nameWithParam, mForcedParameters)) { 428 IConfiguration paramConfig = 429 mConfigFactory.createConfigurationFromArgs(pathArg); 430 // Mark the parameter in the metadata 431 paramConfig 432 .getConfigurationDescription() 433 .addMetadata( 434 ConfigurationDescriptor.ACTIVE_PARAMETER_KEY, 435 param.getParameterIdentifier()); 436 param.addParameterSpecificConfig(paramConfig); 437 setUpConfig( 438 name, 439 nameWithParam, 440 baseId, 441 fullId, 442 paramConfig, 443 abi); 444 param.applySetup(paramConfig); 445 toRun.put(fullId, paramConfig); 446 } 447 } 448 } 449 450 if (mAllowMainlineParameterizedModules) { 451 // If no options defined in a test config, skip generating. 452 // TODO(easoncylee) This is still under discussion. 453 if (mainlineParams.isEmpty()) { 454 continue; 455 } 456 // If we find any parameterized combination for mainline modules. 457 for (String param : mainlineParams) { 458 String fullId = String.format("%s[%s]", baseId, param); 459 String nameWithParam = String.format("%s[%s]", name, param); 460 if (!shouldRunParameterized(baseId, fullId, nameWithParam, null)) { 461 continue; 462 } 463 // Create mainline handler for each defined mainline parameter. 464 MainlineModuleHandler handler = 465 new MainlineModuleHandler( 466 param, 467 abi, 468 mContext, 469 mOptimizeMainlineTest, 470 mIgnoreNonPreloadedMainlineModule); 471 skipCreatingBaseConfig = true; 472 IConfiguration paramConfig = 473 mConfigFactory.createConfigurationFromArgs(pathArg); 474 paramConfig 475 .getConfigurationDescription() 476 .addMetadata(ITestSuite.ACTIVE_MAINLINE_PARAMETER_KEY, param); 477 setUpConfig(name, nameWithParam, baseId, fullId, paramConfig, abi); 478 handler.applySetup(paramConfig); 479 toRun.put(fullId, paramConfig); 480 } 481 } 482 483 primaryAbi = false; 484 // If a parameterized form of the module was forced, we don't create the standard 485 // version of it. 486 if (skipCreatingBaseConfig) { 487 continue; 488 } 489 if (shouldRunModule(baseId)) { 490 // Always add the base regular configuration to the execution. 491 // Do not pass the nameWithParam in because it would cause the module args be 492 // injected into config twice if we pass nameWithParam using name. 493 setUpConfig(name, null, baseId, baseId, config, abi); 494 toRun.put(baseId, config); 495 } 496 } 497 } catch (ConfigurationException e) { 498 throw new HarnessRuntimeException( 499 String.format( 500 "Error parsing configuration: %s: '%s'", 501 configFullName, e.getMessage()), 502 e); 503 } 504 505 return toRun; 506 } 507 508 /** 509 * @return the {@link Set} of modules whose name contains the given pattern. 510 */ getModuleNamesMatching( File directory, String suitePrefix, String pattern)511 public static Set<File> getModuleNamesMatching( 512 File directory, String suitePrefix, String pattern) { 513 List<File> extraTestCasesDirs = Arrays.asList(directory); 514 List<String> patterns = new ArrayList<>(); 515 patterns.add(pattern); 516 Set<File> modules = 517 ConfigurationUtil.getConfigNamesFileFromDirs( 518 suitePrefix, extraTestCasesDirs, patterns); 519 return modules; 520 } 521 522 /** 523 * Utility method that allows to parse and create a structure with the option filters. 524 * 525 * @param stringFilters The original option filters format. 526 * @param filters The filters parsed from the string format. 527 * @param abis The Abis to consider in the filtering. 528 */ addFilters( Set<String> stringFilters, Map<String, LinkedHashSet<SuiteTestFilter>> filters, Set<IAbi> abis, Set<DeviceFoldableState> foldableStates)529 public static void addFilters( 530 Set<String> stringFilters, 531 Map<String, LinkedHashSet<SuiteTestFilter>> filters, 532 Set<IAbi> abis, 533 Set<DeviceFoldableState> foldableStates) { 534 for (String filterString : stringFilters) { 535 SuiteTestFilter parentFilter = SuiteTestFilter.createFrom(filterString); 536 List<SuiteTestFilter> expanded = expandFoldableFilters(parentFilter, foldableStates); 537 for (SuiteTestFilter filter : expanded) { 538 String abi = filter.getAbi(); 539 if (abi == null) { 540 for (IAbi a : abis) { 541 addFilter(a.getName(), filter, filters); 542 } 543 } else { 544 addFilter(abi, filter, filters); 545 } 546 } 547 } 548 } 549 expandFoldableFilters( SuiteTestFilter filter, Set<DeviceFoldableState> foldableStates)550 private static List<SuiteTestFilter> expandFoldableFilters( 551 SuiteTestFilter filter, Set<DeviceFoldableState> foldableStates) { 552 List<SuiteTestFilter> expandedFilters = new ArrayList<>(); 553 if (foldableStates == null || foldableStates.isEmpty()) { 554 expandedFilters.add(filter); 555 return expandedFilters; 556 } 557 if (!ModuleParameters.ALL_FOLDABLE_STATES.toString().equals(filter.getParameterName())) { 558 expandedFilters.add(filter); 559 return expandedFilters; 560 } 561 for (DeviceFoldableState state : foldableStates) { 562 String name = filter.getBaseName() + "[" + state.toString() + "]"; 563 expandedFilters.add( 564 new SuiteTestFilter( 565 filter.getShardIndex(), filter.getAbi(), name, filter.getTest())); 566 } 567 return expandedFilters; 568 } 569 addFilter( String abi, SuiteTestFilter filter, Map<String, LinkedHashSet<SuiteTestFilter>> filters)570 private static void addFilter( 571 String abi, 572 SuiteTestFilter filter, 573 Map<String, LinkedHashSet<SuiteTestFilter>> filters) { 574 getFilterList(filters, AbiUtils.createId(abi, filter.getName())).add(filter); 575 } 576 getFilterList( Map<String, LinkedHashSet<SuiteTestFilter>> filters, String id)577 private static LinkedHashSet<SuiteTestFilter> getFilterList( 578 Map<String, LinkedHashSet<SuiteTestFilter>> filters, String id) { 579 LinkedHashSet<SuiteTestFilter> fs = filters.get(id); 580 if (fs == null) { 581 fs = new LinkedHashSet<>(); 582 filters.put(id, fs); 583 } 584 return fs; 585 } 586 shouldRunModule(String moduleId)587 private boolean shouldRunModule(String moduleId) { 588 LinkedHashSet<SuiteTestFilter> mdIncludes = getFilterList(mIncludeFilters, moduleId); 589 LinkedHashSet<SuiteTestFilter> mdExcludes = getFilterList(mExcludeFilters, moduleId); 590 // if including all modules or includes exist for this module, and there are not excludes 591 // for the entire module, this module should be run. 592 return (mIncludeAll || !mdIncludes.isEmpty()) && !containsModuleExclude(mdExcludes); 593 } 594 595 /** 596 * Except if the parameterized module is explicitly excluded, including the base module result 597 * in including its parameterization variant. 598 */ shouldRunParameterized( String baseModuleId, String parameterModuleId, String nameWithParam, Set<IModuleParameterHandler> forcedModuleParameters)599 private boolean shouldRunParameterized( 600 String baseModuleId, 601 String parameterModuleId, 602 String nameWithParam, 603 Set<IModuleParameterHandler> forcedModuleParameters) { 604 // Explicitly excluded 605 LinkedHashSet<SuiteTestFilter> excluded = getFilterList(mExcludeFilters, parameterModuleId); 606 LinkedHashSet<SuiteTestFilter> excludedParam = 607 getFilterList(mExcludeFilters, nameWithParam); 608 if (containsModuleExclude(excluded) || containsModuleExclude(excludedParam)) { 609 return false; 610 } 611 612 // Implicitly included due to forced parameter 613 if (forcedModuleParameters != null) { 614 LinkedHashSet<SuiteTestFilter> baseInclude = 615 getFilterList(mIncludeFilters, baseModuleId); 616 if (!baseInclude.isEmpty()) { 617 return true; 618 } 619 } 620 // Explicitly included 621 LinkedHashSet<SuiteTestFilter> included = getFilterList(mIncludeFilters, parameterModuleId); 622 LinkedHashSet<SuiteTestFilter> includedParam = 623 getFilterList(mIncludeFilters, nameWithParam); 624 if (mIncludeAll || !included.isEmpty() || !includedParam.isEmpty()) { 625 return true; 626 } 627 return false; 628 } 629 addTestIncludes( ITestFilterReceiver test, Collection<SuiteTestFilter> includes, String moduleId)630 private void addTestIncludes( 631 ITestFilterReceiver test, 632 Collection<SuiteTestFilter> includes, 633 String moduleId) { 634 if (test instanceof ITestFileFilterReceiver) { 635 String escapedFileName = escapeFilterFileName(moduleId); 636 File includeFile = createFilterFile(escapedFileName, ".include", includes); 637 if (includeFile != null) { 638 ((ITestFileFilterReceiver) test).setIncludeTestFile(includeFile); 639 } 640 } else { 641 // add test includes one at a time 642 for (SuiteTestFilter include : includes) { 643 String filterTestName = include.getTest(); 644 if (filterTestName != null) { 645 test.addIncludeFilter(filterTestName); 646 } 647 } 648 } 649 } 650 addTestExcludes( ITestFilterReceiver test, Collection<SuiteTestFilter> excludes, String moduleId)651 private void addTestExcludes( 652 ITestFilterReceiver test, 653 Collection<SuiteTestFilter> excludes, 654 String moduleId) { 655 if (test instanceof ITestFileFilterReceiver) { 656 String escapedFileName = escapeFilterFileName(moduleId); 657 File excludeFile = createFilterFile(escapedFileName, ".exclude", excludes); 658 if (excludeFile != null) { 659 ((ITestFileFilterReceiver) test).setExcludeTestFile(excludeFile); 660 } 661 } else { 662 // add test excludes one at a time 663 for (SuiteTestFilter exclude : excludes) { 664 test.addExcludeFilter(exclude.getTest()); 665 } 666 } 667 } 668 669 /** module id can contain special characters, avoid them for file names. */ escapeFilterFileName(String moduleId)670 private String escapeFilterFileName(String moduleId) { 671 String escaped = UrlEscapers.urlPathSegmentEscaper().escape(moduleId); 672 return escaped; 673 } 674 createFilterFile( String prefix, String suffix, Collection<SuiteTestFilter> filters)675 private File createFilterFile( 676 String prefix, String suffix, Collection<SuiteTestFilter> filters) { 677 if (filters.isEmpty()) { 678 return null; 679 } 680 File filterFile = null; 681 try { 682 filterFile = FileUtil.createTempFile(prefix, suffix); 683 try (PrintWriter out = new PrintWriter(filterFile)) { 684 for (SuiteTestFilter filter : filters) { 685 String filterTest = filter.getTest(); 686 if (filterTest != null) { 687 out.println(filterTest); 688 } 689 } 690 out.flush(); 691 } 692 } catch (IOException e) { 693 throw new HarnessRuntimeException( 694 "Failed to create filter file", e, InfraErrorIdentifier.FAIL_TO_CREATE_FILE); 695 } 696 if (!filterFile.exists()) { 697 return null; 698 } 699 if (filterFile.length() == 0) { 700 FileUtil.deleteFile(filterFile); 701 return null; 702 } 703 filterFile.deleteOnExit(); 704 return filterFile; 705 } 706 707 /** Returns true iff one or more test filters in excludes apply to the entire module. */ containsModuleExclude(Collection<SuiteTestFilter> excludes)708 private boolean containsModuleExclude(Collection<SuiteTestFilter> excludes) { 709 for (SuiteTestFilter exclude : excludes) { 710 if (exclude.getTest() == null) { 711 return true; 712 } 713 } 714 return false; 715 } 716 717 /** A {@link FilenameFilter} to find all the config files in a directory. */ 718 public static class ConfigFilter implements FilenameFilter { 719 720 /** {@inheritDoc} */ 721 @Override accept(File dir, String name)722 public boolean accept(File dir, String name) { 723 return name.endsWith(CONFIG_EXT); 724 } 725 } 726 727 /** 728 * Parse a list of args formatted as expected into {@link OptionDef} to be injected to module 729 * configurations. 730 * 731 * <p>Format: <module name / module id / class runner>:<option name>:[<arg-key>:=]<arg-value> 732 */ parseArgs(List<String> args, Map<String, List<OptionDef>> moduleOptions)733 private void parseArgs(List<String> args, Map<String, List<OptionDef>> moduleOptions) { 734 for (String arg : args) { 735 int moduleSep = arg.indexOf(":"); 736 if (moduleSep == -1) { 737 throw new HarnessRuntimeException( 738 "Expected delimiter ':' for module or class.", 739 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 740 } 741 String moduleName = arg.substring(0, moduleSep); 742 String remainder = arg.substring(moduleSep + 1); 743 List<OptionDef> listOption = moduleOptions.get(moduleName); 744 if (listOption == null) { 745 listOption = new ArrayList<>(); 746 moduleOptions.put(moduleName, listOption); 747 } 748 int optionNameSep = remainder.indexOf(":"); 749 if (optionNameSep == -1) { 750 throw new HarnessRuntimeException( 751 "Expected delimiter ':' between option name and values.", 752 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 753 } 754 String optionName = remainder.substring(0, optionNameSep); 755 Pattern pattern = Pattern.compile("\\{(.*)\\}(.*)"); 756 Matcher match = pattern.matcher(optionName); 757 if (match.find()) { 758 String alias = match.group(1); 759 String name = match.group(2); 760 optionName = alias + ":" + name; 761 } 762 String optionValueString = remainder.substring(optionNameSep + 1); 763 // TODO: See if QuotationTokenizer can be improved for multi-character delimiter. 764 // or change the delimiter to a single char. 765 String[] tokens = optionValueString.split(":=", 2); 766 OptionDef option = null; 767 if (tokens.length == 1) { 768 option = new OptionDef(optionName, tokens[0], moduleName); 769 } else if (tokens.length == 2) { 770 option = new OptionDef(optionName, tokens[0], tokens[1], moduleName); 771 } 772 listOption.add(option); 773 } 774 } 775 776 /** Gets the list of {@link IModuleParameterHandler}s associated with a module. */ getModuleParameters( String moduleName, IConfiguration config)777 private List<IModuleParameterHandler> getModuleParameters( 778 String moduleName, IConfiguration config) throws ConfigurationException { 779 List<IModuleParameterHandler> params = new ArrayList<>(); 780 Set<String> processedParameterArgs = new HashSet<>(); 781 // Track family of the parameters to make sure we have no duplicate. 782 Map<String, ModuleParameters> duplicateModule = new LinkedHashMap<>(); 783 784 List<String> parameters = 785 config.getConfigurationDescription().getMetaData(ITestSuite.PARAMETER_KEY); 786 if (parameters == null || parameters.isEmpty()) { 787 return params; 788 } 789 790 Set<ModuleParameters> expandedExcludedModuleParameters = new HashSet<>(); 791 for (ModuleParameters moduleParameters : mExcludedModuleParameters) { 792 expandedExcludedModuleParameters.addAll( 793 ModuleParametersHelper.resolveParam( 794 moduleParameters, mAllowOptionalParameterizedModules) 795 .keySet()); 796 } 797 798 for (String p : parameters) { 799 if (!processedParameterArgs.add(p)) { 800 // Avoid processing the same parameter twice 801 continue; 802 } 803 Map<ModuleParameters, IModuleParameterHandler> suiteParams = 804 ModuleParametersHelper.resolveParam( 805 ModuleParameters.valueOf(p.toUpperCase()), 806 mAllowOptionalParameterizedModules); 807 for (Entry<ModuleParameters, IModuleParameterHandler> suiteParamEntry : 808 suiteParams.entrySet()) { 809 ModuleParameters suiteParam = suiteParamEntry.getKey(); 810 String family = suiteParam.getFamily(); 811 if (duplicateModule.containsKey(family)) { 812 // Duplicate family members are not accepted. 813 throw new ConfigurationException( 814 String.format( 815 "Module %s is declaring parameter: " 816 + "%s and %s when only one expected.", 817 moduleName, suiteParam, duplicateModule.get(family))); 818 } else { 819 duplicateModule.put(suiteParam.getFamily(), suiteParam); 820 } 821 // Do not consider the excluded parameterization dimension 822 823 if (expandedExcludedModuleParameters.contains(suiteParam)) { 824 continue; 825 } 826 827 if (suiteParamEntry.getValue() instanceof FoldableExpandingHandler) { 828 List<IModuleParameterHandler> foldableHandlers = 829 ((FoldableExpandingHandler) suiteParamEntry.getValue()) 830 .expandHandler(mFoldableStates); 831 params.addAll(foldableHandlers); 832 } else { 833 params.add(suiteParamEntry.getValue()); 834 } 835 } 836 } 837 return params; 838 } 839 840 /** Gets the list of parameterized mainline modules associated with a module. */ 841 @VisibleForTesting getMainlineModuleParameters(IConfiguration config)842 List<String> getMainlineModuleParameters(IConfiguration config) throws ConfigurationException { 843 List<String> params = new ArrayList<>(); 844 845 List<String> parameters = 846 config.getConfigurationDescription().getMetaData(ITestSuite.MAINLINE_PARAMETER_KEY); 847 if (parameters == null || parameters.isEmpty()) { 848 return params; 849 } 850 851 return new ArrayList<>(dedupMainlineParameters(parameters, config.getName())); 852 } 853 854 /** 855 * De-duplicate the given mainline parameters. 856 * 857 * @param parameters The list of given mainline parameters. 858 * @param configName The test configuration name. 859 * @return The de-duplicated mainline modules list. 860 */ 861 @VisibleForTesting dedupMainlineParameters(List<String> parameters, String configName)862 Set<String> dedupMainlineParameters(List<String> parameters, String configName) 863 throws ConfigurationException { 864 Set<String> results = new HashSet<>(); 865 for (String param : parameters) { 866 if (!isValidMainlineParam(param)) { 867 throw new ConfigurationException( 868 String.format( 869 "Illegal mainline module parameter: \"%s\" configured in the test" 870 + " config: %s. Parameter must end with .apk/.apex/.apks and" 871 + " have no any spaces configured.", 872 param, configName)); 873 } 874 if (!isInAlphabeticalOrder(param)) { 875 throw new ConfigurationException( 876 String.format( 877 "Illegal mainline module parameter: \"%s\" configured in the test" 878 + " config: %s. Parameter must be configured in alphabetical" 879 + " order or with no duplicated modules.", 880 param, configName)); 881 } 882 results.add(param); 883 } 884 return results; 885 } 886 887 /** Whether a mainline parameter configured in a test config is in alphabetical order or not. */ 888 @VisibleForTesting isInAlphabeticalOrder(String param)889 boolean isInAlphabeticalOrder(String param) { 890 String previousString = ""; 891 for (String currentString : param.split(String.format("\\+"))) { 892 // This is to check if the parameter is in alphabetical order or duplicated. 893 if (currentString.compareTo(previousString) <= 0) { 894 return false; 895 } 896 previousString = currentString; 897 } 898 return true; 899 } 900 901 /** Whether the mainline parameter configured in the test config is valid or not. */ 902 @VisibleForTesting isValidMainlineParam(String param)903 boolean isValidMainlineParam(String param) { 904 if (param.contains(" ")) { 905 return false; 906 } 907 for (String m : param.split(String.format("\\+"))) { 908 if (!MAINLINE_PARAMETERS_TO_VALIDATE.stream().anyMatch(entry -> m.endsWith(entry))) { 909 return false; 910 } 911 } 912 return true; 913 } 914 915 /** 916 * Setup the options for the module configuration. 917 * 918 * @param name The base name of the module 919 * @param nameWithParam The id of the parameterized mainline module (module name + parameters) 920 * @param id The base id name of the module. 921 * @param fullId The full id of the module (usually abi + module name + parameters) 922 * @param config The module configuration. 923 * @param abi The abi of the module. 924 * @throws ConfigurationException 925 */ setUpConfig( String name, String nameWithParam, String id, String fullId, IConfiguration config, IAbi abi)926 private void setUpConfig( 927 String name, 928 String nameWithParam, 929 String id, 930 String fullId, 931 IConfiguration config, 932 IAbi abi) 933 throws ConfigurationException { 934 List<OptionDef> optionsToInject = new ArrayList<>(); 935 if (mModuleOptions.containsKey(name)) { 936 optionsToInject.addAll(mModuleOptions.get(name)); 937 } 938 if (nameWithParam != null && mModuleOptions.containsKey(nameWithParam)) { 939 optionsToInject.addAll(mModuleOptions.get(nameWithParam)); 940 } 941 if (mModuleOptions.containsKey(id)) { 942 optionsToInject.addAll(mModuleOptions.get(id)); 943 } 944 if (mModuleOptions.containsKey(fullId)) { 945 optionsToInject.addAll(mModuleOptions.get(fullId)); 946 } 947 config.injectOptionValues(optionsToInject); 948 949 for (IMetricCollector collector : config.getMetricCollectors()) { 950 String className = collector.getClass().getName(); 951 if (mTestOrPreparerOptions.containsKey(className)) { 952 OptionSetter collectorSetter = new OptionSetter(collector); 953 for (OptionDef def : mTestOrPreparerOptions.get(className)) { 954 collectorSetter.setOptionValue(def.name, def.key, def.value); 955 } 956 } 957 } 958 959 for (IPostProcessor postProcessor : config.getPostProcessors()) { 960 String className = postProcessor.getClass().getName(); 961 if (mTestOrPreparerOptions.containsKey(className)) { 962 OptionSetter processorSetter = new OptionSetter(postProcessor); 963 for (OptionDef def : mTestOrPreparerOptions.get(className)) { 964 processorSetter.setOptionValue(def.name, def.key, def.value); 965 } 966 } 967 } 968 969 // Set target preparers 970 for (IDeviceConfiguration holder : config.getDeviceConfig()) { 971 for (ITargetPreparer preparer : holder.getTargetPreparers()) { 972 String className = preparer.getClass().getName(); 973 if (mTestOrPreparerOptions.containsKey(className)) { 974 OptionSetter preparerSetter = new OptionSetter(preparer); 975 for (OptionDef def : mTestOrPreparerOptions.get(className)) { 976 preparerSetter.setOptionValue(def.name, def.key, def.value); 977 } 978 } 979 if (preparer instanceof IAbiReceiver) { 980 ((IAbiReceiver) preparer).setAbi(abi); 981 } 982 } 983 } 984 985 // Set IRemoteTests 986 List<IRemoteTest> tests = config.getTests(); 987 for (IRemoteTest test : tests) { 988 String className = test.getClass().getName(); 989 if (mTestOrPreparerOptions.containsKey(className)) { 990 OptionSetter preparerSetter = new OptionSetter(test); 991 for (OptionDef def : mTestOrPreparerOptions.get(className)) { 992 preparerSetter.setOptionValue(def.name, def.key, def.value); 993 } 994 } 995 addFiltersToTest(test, abi, fullId, mIncludeFilters, mExcludeFilters); 996 if (test instanceof IAbiReceiver) { 997 ((IAbiReceiver) test).setAbi(abi); 998 } 999 } 1000 1001 // add the abi and module name to the description 1002 config.getConfigurationDescription().setAbi(abi); 1003 config.getConfigurationDescription().setModuleName(name); 1004 1005 config.validateOptions(); 1006 } 1007 1008 /** Whether or not the base configuration should be created for all abis or not. */ shouldCreateMultiAbiForBase(List<IModuleParameterHandler> params)1009 private boolean shouldCreateMultiAbiForBase(List<IModuleParameterHandler> params) { 1010 for (IModuleParameterHandler param : params) { 1011 if (param instanceof NotMultiAbiHandler) { 1012 return false; 1013 } 1014 } 1015 return true; 1016 } 1017 } 1018