1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.config; 18 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.command.CommandOptions; 21 import com.android.tradefed.config.proxy.TradefedDelegator; 22 import com.android.tradefed.config.remote.ExtendedFile; 23 import com.android.tradefed.config.remote.IRemoteFileResolver.ResolvedFile; 24 import com.android.tradefed.config.yaml.ConfigurationYamlParser; 25 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.tradefed.result.error.InfraErrorIdentifier; 28 import com.android.tradefed.util.ClassPathScanner; 29 import com.android.tradefed.util.ClassPathScanner.IClassPathFilter; 30 import com.android.tradefed.util.DirectedGraph; 31 import com.android.tradefed.util.FileUtil; 32 import com.android.tradefed.util.StreamUtil; 33 import com.android.tradefed.util.SystemUtil; 34 import com.android.tradefed.util.keystore.DryRunKeyStore; 35 import com.android.tradefed.util.keystore.IKeyStoreClient; 36 37 import com.google.common.annotations.VisibleForTesting; 38 import com.google.common.base.Strings; 39 import com.google.common.collect.ImmutableSortedSet; 40 41 import java.io.BufferedInputStream; 42 import java.io.ByteArrayOutputStream; 43 import java.io.File; 44 import java.io.FileInputStream; 45 import java.io.FileNotFoundException; 46 import java.io.IOException; 47 import java.io.InputStream; 48 import java.io.PrintStream; 49 import java.net.URI; 50 import java.net.URISyntaxException; 51 import java.util.ArrayList; 52 import java.util.Arrays; 53 import java.util.Comparator; 54 import java.util.HashMap; 55 import java.util.HashSet; 56 import java.util.Hashtable; 57 import java.util.List; 58 import java.util.Map; 59 import java.util.Set; 60 import java.util.SortedSet; 61 import java.util.TreeSet; 62 import java.util.regex.Pattern; 63 64 /** 65 * Factory for creating {@link IConfiguration}. 66 */ 67 public class ConfigurationFactory implements IConfigurationFactory { 68 69 /** Currently supported extensions for Tradefed configurations */ 70 private static final Set<String> SUPPORTED_EXTENSIONS = 71 ImmutableSortedSet.of(".xml", ".config"); 72 73 private static IConfigurationFactory sInstance = null; 74 private static final String CONFIG_PREFIX = "config/"; 75 private static final String DRY_RUN_TEMPLATE_CONFIG = "empty"; 76 private static final String CONFIG_ERROR_PATTERN = "(Could not find option with name )(.*)"; 77 // TODO(murj) generalize this to a URI matcher 78 private static final String DIRECT_CONFIG_PATTERN = "^(gs|file|http|https)://.*"; 79 80 private Map<ConfigId, ConfigurationDef> mConfigDefMap; 81 82 /** 83 * A simple struct-like class that stores a configuration's name alongside 84 * the arguments for any {@code <template-include>} tags it may contain. 85 * Because the actual bits stored by the configuration may vary with 86 * template arguments, they must be considered as essential a part of the 87 * configuration's identity as the filename. 88 */ 89 static class ConfigId { 90 public String name = null; 91 public Map<String, String> templateMap = new HashMap<>(); 92 93 /** 94 * No-op constructor 95 */ ConfigId()96 public ConfigId() { 97 } 98 99 /** 100 * Convenience constructor. Equivalent to calling two-arg constructor 101 * with {@code null} {@code templateMap}. 102 */ ConfigId(String name)103 public ConfigId(String name) { 104 this(name, null); 105 } 106 107 /** 108 * Two-arg convenience constructor. {@code templateMap} may be null. 109 */ ConfigId(String name, Map<String, String> templateMap)110 public ConfigId(String name, Map<String, String> templateMap) { 111 this.name = name; 112 if (templateMap != null) { 113 this.templateMap.putAll(templateMap); 114 } 115 } 116 117 /** 118 * {@inheritDoc} 119 */ 120 @Override hashCode()121 public int hashCode() { 122 return 2 * ((name == null) ? 0 : name.hashCode()) + 3 * templateMap.hashCode(); 123 } 124 matches(Object a, Object b)125 private boolean matches(Object a, Object b) { 126 if (a == null && b == null) { 127 return true; 128 } 129 if (a == null || b == null) { 130 return false; 131 } 132 return a.equals(b); 133 } 134 135 /** 136 * {@inheritDoc} 137 */ 138 @Override equals(Object other)139 public boolean equals(Object other) { 140 if (other == null) { 141 return false; 142 } 143 if (!(other instanceof ConfigId)) { 144 return false; 145 } 146 147 final ConfigId otherConf = (ConfigId) other; 148 return matches(name, otherConf.name) && matches(templateMap, otherConf.templateMap); 149 } 150 } 151 152 /** 153 * A {@link IClassPathFilter} for configuration XML files. 154 */ 155 private class ConfigClasspathFilter implements IClassPathFilter { 156 157 private String mPrefix = null; 158 ConfigClasspathFilter(String prefix)159 public ConfigClasspathFilter(String prefix) { 160 mPrefix = getConfigPrefix(); 161 if (prefix != null) { 162 mPrefix += prefix; 163 } 164 CLog.d("Searching the '%s' config path", mPrefix); 165 } 166 167 /** 168 * {@inheritDoc} 169 */ 170 @Override accept(String pathName)171 public boolean accept(String pathName) { 172 // only accept entries that match the pattern, and that we don't already know about 173 final ConfigId pathId = new ConfigId(pathName); 174 String extension = FileUtil.getExtension(pathName); 175 return pathName.startsWith(mPrefix) 176 && SUPPORTED_EXTENSIONS.contains(extension) 177 && !mConfigDefMap.containsKey(pathId); 178 } 179 180 /** 181 * {@inheritDoc} 182 */ 183 @Override transform(String pathName)184 public String transform(String pathName) { 185 // strip off CONFIG_PREFIX and config extension 186 int pathStartIndex = getConfigPrefix().length(); 187 String extension = FileUtil.getExtension(pathName); 188 int pathEndIndex = pathName.length() - extension.length(); 189 return pathName.substring(pathStartIndex, pathEndIndex); 190 } 191 } 192 193 /** 194 * A {@link Comparator} for {@link ConfigurationDef} that sorts by 195 * {@link ConfigurationDef#getName()}. 196 */ 197 private static class ConfigDefComparator implements Comparator<ConfigurationDef> { 198 199 /** 200 * {@inheritDoc} 201 */ 202 @Override compare(ConfigurationDef d1, ConfigurationDef d2)203 public int compare(ConfigurationDef d1, ConfigurationDef d2) { 204 return d1.getName().compareTo(d2.getName()); 205 } 206 207 } 208 209 /** 210 * Get a list of {@link File} of the test cases directories 211 * 212 * <p>The wrapper function is for unit test to mock the system calls. 213 * 214 * @return a list of {@link File} of directories of the test cases folder of build output, based 215 * on the value of environment variables. 216 */ 217 @VisibleForTesting getExternalTestCasesDirs()218 List<File> getExternalTestCasesDirs() { 219 return SystemUtil.getExternalTestCasesDirs(); 220 } 221 222 /** 223 * Get the path to the config file for a test case. 224 * 225 * <p>The given name in a test config can be the name of a test case located in an out directory 226 * defined in the following environment variables: 227 * 228 * <p>ANDROID_TARGET_OUT_TESTCASES 229 * 230 * <p>ANDROID_HOST_OUT_TESTCASES 231 * 232 * <p>This method tries to locate the test config name in these directories. If no config is 233 * found, return null. 234 * 235 * @param name Name of a config file. 236 * @return A File object of the config file for the given test case. 237 */ 238 @VisibleForTesting getTestCaseConfigPath(String name)239 File getTestCaseConfigPath(String name) { 240 String[] possibleConfigFileNames = {name}; 241 if (Strings.isNullOrEmpty(FileUtil.getExtension(name))) { 242 possibleConfigFileNames = new String[SUPPORTED_EXTENSIONS.size()]; 243 int i = 0; 244 for (String supportedExtension : SUPPORTED_EXTENSIONS) { 245 possibleConfigFileNames[i] = (name + supportedExtension); 246 i++; 247 } 248 } 249 250 for (File testCasesDir : getExternalTestCasesDirs()) { 251 for (String configFileName : possibleConfigFileNames) { 252 File config = FileUtil.findFile(testCasesDir, configFileName); 253 if (config != null) { 254 CLog.d("Using config: %s/%s", testCasesDir.getAbsoluteFile(), configFileName); 255 return config; 256 } 257 } 258 } 259 return null; 260 } 261 262 /** 263 * Implementation of {@link IConfigDefLoader} that tracks the included configurations from one 264 * root config, and throws an exception on circular includes. 265 */ 266 protected class ConfigLoader implements IConfigDefLoader { 267 268 private final boolean mIsGlobalConfig; 269 private DirectedGraph<String> mConfigGraph = new DirectedGraph<String>(); 270 ConfigLoader(boolean isGlobalConfig)271 public ConfigLoader(boolean isGlobalConfig) { 272 mIsGlobalConfig = isGlobalConfig; 273 } 274 275 /** 276 * {@inheritDoc} 277 */ 278 @Override getConfigurationDef(String name, Map<String, String> templateMap)279 public ConfigurationDef getConfigurationDef(String name, Map<String, String> templateMap) 280 throws ConfigurationException { 281 282 String configName = findConfigName(name, null); 283 final ConfigId configId = new ConfigId(name, templateMap); 284 ConfigurationDef def = mConfigDefMap.get(configId); 285 286 if (def == null || def.isStale()) { 287 def = new ConfigurationDef(configName); 288 loadConfiguration(configName, def, null, templateMap, null); 289 mConfigDefMap.put(configId, def); 290 } else { 291 if (templateMap != null) { 292 // Clearing the map before returning the cached config to 293 // avoid seeing them as unused. 294 templateMap.clear(); 295 } 296 } 297 return def; 298 } 299 300 /** Returns true if it is a config file found inside the classpath. */ isBundledConfig(String name)301 protected boolean isBundledConfig(String name) { 302 InputStream configStream = getBundledConfigStream(name); 303 return configStream != null; 304 } 305 306 /** 307 * Get the absolute path of a local config file. 308 * 309 * @param root parent path of config file 310 * @param name config file 311 * @return absolute path for local config file. 312 * @throws ConfigurationException 313 */ getAbsolutePath(String root, String name)314 private String getAbsolutePath(String root, String name) throws ConfigurationException { 315 File file = new File(name); 316 if (!file.isAbsolute()) { 317 if (root == null) { 318 // if root directory was not specified, get the current 319 // working directory. 320 root = System.getProperty("user.dir"); 321 } 322 file = new File(root, name); 323 } 324 try { 325 return file.getCanonicalPath(); 326 } catch (IOException e) { 327 throw new ConfigurationException( 328 String.format( 329 "Failure when trying to determine local file canonical path %s", e), 330 InfraErrorIdentifier.CONFIGURATION_NOT_FOUND); 331 } 332 } 333 334 /** 335 * Find config's name based on its name and its parent name. This is used to properly handle 336 * bundle configs and local configs. 337 * 338 * @param name config's name 339 * @param parentName config's parent's name. 340 * @return the config's full name. 341 * @throws ConfigurationException 342 */ findConfigName(String name, String parentName)343 protected String findConfigName(String name, String parentName) 344 throws ConfigurationException { 345 if (isBundledConfig(name)) { 346 return name; 347 } 348 if (parentName == null || isBundledConfig(parentName)) { 349 // Search files for config. 350 String configName = getAbsolutePath(null, name); 351 File localConfig = new File(configName); 352 if (!localConfig.exists()) { 353 localConfig = getTestCaseConfigPath(name); 354 } 355 if (localConfig != null) { 356 return localConfig.getAbsolutePath(); 357 } 358 // Can not find local config. 359 if (parentName == null) { 360 throw new ConfigurationException( 361 String.format("Can not find local config %s.", name), 362 InfraErrorIdentifier.CONFIGURATION_NOT_FOUND); 363 364 } else { 365 throw new ConfigurationException( 366 String.format( 367 "Bundled config '%s' is including a config '%s' that's neither " 368 + "local nor bundled.", 369 parentName, name), 370 InfraErrorIdentifier.CONFIGURATION_NOT_FOUND); 371 } 372 } 373 try { 374 // Local configs' include should be relative to their parent's path. 375 String parentRoot = new File(parentName).getParentFile().getCanonicalPath(); 376 return getAbsolutePath(parentRoot, name); 377 } catch (IOException e) { 378 throw new ConfigurationException( 379 e.getMessage(), e.getCause(), InfraErrorIdentifier.CONFIGURATION_NOT_FOUND); 380 } 381 } 382 383 /** 384 * Configs that are bundled inside the tradefed.jar can only include other configs also 385 * bundled inside tradefed.jar. However, local (external) configs can include both local 386 * (external) and bundled configs. 387 */ 388 @Override loadIncludedConfiguration( ConfigurationDef def, String parentName, String name, String deviceTagObject, Map<String, String> templateMap, Set<String> templateSeen)389 public void loadIncludedConfiguration( 390 ConfigurationDef def, 391 String parentName, 392 String name, 393 String deviceTagObject, 394 Map<String, String> templateMap, 395 Set<String> templateSeen) 396 throws ConfigurationException { 397 398 String config_name = findConfigName(name, parentName); 399 mConfigGraph.addEdge(parentName, config_name); 400 // If the inclusion of configurations is a cycle we throw an exception. 401 if (!mConfigGraph.isDag()) { 402 CLog.e("%s", mConfigGraph); 403 throw new ConfigurationException( 404 String.format( 405 "Circular configuration include: config '%s' is already included", 406 config_name), 407 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 408 } 409 loadConfiguration(config_name, def, deviceTagObject, templateMap, templateSeen); 410 } 411 412 /** 413 * Loads a configuration. 414 * 415 * @param name the name of a built-in configuration to load or a file path to configuration 416 * file to load 417 * @param def the loaded {@link ConfigurationDef} 418 * @param deviceTagObject name of the current deviceTag if we are loading from a config 419 * inside an <include>. Null otherwise. 420 * @param templateMap map from template-include names to their respective concrete 421 * configuration files 422 * @param templateSeen set of template placeholder name already encountered 423 * @throws ConfigurationException if a configuration with given name/file path cannot be 424 * loaded or parsed 425 */ loadConfiguration( String name, ConfigurationDef def, String deviceTagObject, Map<String, String> templateMap, Set<String> templateSeen)426 void loadConfiguration( 427 String name, 428 ConfigurationDef def, 429 String deviceTagObject, 430 Map<String, String> templateMap, 431 Set<String> templateSeen) 432 throws ConfigurationException { 433 BufferedInputStream bufStream = getConfigStream(name); 434 String extension = FileUtil.getExtension(name); 435 switch (extension) { 436 case ".xml": 437 case ".config": 438 case "": 439 ConfigurationXmlParser parser = 440 new ConfigurationXmlParser(this, deviceTagObject); 441 parser.parse(def, name, bufStream, templateMap, templateSeen); 442 break; 443 case ".tf_yaml": 444 ConfigurationYamlParser yamlParser = new ConfigurationYamlParser(); 445 yamlParser.parse(def, name, bufStream, false); 446 break; 447 default: 448 throw new ConfigurationException( 449 String.format("The config format for %s is not supported.", name)); 450 } 451 trackConfig(name, def); 452 } 453 454 /** 455 * Track config for dynamic loading. Right now only local files are supported. 456 * 457 * @param name config's name 458 * @param def config's def. 459 */ trackConfig(String name, ConfigurationDef def)460 protected void trackConfig(String name, ConfigurationDef def) { 461 // Track local config source files 462 if (!isBundledConfig(name)) { 463 def.registerSource(new File(name)); 464 } 465 } 466 467 /** 468 * Should track the config's life cycle or not. 469 * 470 * @param name config's name 471 * @return <code>true</code> if the config is trackable, otherwise <code>false</code>. 472 */ isTrackableConfig(String name)473 protected boolean isTrackableConfig(String name) { 474 return !isBundledConfig(name); 475 } 476 477 /** 478 * {@inheritDoc} 479 */ 480 @Override isGlobalConfig()481 public boolean isGlobalConfig() { 482 return mIsGlobalConfig; 483 } 484 485 } 486 ConfigurationFactory()487 protected ConfigurationFactory() { 488 mConfigDefMap = new Hashtable<ConfigId, ConfigurationDef>(); 489 } 490 491 /** 492 * Get the singleton {@link IConfigurationFactory} instance. 493 */ getInstance()494 public static IConfigurationFactory getInstance() { 495 if (sInstance == null) { 496 sInstance = new ConfigurationFactory(); 497 } 498 return sInstance; 499 } 500 501 /** 502 * Retrieve the {@link ConfigurationDef} for the given name 503 * 504 * @param name the name of a built-in configuration to load or a file path to configuration file 505 * to load 506 * @return {@link ConfigurationDef} 507 * @throws ConfigurationException if an error occurred loading the config 508 */ getConfigurationDef( String name, boolean isGlobal, Map<String, String> templateMap)509 protected ConfigurationDef getConfigurationDef( 510 String name, boolean isGlobal, Map<String, String> templateMap) 511 throws ConfigurationException { 512 try (CloseableTraceScope ignored = new CloseableTraceScope("getConfigurationDef")) { 513 return new ConfigLoader(isGlobal).getConfigurationDef(name, templateMap); 514 } 515 } 516 517 /** 518 * {@inheritDoc} 519 */ 520 @Override createConfigurationFromArgs(String[] arrayArgs)521 public IConfiguration createConfigurationFromArgs(String[] arrayArgs) 522 throws ConfigurationException { 523 return createConfigurationFromArgs(arrayArgs, null); 524 } 525 526 /** 527 * {@inheritDoc} 528 */ 529 @Override createConfigurationFromArgs(String[] arrayArgs, List<String> unconsumedArgs)530 public IConfiguration createConfigurationFromArgs(String[] arrayArgs, 531 List<String> unconsumedArgs) throws ConfigurationException { 532 return createConfigurationFromArgs(arrayArgs, unconsumedArgs, null); 533 } 534 535 /** 536 * {@inheritDoc} 537 */ 538 @Override createConfigurationFromArgs(String[] arrayArgs, List<String> unconsumedArgs, IKeyStoreClient keyStoreClient)539 public IConfiguration createConfigurationFromArgs(String[] arrayArgs, 540 List<String> unconsumedArgs, IKeyStoreClient keyStoreClient) 541 throws ConfigurationException { 542 if (arrayArgs.length == 0) { 543 throw new ConfigurationException("Configuration to run was not specified"); 544 } 545 546 List<String> listArgs = new ArrayList<String>(arrayArgs.length); 547 // FIXME: Update parsing to not care about arg order. 548 String[] reorderedArrayArgs = reorderArgs(arrayArgs); 549 IConfiguration config = 550 internalCreateConfigurationFromArgs( 551 reorderedArrayArgs, listArgs, keyStoreClient, null); 552 config.setCommandLine(arrayArgs); 553 if (listArgs.contains("--" + CommandOptions.DRY_RUN_OPTION) 554 || listArgs.contains("--" + CommandOptions.NOISY_DRY_RUN_OPTION)) { 555 // In case of dry-run, we replace the KeyStore by a dry-run one. 556 CLog.w("dry-run detected, we are using a dryrun keystore"); 557 keyStoreClient = new DryRunKeyStore(); 558 } 559 final List<String> tmpUnconsumedArgs = 560 config.setOptionsFromCommandLineArgs(listArgs, keyStoreClient); 561 562 if (unconsumedArgs == null && tmpUnconsumedArgs.size() > 0) { 563 // (unconsumedArgs == null) is taken as a signal that the caller 564 // expects all args to 565 // be processed. 566 throw new ConfigurationException( 567 String.format( 568 "Invalid arguments provided. Unprocessed arguments: %s", 569 tmpUnconsumedArgs), 570 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 571 } else if (unconsumedArgs != null) { 572 // Return the unprocessed args 573 unconsumedArgs.addAll(tmpUnconsumedArgs); 574 } 575 576 return config; 577 } 578 579 /** {@inheritDoc} */ 580 @Override createPartialConfigurationFromArgs( String[] arrayArgs, IKeyStoreClient keyStoreClient, Set<String> allowedObjects, TradefedDelegator delegator)581 public IConfiguration createPartialConfigurationFromArgs( 582 String[] arrayArgs, 583 IKeyStoreClient keyStoreClient, 584 Set<String> allowedObjects, 585 TradefedDelegator delegator) 586 throws ConfigurationException { 587 if (arrayArgs.length == 0) { 588 throw new ConfigurationException( 589 "Configuration to run was not specified", 590 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 591 } 592 593 List<String> listArgs = new ArrayList<String>(arrayArgs.length); 594 String[] reorderedArrayArgs = reorderArgs(arrayArgs); 595 IConfiguration config = 596 internalCreateConfigurationFromArgs( 597 reorderedArrayArgs, listArgs, keyStoreClient, allowedObjects); 598 if (delegator != null) { 599 config.setConfigurationObject(TradefedDelegator.DELEGATE_OBJECT, delegator); 600 } 601 config.setCommandLine(arrayArgs); 602 List<String> leftOver = 603 config.setBestEffortOptionsFromCommandLineArgs(listArgs, keyStoreClient); 604 CLog.d("Non-applied arguments: %s", leftOver); 605 return config; 606 } 607 608 @VisibleForTesting isDirectConfiguration(String configName)609 protected boolean isDirectConfiguration(String configName) { 610 return Pattern.matches(DIRECT_CONFIG_PATTERN, configName); 611 } 612 613 /** 614 * Creates a {@link Configuration} from the name given in arguments. 615 * 616 * <p>Note will not populate configuration with values from options 617 * 618 * @param arrayArgs the full list of command line arguments, including the config name 619 * @param optionArgsRef an empty list, that will be populated with the option arguments left to 620 * be interpreted 621 * @param keyStoreClient {@link IKeyStoreClient} keystore client to use if any. 622 * @param allowedObjects config object that are allowed to be created. 623 * @return An {@link IConfiguration} object representing the configuration that was loaded 624 * @throws ConfigurationException 625 */ internalCreateConfigurationFromArgs( String[] arrayArgs, List<String> optionArgsRef, IKeyStoreClient keyStoreClient, Set<String> allowedObjects)626 private IConfiguration internalCreateConfigurationFromArgs( 627 String[] arrayArgs, 628 List<String> optionArgsRef, 629 IKeyStoreClient keyStoreClient, 630 Set<String> allowedObjects) 631 throws ConfigurationException { 632 final List<String> listArgs = new ArrayList<>(Arrays.asList(arrayArgs)); 633 // first arg is config name 634 final String configName = listArgs.remove(0); 635 636 // ATTN This section short-circuts the rest of the configuration pipeline 637 if (isDirectConfiguration(configName)) { 638 return internalCreateDirectConfiguration( 639 configName, listArgs, optionArgsRef, keyStoreClient, allowedObjects); 640 } 641 642 Map<String, String> uniqueMap = 643 extractTemplates(configName, listArgs, optionArgsRef, keyStoreClient); 644 if (allowedObjects != null && !allowedObjects.isEmpty()) { 645 ConfigLoader tmpLoader = new ConfigLoader(false); 646 // For partial loading be lenient about templates and let the delegate deal with it. 647 // In some cases this won't be 100% correct but it's better than failing on all new 648 // configs. 649 for (String key : uniqueMap.keySet()) { 650 try { 651 tmpLoader.findConfigName(uniqueMap.get(key), null); 652 } catch (ConfigurationException e) { 653 uniqueMap.put(key, "empty"); 654 } 655 } 656 } 657 ConfigurationDef configDef = getConfigurationDef(configName, false, uniqueMap); 658 if (!uniqueMap.isEmpty()) { 659 // remove the bad ConfigDef from the cache. 660 for (ConfigId cid : mConfigDefMap.keySet()) { 661 if (mConfigDefMap.get(cid) == configDef) { 662 CLog.d("Cleaning the cache for this configdef"); 663 mConfigDefMap.remove(cid); 664 break; 665 } 666 } 667 throw new ConfigurationException( 668 String.format("Unused template:map parameters: %s", uniqueMap.toString()), 669 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 670 } 671 return configDef.createConfiguration(allowedObjects); 672 } 673 extractTemplates( String configName, List<String> listArgs, List<String> optionArgsRef, IKeyStoreClient keyStoreClient)674 private Map<String, String> extractTemplates( 675 String configName, 676 List<String> listArgs, 677 List<String> optionArgsRef, 678 IKeyStoreClient keyStoreClient) 679 throws ConfigurationException { 680 try (CloseableTraceScope ignored = new CloseableTraceScope("extractTemplates")) { 681 final String extension = FileUtil.getExtension(configName); 682 switch (extension) { 683 case ".xml": 684 case ".config": 685 case "": 686 final ConfigurationXmlParserSettings parserSettings = 687 new ConfigurationXmlParserSettings(); 688 final ArgsOptionParser templateArgParser = new ArgsOptionParser(parserSettings); 689 if (keyStoreClient != null) { 690 templateArgParser.setKeyStore(keyStoreClient); 691 } 692 optionArgsRef.addAll(templateArgParser.parseBestEffort(listArgs)); 693 // Check that the same template is not attempted to be loaded twice. 694 for (String key : parserSettings.templateMap.keySet()) { 695 if (parserSettings.templateMap.get(key).size() > 1) { 696 throw new ConfigurationException( 697 String.format( 698 "More than one template specified for key '%s'", key), 699 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 700 } 701 } 702 return parserSettings.templateMap.getUniqueMap(); 703 case ".tf_yaml": 704 // We parse the arguments but don't support template for YAML 705 final ArgsOptionParser allArgsParser = new ArgsOptionParser(); 706 if (keyStoreClient != null) { 707 allArgsParser.setKeyStore(keyStoreClient); 708 } 709 optionArgsRef.addAll(allArgsParser.parseBestEffort(listArgs)); 710 return new HashMap<>(); 711 default: 712 return new HashMap<>(); 713 } 714 } 715 } 716 717 /** 718 * A delegation method from the other `internalCreateConfiguration...` methods for direct 719 * configs 720 * 721 * <p>This method encapsulates the direct configuration flow so that we can separate it from the 722 * legacy flows for future refactoring. 723 * 724 * @param listArgs list of command arguments **not including the config name** 725 * @param optionArgsRef an empty list, that will be populated with the option arguments left to 726 * be interpreted (should be populated with all non-template option arguments) 727 * @param keyStoreClient {@link IKeyStoreClient} keystore client to use if any. 728 * @param allowedObjects config object that are allowed to be created. 729 * @return An {@link IConfiguration} object representing the configuration that was loaded 730 * @throws ConfigurationException 731 */ 732 @VisibleForTesting internalCreateDirectConfiguration( String configName, List<String> listArgs, List<String> optionArgsRef, IKeyStoreClient keyStoreClient, Set<String> allowedObjects)733 private IConfiguration internalCreateDirectConfiguration( 734 String configName, 735 List<String> listArgs, 736 List<String> optionArgsRef, 737 IKeyStoreClient keyStoreClient, 738 Set<String> allowedObjects) 739 throws ConfigurationException { 740 // Download the file and do some error handling here 741 try { 742 URI configURI = new URI(configName); 743 String name = 744 Arrays.stream(configURI.getPath().split("/")) 745 .reduce((first, second) -> second) 746 .orElseThrow(); 747 CLog.i("Determined the config name was %s", name); 748 749 // GCS resolver doesn't respect this, but just in case others do, 750 // we'd prefer them here. 751 File destDir = FileUtil.createTempDir("tf-configs"); 752 753 ResolvedFile resolvedConfigFile = resolveRemoteFile(configURI, destDir.toURI()); 754 File configFile = resolvedConfigFile.getResolvedFile(); 755 if (configFile instanceof ExtendedFile) { 756 ((ExtendedFile) configFile).waitForDownload(); 757 } 758 759 CLog.i("Attempting to read from file: %s", configFile.getPath()); 760 try (BufferedInputStream configInputStream = 761 new BufferedInputStream(new FileInputStream(configFile))) { 762 ConfigurationDef configDef = new ConfigurationDef(configName); 763 764 switch (FileUtil.getExtension(configFile.getPath())) { 765 case ".xml": 766 case ".config": 767 case "": 768 // Note: this disabled config loader both prevents templates from being 769 // instantiated and allows me to use the ConfigurationXMLParser without 770 // substantial modification, for now. 771 ConfigLoader exceptionLoader = new ExceptionLoader(false); 772 ConfigurationXmlParser parser = 773 new ConfigurationXmlParser(exceptionLoader, null); 774 775 parser.parse( 776 configDef, 777 configName, 778 configInputStream, 779 new HashMap<String, String>(), 780 new HashSet<String>()); 781 break; 782 case ".tf_yaml": 783 ConfigurationYamlParser yamlParser = new ConfigurationYamlParser(); 784 yamlParser.parse(configDef, configName, configInputStream, false); 785 break; 786 default: 787 throw new ConfigurationException( 788 String.format( 789 "The config format for %s is not supported.", configName), 790 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 791 } 792 793 if (resolvedConfigFile.shouldCleanUp()) { 794 FileUtil.deleteFile(configFile); 795 } 796 FileUtil.recursiveDelete(destDir); 797 798 final ConfigurationXmlParserSettings parserSettings = 799 new ConfigurationXmlParserSettings(); 800 final ArgsOptionParser cmdArgParser = new ArgsOptionParser(parserSettings); 801 if (keyStoreClient != null) { 802 cmdArgParser.setKeyStore(keyStoreClient); 803 } 804 optionArgsRef.addAll(cmdArgParser.parseBestEffort(listArgs)); 805 806 return configDef.createConfiguration(allowedObjects); 807 } 808 } catch (FileNotFoundException e) { 809 throw new ConfigurationException(e.toString(), e); 810 } catch (BuildRetrievalError e) { 811 throw new ConfigurationException(e.toString(), e); 812 } catch (URISyntaxException e) { 813 throw new ConfigurationException( 814 String.format("Invalid URI specified: %s", configName), e); 815 } catch (IOException e) { 816 throw new RuntimeException("Failed to create temp dir for config", e); 817 } 818 } 819 820 @VisibleForTesting resolveRemoteFile(URI configURI, URI destDir)821 protected ResolvedFile resolveRemoteFile(URI configURI, URI destDir) 822 throws BuildRetrievalError { 823 return RemoteFileResolver.resolveRemoteFile(configURI, destDir); 824 } 825 826 protected class ExceptionLoader extends ConfigLoader { ExceptionLoader(boolean isGlobal)827 public ExceptionLoader(boolean isGlobal) { 828 super(isGlobal); 829 } 830 831 @Override getConfigurationDef(String name, Map<String, String> templateMap)832 public ConfigurationDef getConfigurationDef(String name, Map<String, String> templateMap) 833 throws ConfigurationException { 834 throw new ConfigurationException( 835 "Templates are not allowed in direct configuration contexts"); 836 } 837 838 @Override isBundledConfig(String name)839 public boolean isBundledConfig(String name) { 840 throw new RuntimeException( 841 new ConfigurationException( 842 "Templates are not allowed in direct configuration contexts")); 843 } 844 845 @Override findConfigName(String name, String parentName)846 protected String findConfigName(String name, String parentName) { 847 throw new RuntimeException( 848 new ConfigurationException( 849 "Templates are not allowed in direct configuration contexts")); 850 } 851 852 @Override loadIncludedConfiguration( ConfigurationDef def, String parentName, String name, String deviceTagObject, Map<String, String> templateMap, Set<String> templateSeen)853 public void loadIncludedConfiguration( 854 ConfigurationDef def, 855 String parentName, 856 String name, 857 String deviceTagObject, 858 Map<String, String> templateMap, 859 Set<String> templateSeen) 860 throws ConfigurationException { 861 throw new ConfigurationException( 862 "Templates are not allowed in direct configuration contexts", 863 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 864 } 865 866 @Override loadConfiguration( String name, ConfigurationDef def, String deviceTagObject, Map<String, String> templateMap, Set<String> templateSeen)867 public void loadConfiguration( 868 String name, 869 ConfigurationDef def, 870 String deviceTagObject, 871 Map<String, String> templateMap, 872 Set<String> templateSeen) 873 throws ConfigurationException { 874 throw new ConfigurationException( 875 "Templates are not allowed in direct configuration contexts", 876 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 877 } 878 879 @Override trackConfig(String name, ConfigurationDef def)880 protected void trackConfig(String name, ConfigurationDef def) { 881 throw new RuntimeException( 882 new ConfigurationException( 883 "Templates are not allowed in direct configuration contexts")); 884 } 885 886 @Override isTrackableConfig(String name)887 protected boolean isTrackableConfig(String name) { 888 throw new RuntimeException( 889 new ConfigurationException( 890 "Templates are not allowed in direct configuration contexts")); 891 } 892 893 // `private String getAbsolutePath(String root, String name)` is private 894 // and so cannot be overridden 895 896 // `public boolean isGlobalConfig()` not overridden because it is probably 897 // fine to call in this context 898 } 899 900 /** 901 * {@inheritDoc} 902 */ 903 @Override createGlobalConfigurationFromArgs(String[] arrayArgs, List<String> remainingArgs)904 public IGlobalConfiguration createGlobalConfigurationFromArgs(String[] arrayArgs, 905 List<String> remainingArgs) throws ConfigurationException { 906 List<String> listArgs = new ArrayList<String>(arrayArgs.length); 907 IGlobalConfiguration config = internalCreateGlobalConfigurationFromArgs(arrayArgs, 908 listArgs); 909 remainingArgs.addAll(config.setOptionsFromCommandLineArgs(listArgs)); 910 911 return config; 912 } 913 914 /** 915 * Creates a {@link GlobalConfiguration} from the name given in arguments. 916 * <p/> 917 * Note will not populate configuration with values from options 918 * 919 * @param arrayArgs the full list of command line arguments, including the config name 920 * @param optionArgsRef an empty list, that will be populated with the 921 * remaining option arguments 922 * @return a {@link IGlobalConfiguration} created from the args 923 * @throws ConfigurationException 924 */ internalCreateGlobalConfigurationFromArgs(String[] arrayArgs, List<String> optionArgsRef)925 private IGlobalConfiguration internalCreateGlobalConfigurationFromArgs(String[] arrayArgs, 926 List<String> optionArgsRef) throws ConfigurationException { 927 if (arrayArgs.length == 0) { 928 throw new ConfigurationException("Configuration to run was not specified"); 929 } 930 optionArgsRef.addAll(Arrays.asList(arrayArgs)); 931 // first arg is config name 932 final String configName = optionArgsRef.remove(0); 933 ConfigurationDef configDef = getConfigurationDef(configName, true, null); 934 IGlobalConfiguration config = configDef.createGlobalConfiguration(); 935 config.setOriginalConfig(configName); 936 config.setConfigurationFactory(this); 937 return config; 938 } 939 940 /** 941 * {@inheritDoc} 942 */ 943 @Override printHelp(PrintStream out)944 public void printHelp(PrintStream out) { 945 try { 946 loadAllConfigs(true); 947 } catch (ConfigurationException e) { 948 // ignore, should never happen 949 } 950 // sort the configs by name before displaying 951 SortedSet<ConfigurationDef> configDefs = new TreeSet<ConfigurationDef>( 952 new ConfigDefComparator()); 953 configDefs.addAll(mConfigDefMap.values()); 954 StringBuilder sb = new StringBuilder(); 955 for (ConfigurationDef def : configDefs) { 956 sb.append(String.format(" %s: %s\n", def.getName(), def.getDescription())); 957 } 958 out.printf(sb.toString()); 959 } 960 961 /** 962 * {@inheritDoc} 963 */ 964 @Override getConfigList()965 public List<String> getConfigList() { 966 return getConfigList(null, true); 967 } 968 969 /** {@inheritDoc} */ 970 @Override getConfigList(String subPath, boolean loadFromEnv)971 public List<String> getConfigList(String subPath, boolean loadFromEnv) { 972 Set<String> configNames = getConfigSetFromClasspath(subPath); 973 if (loadFromEnv) { 974 // list config on variable path too 975 configNames.addAll(getConfigNamesFromTestCases(subPath)); 976 } 977 // sort the configs by name before adding to list 978 SortedSet<String> configDefs = new TreeSet<String>(); 979 configDefs.addAll(configNames); 980 List<String> configs = new ArrayList<String>(); 981 configs.addAll(configDefs); 982 return configs; 983 } 984 985 /** 986 * Private helper to get the full set of configurations. 987 */ getConfigSetFromClasspath(String subPath)988 private Set<String> getConfigSetFromClasspath(String subPath) { 989 ClassPathScanner cpScanner = new ClassPathScanner(); 990 return cpScanner.getClassPathEntries(new ConfigClasspathFilter(subPath)); 991 } 992 993 /** 994 * Helper to get the test config files from test cases directories from build output. 995 * 996 * @param subPath where to look for configuration. Can be null. 997 */ 998 @VisibleForTesting getConfigNamesFromTestCases(String subPath)999 Set<String> getConfigNamesFromTestCases(String subPath) { 1000 return ConfigurationUtil.getConfigNamesFromDirs(subPath, getExternalTestCasesDirs()); 1001 } 1002 1003 @VisibleForTesting getConfigSetFromClasspathFromJar(String subPath)1004 Map<String, String> getConfigSetFromClasspathFromJar(String subPath) { 1005 ClassPathScanner cpScanner = new ClassPathScanner(); 1006 return cpScanner.getClassPathEntriesFromJar(new ConfigClasspathFilter(subPath)); 1007 } 1008 1009 /** 1010 * Loads all configurations found in classpath and test cases directories. 1011 * 1012 * @param discardExceptions true if any ConfigurationException should be ignored. 1013 * @throws ConfigurationException 1014 */ loadAllConfigs(boolean discardExceptions)1015 public void loadAllConfigs(boolean discardExceptions) throws ConfigurationException { 1016 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 1017 PrintStream ps = new PrintStream(baos); 1018 boolean failed = false; 1019 Set<String> configNames = getConfigSetFromClasspath(null); 1020 // TODO: split the configs into two lists, one from the jar packages and one from test 1021 // cases directories. 1022 configNames.addAll(getConfigNamesFromTestCases(null)); 1023 for (String configName : configNames) { 1024 final ConfigId configId = new ConfigId(configName); 1025 try { 1026 ConfigurationDef configDef = attemptLoad(configId, null); 1027 mConfigDefMap.put(configId, configDef); 1028 } catch (ConfigurationException e) { 1029 ps.printf("Failed to load %s: %s", configName, e.getMessage()); 1030 ps.println(); 1031 failed = true; 1032 } 1033 } 1034 if (failed) { 1035 if (discardExceptions) { 1036 CLog.e("Failure loading configs"); 1037 CLog.e(baos.toString()); 1038 } else { 1039 throw new ConfigurationException(baos.toString()); 1040 } 1041 } 1042 } 1043 1044 /** 1045 * Helper to load a configuration. 1046 */ attemptLoad(ConfigId configId, Map<String, String> templateMap)1047 private ConfigurationDef attemptLoad(ConfigId configId, Map<String, String> templateMap) 1048 throws ConfigurationException { 1049 ConfigurationDef configDef = null; 1050 try { 1051 configDef = getConfigurationDef(configId.name, false, templateMap); 1052 return configDef; 1053 } catch (TemplateResolutionError tre) { 1054 // When a template does not have a default, we try again with known good template 1055 // to make sure file formatting at the very least is fine. 1056 Map<String, String> fakeTemplateMap = new HashMap<String, String>(); 1057 if (templateMap != null) { 1058 fakeTemplateMap.putAll(templateMap); 1059 } 1060 fakeTemplateMap.put(tre.getTemplateKey(), DRY_RUN_TEMPLATE_CONFIG); 1061 // We go recursively in case there are several template to dry run. 1062 return attemptLoad(configId, fakeTemplateMap); 1063 } 1064 } 1065 1066 /** 1067 * {@inheritDoc} 1068 */ 1069 @Override printHelpForConfig(String[] args, boolean importantOnly, PrintStream out)1070 public void printHelpForConfig(String[] args, boolean importantOnly, PrintStream out) { 1071 try { 1072 IConfiguration config = 1073 internalCreateConfigurationFromArgs( 1074 args, new ArrayList<String>(args.length), null, null); 1075 config.printCommandUsage(importantOnly, out); 1076 } catch (ConfigurationException e) { 1077 // config must not be specified. Print generic help 1078 printHelp(out); 1079 } 1080 } 1081 1082 /** 1083 * {@inheritDoc} 1084 */ 1085 @Override dumpConfig(String configName, PrintStream out)1086 public void dumpConfig(String configName, PrintStream out) { 1087 try { 1088 InputStream configStream = getConfigStream(configName); 1089 StreamUtil.copyStreams(configStream, out); 1090 } catch (ConfigurationException e) { 1091 CLog.e(e); 1092 } catch (IOException e) { 1093 CLog.e(e); 1094 } 1095 } 1096 1097 /** 1098 * Return the path prefix of config xml files on classpath 1099 * 1100 * <p>Exposed so unit tests can mock. 1101 * 1102 * @return {@link String} path with trailing / 1103 */ getConfigPrefix()1104 protected String getConfigPrefix() { 1105 return CONFIG_PREFIX; 1106 } 1107 1108 /** 1109 * Loads an InputStream for given config name 1110 * 1111 * @param name the configuration name to load 1112 * @return a {@link BufferedInputStream} for reading config contents 1113 * @throws ConfigurationException if config could not be found 1114 */ getConfigStream(String name)1115 protected BufferedInputStream getConfigStream(String name) throws ConfigurationException { 1116 InputStream configStream = getBundledConfigStream(name); 1117 if (configStream == null) { 1118 // now try to load from file 1119 try { 1120 configStream = new FileInputStream(name); 1121 } catch (FileNotFoundException e) { 1122 throw new ConfigurationException(String.format("Could not find configuration '%s'", 1123 name)); 1124 } 1125 } 1126 // buffer input for performance - just in case config file is large 1127 return new BufferedInputStream(configStream); 1128 } 1129 getBundledConfigStream(String name)1130 protected InputStream getBundledConfigStream(String name) { 1131 String extension = FileUtil.getExtension(name); 1132 if (Strings.isNullOrEmpty(extension)) { 1133 // If the default name doesn't have an extension, search all possible extensions. 1134 for (String supportExtension : SUPPORTED_EXTENSIONS) { 1135 InputStream res = 1136 getClass() 1137 .getResourceAsStream( 1138 String.format( 1139 "/%s%s%s", 1140 getConfigPrefix(), name, supportExtension)); 1141 if (res != null) { 1142 return res; 1143 } 1144 } 1145 return null; 1146 } 1147 // Check directly with extension if it has one. 1148 return getClass().getResourceAsStream(String.format("/%s%s", getConfigPrefix(), name)); 1149 } 1150 1151 /** 1152 * Utility method that checks that all configs can be loaded, parsed, and 1153 * all option values set. 1154 * Only exposed so that depending project can validate their configs. 1155 * Should not be exposed in the console. 1156 * 1157 * @throws ConfigurationException if one or more configs failed to load 1158 */ loadAndPrintAllConfigs()1159 public void loadAndPrintAllConfigs() throws ConfigurationException { 1160 loadAllConfigs(false); 1161 boolean failed = false; 1162 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 1163 PrintStream ps = new PrintStream(baos); 1164 1165 for (ConfigurationDef def : mConfigDefMap.values()) { 1166 try { 1167 def.createConfiguration().printCommandUsage(false, 1168 new PrintStream(StreamUtil.nullOutputStream())); 1169 } catch (ConfigurationException e) { 1170 if (e.getCause() != null && 1171 e.getCause() instanceof ClassNotFoundException) { 1172 ClassNotFoundException cnfe = (ClassNotFoundException) e.getCause(); 1173 String className = cnfe.getLocalizedMessage(); 1174 // Some Cts configs are shipped with Trade Federation, we exclude those from 1175 // the failure since these packages are not available for loading. 1176 if (className != null && className.startsWith("com.android.cts.")) { 1177 CLog.w("Could not confirm %s: %s because not part of Trade Federation " 1178 + "packages.", def.getName(), e.getMessage()); 1179 continue; 1180 } 1181 } else if (Pattern.matches(CONFIG_ERROR_PATTERN, e.getMessage())) { 1182 // If options are inside configuration object tag we are able to validate them 1183 if (!e.getMessage().contains("com.android.") && 1184 !e.getMessage().contains("com.google.android.")) { 1185 // We cannot confirm if an option is indeed missing since a template of 1186 // option only is possible to avoid repetition in configuration with the 1187 // same base. 1188 CLog.w("Could not confirm %s: %s", def.getName(), e.getMessage()); 1189 continue; 1190 } 1191 } 1192 ps.printf("Failed to print %s: %s", def.getName(), e.getMessage()); 1193 ps.println(); 1194 failed = true; 1195 } 1196 } 1197 if (failed) { 1198 throw new ConfigurationException(baos.toString()); 1199 } 1200 } 1201 1202 /** 1203 * Exposed for testing. Return a copy of the Map. 1204 */ getMapConfig()1205 protected Map<ConfigId, ConfigurationDef> getMapConfig() { 1206 // We return a copy to ensure it is not modified outside 1207 return new HashMap<ConfigId, ConfigurationDef>(mConfigDefMap); 1208 } 1209 1210 /** In some particular case, we need to clear the map. */ 1211 @VisibleForTesting clearMapConfig()1212 public void clearMapConfig() { 1213 mConfigDefMap.clear(); 1214 } 1215 1216 /** Reorder the args so that template:map args are all moved to the front. */ 1217 @VisibleForTesting reorderArgs(String[] args)1218 protected String[] reorderArgs(String[] args) { 1219 List<String> nonTemplateArgs = new ArrayList<String>(); 1220 List<String> reorderedArgs = new ArrayList<String>(); 1221 String[] reorderedArgsArray = new String[args.length]; 1222 String arg; 1223 1224 // First arg is the config. 1225 if (args.length > 0) { 1226 reorderedArgs.add(args[0]); 1227 } 1228 1229 // Split out the template and non-template args so we can add 1230 // non-template args at the end while maintaining their order. 1231 for (int i = 1; i < args.length; i++) { 1232 arg = args[i]; 1233 if (arg.equals("--template:map")) { 1234 // We need to account for these two types of template:map args. 1235 // --template:map tm=tm1 1236 // --template:map tm tm1 1237 reorderedArgs.add(arg); 1238 for (int j = i + 1; j < args.length; j++) { 1239 if (args[j].startsWith("-")) { 1240 break; 1241 } else { 1242 reorderedArgs.add(args[j]); 1243 i++; 1244 } 1245 } 1246 } else { 1247 nonTemplateArgs.add(arg); 1248 } 1249 } 1250 reorderedArgs.addAll(nonTemplateArgs); 1251 return reorderedArgs.toArray(reorderedArgsArray); 1252 } 1253 } 1254