1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.config; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.config.OptionSetter.OptionFieldsForName; 21 import com.android.tradefed.config.remote.ExtendedFile; 22 import com.android.tradefed.config.remote.IRemoteFileResolver; 23 import com.android.tradefed.config.remote.IRemoteFileResolver.RemoteFileResolverArgs; 24 import com.android.tradefed.config.remote.IRemoteFileResolver.ResolvedFile; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.error.HarnessRuntimeException; 27 import com.android.tradefed.error.IHarnessException; 28 import com.android.tradefed.invoker.logger.CurrentInvocation; 29 import com.android.tradefed.invoker.logger.CurrentInvocation.InvocationInfo; 30 import com.android.tradefed.invoker.logger.InvocationLocal; 31 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.result.error.ErrorIdentifier; 34 import com.android.tradefed.result.error.InfraErrorIdentifier; 35 import com.android.tradefed.testtype.suite.ITestSuite; 36 import com.android.tradefed.util.FileUtil; 37 import com.android.tradefed.util.IDisableable; 38 import com.android.tradefed.util.MultiMap; 39 import com.android.tradefed.util.ZipUtil; 40 import com.android.tradefed.util.ZipUtil2; 41 42 import com.google.common.collect.ImmutableList; 43 import com.google.common.collect.ImmutableMap; 44 import com.google.common.collect.Maps; 45 46 import java.io.File; 47 import java.io.IOException; 48 import java.lang.reflect.Field; 49 import java.net.URI; 50 import java.net.URISyntaxException; 51 import java.util.ArrayList; 52 import java.util.Collection; 53 import java.util.HashMap; 54 import java.util.HashSet; 55 import java.util.LinkedHashMap; 56 import java.util.List; 57 import java.util.Map; 58 import java.util.Map.Entry; 59 import java.util.ServiceLoader; 60 import java.util.Set; 61 import java.util.function.Supplier; 62 63 import javax.annotation.Nullable; 64 import javax.annotation.concurrent.GuardedBy; 65 import javax.annotation.concurrent.ThreadSafe; 66 67 /** 68 * Class that helps resolving path to remote files. 69 * 70 * <p>For example: gs://bucket/path/file.txt will be resolved by downloading the file from the GCS 71 * bucket. 72 * 73 * <p>New protocols should be added to META_INF/services. 74 */ 75 public class DynamicRemoteFileResolver { 76 77 // Query key for requesting to unzip a downloaded file automatically. 78 public static final String UNZIP_KEY = "unzip"; 79 // Query key for requesting a download to be optional, so if it fails we don't replace it. 80 public static final String OPTIONAL_KEY = "optional"; 81 // Query key for the option name being resolved. 82 public static final String OPTION_NAME_KEY = "option_name"; 83 // Query key for the parallel setting 84 public static final String OPTION_PARALLEL_KEY = "parallel"; 85 86 /** 87 * Loads file resolvers using a dedicated {@link ServiceFileResolverLoader} that is scoped to 88 * each invocation. 89 */ 90 // TODO(hzalek): Store a DynamicRemoteFileResolver instance per invocation to avoid locals. 91 private static final FileResolverLoader DEFAULT_FILE_RESOLVER_LOADER = 92 new FileResolverLoader() { 93 private final InvocationLocal<FileResolverLoader> mInvocationLoader = 94 new InvocationLocal<FileResolverLoader>() { 95 @Override 96 protected FileResolverLoader initialValue() { 97 return new ServiceFileResolverLoader(); 98 } 99 }; 100 101 @Override 102 public IRemoteFileResolver load(String scheme, Map<String, String> config) { 103 return mInvocationLoader.get().load(scheme, config); 104 } 105 }; 106 107 private final FileResolverLoader mFileResolverLoader; 108 private final boolean mAllowParallelization; 109 110 private Map<String, OptionFieldsForName> mOptionMap; 111 // Populated from {@link ICommandOptions#getDynamicDownloadArgs()} 112 private Map<String, String> mExtraArgs = new LinkedHashMap<>(); 113 private ITestDevice mDevice; 114 private List<ExtendedFile> mParallelExtendedFiles = new ArrayList<>(); 115 DynamicRemoteFileResolver()116 public DynamicRemoteFileResolver() { 117 this(DEFAULT_FILE_RESOLVER_LOADER); 118 } 119 DynamicRemoteFileResolver(boolean allowParallel)120 public DynamicRemoteFileResolver(boolean allowParallel) { 121 this(DEFAULT_FILE_RESOLVER_LOADER, allowParallel); 122 } 123 124 @VisibleForTesting DynamicRemoteFileResolver(FileResolverLoader loader)125 public DynamicRemoteFileResolver(FileResolverLoader loader) { 126 this(loader, false); 127 } 128 129 @VisibleForTesting DynamicRemoteFileResolver(FileResolverLoader loader, boolean allowParallel)130 public DynamicRemoteFileResolver(FileResolverLoader loader, boolean allowParallel) { 131 this.mFileResolverLoader = loader; 132 this.mAllowParallelization = allowParallel; 133 } 134 135 /** Sets the map of options coming from {@link OptionSetter} */ setOptionMap(Map<String, OptionFieldsForName> optionMap)136 public void setOptionMap(Map<String, OptionFieldsForName> optionMap) { 137 mOptionMap = optionMap; 138 } 139 140 /** Sets the device under tests */ setDevice(ITestDevice device)141 public void setDevice(ITestDevice device) { 142 mDevice = device; 143 } 144 145 /** Add extra args for the query. */ addExtraArgs(Map<String, String> extraArgs)146 public void addExtraArgs(Map<String, String> extraArgs) { 147 mExtraArgs.putAll(extraArgs); 148 } 149 getParallelDownloads()150 public List<ExtendedFile> getParallelDownloads() { 151 return mParallelExtendedFiles; 152 } 153 154 /** 155 * Runs through all the {@link File} option type and check if their path should be resolved. 156 * 157 * @return The list of {@link File} that was resolved that way. 158 * @throws BuildRetrievalError 159 */ validateRemoteFilePath()160 public final Set<File> validateRemoteFilePath() throws BuildRetrievalError { 161 Set<File> downloadedFiles = new HashSet<>(); 162 try { 163 Map<Field, Object> fieldSeen = new HashMap<>(); 164 for (Map.Entry<String, OptionFieldsForName> optionPair : mOptionMap.entrySet()) { 165 final OptionFieldsForName optionFields = optionPair.getValue(); 166 for (Map.Entry<Object, Field> fieldEntry : optionFields) { 167 168 final Object obj = fieldEntry.getKey(); 169 if (obj instanceof IDisableable && ((IDisableable) obj).isDisabled()) { 170 continue; 171 } 172 final Field field = fieldEntry.getValue(); 173 final Option option = field.getAnnotation(Option.class); 174 if (option == null) { 175 continue; 176 } 177 // At this point, we know this is an option field; make sure it's set 178 field.setAccessible(true); 179 final Object value; 180 try { 181 value = field.get(obj); 182 if (value == null) { 183 continue; 184 } 185 } catch (IllegalAccessException e) { 186 throw new BuildRetrievalError( 187 String.format("internal error: %s", e.getMessage()), 188 InfraErrorIdentifier.ARTIFACT_UNSUPPORTED_PATH); 189 } 190 191 if (fieldSeen.get(field) != null && fieldSeen.get(field).equals(obj)) { 192 continue; 193 } 194 // Keep track of the field set on each object 195 fieldSeen.put(field, obj); 196 197 // The below contains unchecked casts that are mostly safe because we add/remove 198 // items of a type already in the collection; assuming they're not instances of 199 // some subclass of File. This is unlikely since we populate the items during 200 // option injection. The possibility still exists that constructors of 201 // initialized objects add objects that are instances of a File subclass. A 202 // safer approach would be to have a custom type that can be deferenced to 203 // access the resolved target file. This would also have the benefit of not 204 // having to modify any user collections and preserve the ordering. 205 206 if (value instanceof File) { 207 File consideredFile = (File) value; 208 ResolvedFile resolvedFile = resolveRemoteFiles(consideredFile, option); 209 if (resolvedFile != null) { 210 File downloadedFile = resolvedFile.getResolvedFile(); 211 if (resolvedFile.shouldCleanUp()) { 212 downloadedFiles.add(downloadedFile); 213 } 214 // Replace the field value 215 try { 216 field.set(obj, downloadedFile); 217 } catch (IllegalAccessException e) { 218 CLog.e(e); 219 throw new BuildRetrievalError( 220 String.format( 221 "Failed to download %s due to '%s'", 222 consideredFile.getPath(), e.getMessage()), 223 e); 224 } 225 } 226 } else if (value instanceof Collection) { 227 @SuppressWarnings("unchecked") // Mostly-safe, see above comment. 228 Collection<Object> c = (Collection<Object>) value; 229 synchronized (c) { 230 Collection<Object> copy = new ArrayList<>(c); 231 for (Object o : copy) { 232 if (o instanceof File) { 233 File consideredFile = (File) o; 234 ResolvedFile resolvedFile = 235 resolveRemoteFiles(consideredFile, option); 236 if (resolvedFile != null) { 237 File downloadedFile = resolvedFile.getResolvedFile(); 238 if (resolvedFile.shouldCleanUp()) { 239 downloadedFiles.add(downloadedFile); 240 } 241 // TODO: See if order could be preserved. 242 c.remove(consideredFile); 243 c.add(downloadedFile); 244 } 245 } 246 } 247 } 248 } else if (value instanceof Map) { 249 @SuppressWarnings("unchecked") // Mostly-safe, see above comment. 250 Map<Object, Object> m = (Map<Object, Object>) value; 251 Map<Object, Object> copy = new LinkedHashMap<>(m); 252 for (Entry<Object, Object> entry : copy.entrySet()) { 253 Object key = entry.getKey(); 254 Object val = entry.getValue(); 255 256 Object finalKey = key; 257 Object finalVal = val; 258 if (key instanceof File) { 259 ResolvedFile resolved = resolveRemoteFiles((File) key, option); 260 if (resolved != null) { 261 File downloaded = resolved.getResolvedFile(); 262 if (resolved.shouldCleanUp()) { 263 downloadedFiles.add(downloaded); 264 } 265 finalKey = downloaded; 266 } 267 } 268 if (val instanceof File) { 269 ResolvedFile resolved = resolveRemoteFiles((File) val, option); 270 if (resolved != null) { 271 File downloaded = resolved.getResolvedFile(); 272 if (resolved.shouldCleanUp()) { 273 downloadedFiles.add(downloaded); 274 } 275 finalVal = downloaded; 276 } 277 } 278 279 m.remove(entry.getKey()); 280 m.put(finalKey, finalVal); 281 } 282 } else if (value instanceof MultiMap) { 283 @SuppressWarnings("unchecked") // Mostly-safe, see above comment. 284 MultiMap<Object, Object> m = (MultiMap<Object, Object>) value; 285 synchronized (m) { 286 MultiMap<Object, Object> copy = new MultiMap<>(m); 287 for (Object key : copy.keySet()) { 288 List<Object> mapValues = copy.get(key); 289 290 m.remove(key); 291 Object finalKey = key; 292 if (key instanceof File) { 293 ResolvedFile resolved = resolveRemoteFiles((File) key, option); 294 if (resolved != null) { 295 File downloaded = resolved.getResolvedFile(); 296 if (resolved.shouldCleanUp()) { 297 downloadedFiles.add(downloaded); 298 } 299 finalKey = downloaded; 300 } 301 } 302 for (Object mapValue : mapValues) { 303 if (mapValue instanceof File) { 304 ResolvedFile resolvedFile = 305 resolveRemoteFiles((File) mapValue, option); 306 if (resolvedFile != null) { 307 if (resolvedFile.shouldCleanUp()) { 308 downloadedFiles.add(resolvedFile.getResolvedFile()); 309 } 310 mapValue = resolvedFile.getResolvedFile(); 311 } 312 } 313 m.put(finalKey, mapValue); 314 } 315 } 316 } 317 } 318 } 319 } 320 } catch (RuntimeException | BuildRetrievalError e) { 321 // Clean up the files before throwing 322 for (File f : downloadedFiles) { 323 FileUtil.recursiveDelete(f); 324 } 325 throw e; 326 } 327 return downloadedFiles; 328 } 329 330 /** 331 * Download the files matching given filters in a remote zip file. 332 * 333 * <p>A file inside the remote zip file is only downloaded if its path matches any of the 334 * include filters but not the exclude filters. 335 * 336 * @param destDir the file to place the downloaded contents into. 337 * @param remoteZipFilePath the remote path to the zip file to download, relative to an 338 * implementation specific root. 339 * @param includeFilters a list of regex strings to download matching files. A file's path 340 * matching any filter will be downloaded. 341 * @param excludeFilters a list of regex strings to skip downloading matching files. A file's 342 * path matching any filter will not be downloaded. 343 * @throws BuildRetrievalError if files could not be downloaded. 344 */ resolvePartialDownloadZip( File destDir, String remoteZipFilePath, List<String> includeFilters, List<String> excludeFilters)345 public void resolvePartialDownloadZip( 346 File destDir, 347 String remoteZipFilePath, 348 List<String> includeFilters, 349 List<String> excludeFilters) 350 throws BuildRetrievalError { 351 Map<String, String> queryArgs; 352 String protocol; 353 try { 354 URI uri = new URI(remoteZipFilePath); 355 protocol = uri.getScheme(); 356 queryArgs = parseQuery(uri.getQuery()); 357 } catch (URISyntaxException e) { 358 throw new BuildRetrievalError( 359 String.format( 360 "Failed to parse the remote zip file path: %s", remoteZipFilePath), 361 e); 362 } 363 queryArgs.put("partial_download_dir", destDir.getAbsolutePath()); 364 if (includeFilters != null) { 365 queryArgs.put("include_filters", String.join(";", includeFilters)); 366 } 367 if (excludeFilters != null) { 368 queryArgs.put("exclude_filters", String.join(";", excludeFilters)); 369 } 370 371 // TODO(rbraunstein): Consider changing to take map of args. 372 for (String key : ImmutableList.of(ITestSuite.ENABLE_RESOLVE_SYM_LINKS)) { 373 String value = mExtraArgs.get(key); 374 if (value != null) { 375 queryArgs.put(key, value); 376 } 377 } 378 // Downloaded individual files should be saved to destDir, return value is not needed. 379 try (CloseableTraceScope ignored = 380 new CloseableTraceScope( 381 String.format( 382 "resolvePartialDownload %s, %s, %s", 383 remoteZipFilePath, protocol, queryArgs))) { 384 385 IRemoteFileResolver resolver = getResolver(protocol); 386 resolver.setPrimaryDevice(mDevice); 387 RemoteFileResolverArgs args = new RemoteFileResolverArgs(); 388 args.setConsideredFile(new File(remoteZipFilePath)) 389 .addQueryArgs(queryArgs) 390 .setDestinationDir(destDir); 391 resolver.resolveRemoteFile(args); 392 } catch (BuildRetrievalError e) { 393 if (isOptional(queryArgs)) { 394 CLog.d( 395 "Failed to partially download '%s' but marked optional so skipping: %s", 396 remoteZipFilePath, e.getMessage()); 397 return; 398 } 399 400 throw e; 401 } 402 } 403 getResolver(String protocol)404 private IRemoteFileResolver getResolver(String protocol) throws BuildRetrievalError { 405 try { 406 return mFileResolverLoader.load(protocol, mExtraArgs); 407 } catch (ResolverLoadingException e) { 408 throw new BuildRetrievalError( 409 String.format("Could not load resolver for protocol %s", protocol), e); 410 } 411 } 412 413 @VisibleForTesting getGlobalConfig()414 IGlobalConfiguration getGlobalConfig() { 415 return GlobalConfiguration.getInstance(); 416 } 417 418 /** 419 * Utility that allows to check whether or not a file should be unzip and unzip it if required. 420 */ unzipIfRequired(File downloadedFile, Map<String, String> query)421 public static final File unzipIfRequired(File downloadedFile, Map<String, String> query) 422 throws IOException { 423 String unzipValue = query.get(UNZIP_KEY); 424 if (unzipValue != null && "true".equals(unzipValue.toLowerCase())) { 425 if (downloadedFile.isDirectory()) { 426 return downloadedFile; 427 } 428 // File was requested to be unzipped. 429 try (CloseableTraceScope ignored = 430 new CloseableTraceScope("unzip " + downloadedFile.getName())) { 431 if (ZipUtil.isZipFileValid(downloadedFile, false)) { 432 File extractedDir = 433 FileUtil.createTempDir( 434 FileUtil.getBaseName(downloadedFile.getName()), 435 CurrentInvocation.getInfo(InvocationInfo.WORK_FOLDER)); 436 ZipUtil2.extractZip(downloadedFile, extractedDir); 437 FileUtil.deleteFile(downloadedFile); 438 return extractedDir; 439 } else { 440 throw new IOException( 441 String.format( 442 "%s was requested to be unzipped but is not a valid zip.", 443 downloadedFile)); 444 } 445 } 446 } 447 // Return the original file untouched 448 return downloadedFile; 449 } 450 resolveRemoteFiles(File consideredFile, Option option)451 private ResolvedFile resolveRemoteFiles(File consideredFile, Option option) 452 throws BuildRetrievalError { 453 File fileToResolve; 454 String path = consideredFile.getPath(); 455 String protocol; 456 Map<String, String> query; 457 try { 458 URI uri = new URI(path.replace('\\', '/')); 459 protocol = uri.getScheme(); 460 query = parseQuery(uri.getQuery()); 461 fileToResolve = new File(protocol + ":" + uri.getPath()); 462 } catch (URISyntaxException e) { 463 CLog.e(e); 464 throw new BuildRetrievalError( 465 e.getMessage(), e, InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 466 } 467 query.put(OPTION_NAME_KEY, option.name()); 468 if (!mAllowParallelization) { 469 query.put(OPTION_PARALLEL_KEY, "false"); 470 } 471 472 try { 473 IRemoteFileResolver resolver = getResolver(protocol); 474 if (resolver == null) { 475 return null; 476 } 477 CLog.d("Considering option '%s' with path: '%s' for download.", option.name(), path); 478 resolver.setPrimaryDevice(mDevice); 479 RemoteFileResolverArgs args = new RemoteFileResolverArgs(); 480 args.setConsideredFile(fileToResolve).addQueryArgs(query); 481 ResolvedFile resolvedFile = resolver.resolveRemoteFile(args); 482 if (resolvedFile != null && resolvedFile.getResolvedFile() instanceof ExtendedFile) { 483 // It is possible for dynamic download to download in parallel 484 // as long as they do so for the expected output file location 485 ExtendedFile trackingFile = (ExtendedFile) resolvedFile.getResolvedFile(); 486 if (trackingFile.isDownloadingInParallel()) { 487 mParallelExtendedFiles.add(trackingFile); 488 } 489 } 490 return resolvedFile; 491 } catch (BuildRetrievalError e) { 492 if (isOptional(query)) { 493 CLog.d( 494 "Failed to resolve '%s' but marked optional so skipping: %s", 495 fileToResolve, e.getMessage()); 496 return null; 497 } 498 499 throw e; 500 } 501 } 502 503 /** 504 * Parse a URL query style. Delimited by &, and map values represented by =. Example: 505 * ?key=value&key2=value2 506 */ parseQuery(String query)507 private Map<String, String> parseQuery(String query) { 508 Map<String, String> values = new HashMap<>(); 509 if (query == null) { 510 return values; 511 } 512 for (String maps : query.split("&")) { 513 String[] keyVal = maps.split("="); 514 values.put(keyVal[0], keyVal[1]); 515 } 516 return values; 517 } 518 519 /** Whether or not a link was requested as optional. */ isOptional(Map<String, String> query)520 private boolean isOptional(Map<String, String> query) { 521 String value = query.get(OPTIONAL_KEY); 522 if (value == null) { 523 return false; 524 } 525 return "true".equals(value.toLowerCase()); 526 } 527 528 /** Loads implementations of {@link IRemoteFileResolver}. */ 529 @VisibleForTesting 530 public interface FileResolverLoader { 531 /** 532 * Loads a resolver that can handle the provided scheme. 533 * 534 * @param scheme the URI scheme that the loaded resolver is expected to handle. 535 * @param config a map of all dynamic resolver configuration key-value pairs specified by 536 * the 'dynamic-resolver-args' TF command-line flag. 537 * @throws ResolverLoadingException if the resolver that handles the specified scheme cannot 538 * be loaded and/or initialized. 539 */ 540 @Nullable load(String scheme, Map<String, String> config)541 IRemoteFileResolver load(String scheme, Map<String, String> config); 542 } 543 544 /** Exception thrown if a resolver cannot be loaded or initialized. */ 545 @VisibleForTesting 546 static final class ResolverLoadingException extends HarnessRuntimeException { ResolverLoadingException(@ullable String message, ErrorIdentifier errorId)547 public ResolverLoadingException(@Nullable String message, ErrorIdentifier errorId) { 548 super(message, errorId); 549 } 550 ResolverLoadingException(@ullable String message, IHarnessException cause)551 public ResolverLoadingException(@Nullable String message, IHarnessException cause) { 552 super(message, cause); 553 } 554 } 555 556 /** 557 * Loads and caches file resolvers using the service loading facility. 558 * 559 * <p>This implementation uses the service loading facility to find and cache available 560 * resolvers on the first call to {@code load}. 561 * 562 * <p>Any {@link Option}-annotated fields defined in loaded resolvers are initialized from the 563 * provided key-value pairs using the standard TF option-setting mechanism. Resolvers can define 564 * options that themselves require resolution as long as it causes no cycles during 565 * initialization. 566 * 567 * <p>Resolvers are loaded eagerly using ServiceLoader but have their options initialized only 568 * when first used. This avoids exceptions due to missing options in resolvers that are 569 * available on the class path but never used to load any files. 570 * 571 * <p>This implementation is thread-safe and ensures that any loaded resolvers are loaded at 572 * most once per instance. 573 */ 574 @ThreadSafe 575 @VisibleForTesting 576 static final class ServiceFileResolverLoader implements FileResolverLoader { 577 // We need the indirection since in production we use the context class loader that is 578 // defined when loading and not the one at construction. 579 private final Supplier<ClassLoader> mClassLoaderSupplier; 580 581 @GuardedBy("this") 582 private @Nullable LoaderState mLoaderState; 583 ServiceFileResolverLoader()584 ServiceFileResolverLoader() { 585 mClassLoaderSupplier = () -> Thread.currentThread().getContextClassLoader(); 586 } 587 ServiceFileResolverLoader(ClassLoader classLoader)588 ServiceFileResolverLoader(ClassLoader classLoader) { 589 mClassLoaderSupplier = () -> classLoader; 590 } 591 592 @Override load(String scheme, Map<String, String> config)593 public synchronized IRemoteFileResolver load(String scheme, Map<String, String> config) { 594 if (mLoaderState != null) { 595 return mLoaderState.getAndInit(scheme); 596 } 597 598 // We use an intermediate map because the ImmutableMap builder throws if we add multiple 599 // entries with the same key. Note that we don't worry about setting any state that 600 // prevents this code from re-executing since failures loading service providers throws 601 // an Error which bubbles all the way to the top. 602 Map<String, IRemoteFileResolver> resolvers = new HashMap<>(); 603 ServiceLoader<IRemoteFileResolver> serviceLoader = 604 ServiceLoader.load(IRemoteFileResolver.class, mClassLoaderSupplier.get()); 605 606 for (IRemoteFileResolver resolver : serviceLoader) { 607 resolvers.putIfAbsent(resolver.getSupportedProtocol(), resolver); 608 } 609 610 mLoaderState = new LoaderState(resolvers, config); 611 return mLoaderState.getAndInit(scheme); 612 } 613 614 /** Stores the state of loaded file resolvers. */ 615 private static final class LoaderState { 616 private final ImmutableMap<String, String> mConfig; 617 private final ImmutableMap<String, ResolverState> mState; 618 LoaderState(Map<String, IRemoteFileResolver> resolvers, Map<String, String> config)619 LoaderState(Map<String, IRemoteFileResolver> resolvers, Map<String, String> config) { 620 this.mState = 621 ImmutableMap.copyOf( 622 Maps.transformValues(resolvers, r -> new ResolverState(r))); 623 this.mConfig = ImmutableMap.copyOf(config); 624 } 625 626 /** Returns an initialized resolver instance for the specified scheme. */ 627 @Nullable getAndInit(String scheme)628 IRemoteFileResolver getAndInit(String scheme) { 629 ResolverState state = mState.get(scheme); 630 if (state == null) { 631 return null; 632 } 633 634 return state.getAndInit(this); 635 } 636 resolve(IRemoteFileResolver resolver)637 void resolve(IRemoteFileResolver resolver) 638 throws ConfigurationException, BuildRetrievalError { 639 // The device isn't set when resolving dynamic options because we don't want to load 640 // device-specific configuration when initializing pseudo-static resolvers that 641 // could out-live a particular device. 642 OptionSetter setter = new OptionSetter(resolver); 643 644 for (Map.Entry<String, String> e : mConfig.entrySet()) { 645 String name = e.getKey(); 646 647 // Note that we don't throw for options that don't exist. 648 if (setter.fieldsForArgNoThrow(name) == null) { 649 // TODO(hzalek): Consider throwing when the option doesn't exist and is 650 // qualified using one of the option source's aliases. 651 // option name uses one of 652 // the option source's aliases 653 continue; 654 } 655 656 if (setter.isMapOption(name)) { 657 throw new ConfigurationException( 658 "Map options are not supported: " + name, 659 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 660 } 661 662 setter.setOptionValue(name, e.getValue()); 663 } 664 665 Collection<String> missingOptions = setter.getUnsetMandatoryOptions(); 666 if (!missingOptions.isEmpty()) { 667 throw new ConfigurationException( 668 String.format( 669 "Found missing mandatory options %s for resolver %s", 670 missingOptions, resolver.toString()), 671 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 672 } 673 674 DynamicRemoteFileResolver dynamicResolver = 675 new DynamicRemoteFileResolver((scheme, unused) -> getAndInit(scheme)); 676 dynamicResolver.addExtraArgs(mConfig); 677 setter.validateRemoteFilePath(dynamicResolver); 678 } 679 680 /** Stores the resolver and its initialization state. */ 681 static final class ResolverState { 682 final IRemoteFileResolver mResolver; 683 684 /** 685 * The initialization state where {@code null} means never initialized, {@code 686 * false} means started, and {@code true} means done. 687 */ 688 @Nullable Boolean mDone; 689 690 /** 691 * The exception thrown when initializing the resolver to ensure that we only do it 692 * once. 693 */ 694 @Nullable ResolverLoadingException mException; 695 ResolverState(IRemoteFileResolver resolver)696 ResolverState(IRemoteFileResolver resolver) { 697 this.mResolver = resolver; 698 } 699 getAndInit(LoaderState context)700 IRemoteFileResolver getAndInit(LoaderState context) { 701 if (Boolean.TRUE.equals(mDone)) { 702 return getOrThrow(); 703 } 704 705 if (Boolean.FALSE.equals(mDone)) { 706 // No need to catch or store the exception since it gets thrown in the 707 // recursive 708 // call to the dynamic resolver as a BuildRetrievalError which we already 709 // catch. 710 throw new ResolverLoadingException( 711 "Cycle detected while initializing resolver options: " 712 + mResolver.toString(), 713 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 714 } 715 716 CLog.i("Initializing file resolver options: %s", mResolver); 717 mDone = Boolean.FALSE; 718 719 try { 720 context.resolve(mResolver); 721 } catch (BuildRetrievalError | ConfigurationException e) { 722 mException = 723 new ResolverLoadingException( 724 "Could not initialize resolver options: " 725 + mResolver.toString(), 726 e); 727 throw mException; 728 } finally { 729 mDone = Boolean.TRUE; 730 } 731 732 return mResolver; 733 } 734 getOrThrow()735 private IRemoteFileResolver getOrThrow() { 736 if (mException != null) { 737 throw mException; 738 } 739 return mResolver; 740 } 741 } 742 } 743 } 744 } 745