1 /* 2 * Copyright 2022 Google LLC 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.libraries.mobiledatadownload.populator; 17 18 import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncCallable; 19 import static com.google.common.util.concurrent.Futures.immediateFailedFuture; 20 import static com.google.common.util.concurrent.Futures.immediateVoidFuture; 21 22 import android.content.Context; 23 import android.net.Uri; 24 import androidx.annotation.VisibleForTesting; 25 import com.google.android.libraries.mobiledatadownload.AggregateException; 26 import com.google.android.libraries.mobiledatadownload.DownloadException; 27 import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; 28 import com.google.android.libraries.mobiledatadownload.FileGroupPopulator; 29 import com.google.android.libraries.mobiledatadownload.Flags; 30 import com.google.android.libraries.mobiledatadownload.Logger; 31 import com.google.android.libraries.mobiledatadownload.MobileDataDownload; 32 import com.google.android.libraries.mobiledatadownload.downloader.CheckContentChangeRequest; 33 import com.google.android.libraries.mobiledatadownload.downloader.CheckContentChangeResponse; 34 import com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints; 35 import com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest; 36 import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; 37 import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; 38 import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; 39 import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; 40 import com.google.android.libraries.mobiledatadownload.logger.FileGroupPopulatorLogger; 41 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; 42 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; 43 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; 44 import com.google.common.base.Optional; 45 import com.google.common.base.Preconditions; 46 import com.google.common.base.Supplier; 47 import com.google.common.collect.ImmutableList; 48 import com.google.common.util.concurrent.ExecutionSequencer; 49 import com.google.common.util.concurrent.FutureCallback; 50 import com.google.common.util.concurrent.ListenableFuture; 51 import com.google.errorprone.annotations.CanIgnoreReturnValue; 52 import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; 53 import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; 54 import com.google.mobiledatadownload.DownloadConfigProto.ManifestFileFlag; 55 import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; 56 import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; 57 import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping.Status; 58 import java.io.IOException; 59 import java.util.concurrent.Executor; 60 import java.util.concurrent.atomic.AtomicReference; 61 import javax.annotation.Nullable; 62 import javax.inject.Singleton; 63 64 /** 65 * File group populator that gets {@link ManifestFileFlag} from the caller, downloads the 66 * corresponding manifest file, parses the file into {@link ManifestConfig}, and processes {@link 67 * ManifestConfig}. 68 * 69 * <p>Client can set an optional {@link ManifestConfigOverrider} to return a list of {@link 70 * DataFileGroup}'s to be added to MDD. The overrider will enable the on device targeting. 71 * 72 * <p>Client is responsible of reading {@link ManifestFileFlag} from P/H, and this populator would 73 * get the flag via {@link Supplier<ManifestFileFlag>}. 74 * 75 * <p>On calling {@link #refreshFileGroups(MobileDataDownload)}, this populator would sync up with 76 * server to verify if the manifest file on server has changed since last download. It would 77 * re-download the file if a newer version is available. More specifically, there are 3 scenarios: 78 * 79 * <ul> 80 * <li>1. Current file up-to-date, status PENDING. Resume download. 81 * <li>2. Current file up-to-date, status (DOWNLOADED | COMMITTED). No download will happen. 82 * <li>3. Current file outdated. Delete the outdated file and re-download. 83 * </ul> 84 * 85 * <p>To ensure that each time we download the most up-to-date manifest file correctly, we will 86 * check for {@link FileDownloader#isContentChanged(CheckContentChangeRequest)} twice: 87 * 88 * <ul> 89 * <li>1. Before the download to check if the new download is necessary. 90 * <li>2. After the download to make sure that the content is not out of date. 91 * </ul> 92 * 93 * <p>Note that the current prerequisite of using {@link ManifestFileGroupPopulator} is that, the 94 * hosting service needs to support ETag (e.g. Lorry), otherwise the behavior will be unexpected. 95 * Talk to <internal>@ if you are not sure if the hosting service supports ETag. 96 * 97 * <p> 98 * 99 * <p>This class is @Singleton, because it provides the guarantee that all the operations are 100 * serialized correctly by {@link ExecutionSequencer}. 101 */ 102 @Singleton 103 public final class ManifestFileGroupPopulator implements FileGroupPopulator { 104 105 private static final String TAG = "ManifestFileGroupPopulator"; 106 107 /** The parser of the manifest file. */ 108 public interface ManifestConfigParser { 109 110 /** Parses the input file and returns the {@link ManifestConfig}. */ parse(Uri fileUri)111 ListenableFuture<ManifestConfig> parse(Uri fileUri); 112 } 113 114 /** Client-provided supplier of a condition whether the populator should be enabled. */ 115 public interface EnabledSupplier { isEnabled()116 boolean isEnabled(); 117 } 118 119 /** Builder for {@link ManifestFileGroupPopulator}. */ 120 public static final class Builder { 121 private boolean allowsInsecureHttp = false; 122 private boolean dedupDownloadWithEtag = true; 123 private boolean forceManifestSyncs = true; 124 private Context context; 125 private Supplier<ManifestFileFlag> manifestFileFlagSupplier; 126 private Supplier<FileDownloader> fileDownloader; 127 private ManifestConfigParser manifestConfigParser; 128 private SynchronousFileStorage fileStorage; 129 private Executor backgroundExecutor; 130 private ManifestFileMetadataStore manifestFileMetadataStore; 131 private Logger logger; 132 private Optional<ManifestConfigOverrider> overriderOptional = Optional.absent(); 133 private Optional<String> instanceIdOptional = Optional.absent(); 134 private Flags flags = new Flags() {}; 135 // Enabled the populator if no EnabledSupplier is provided. 136 private EnabledSupplier enabledSupplier = () -> true; 137 138 /** 139 * Sets the flag that allows insecure http. 140 * 141 * <p>For testing only. 142 */ 143 @CanIgnoreReturnValue 144 @VisibleForTesting setAllowsInsecureHttp(boolean allowsInsecureHttp)145 Builder setAllowsInsecureHttp(boolean allowsInsecureHttp) { 146 this.allowsInsecureHttp = allowsInsecureHttp; 147 return this; 148 } 149 150 /** 151 * By default, an HTTP HEAD request is made to avoid duplicate downloads of the manifest file. 152 * Setting this to false disables that behavior. 153 */ 154 @CanIgnoreReturnValue setDedupDownloadWithEtag(boolean dedup)155 public Builder setDedupDownloadWithEtag(boolean dedup) { 156 this.dedupDownloadWithEtag = dedup; 157 return this; 158 } 159 160 /** 161 * Force manifest syncs when {@link setDedupDownloadWithEtag} is set to false. 162 * 163 * <p>When NOT deduping with ETag, it's possible that a downloaded version of a manifest may 164 * override a potentially newer version of a manifest, preventing new file groups from being 165 * synced. 166 * 167 * <p>This flag controls whether or not the fix (always downloading the manifest) should be 168 * used. 169 * 170 * <p>NOTE: By default, this flag will be set to true -- if clients would rather have a 171 * controlled rollout of this behavior change, they should include this option in their builder 172 * and connect this to an experimental rollout system. See b/243926815 for more details. 173 */ 174 @CanIgnoreReturnValue setForceManifestSyncsWithoutETag(boolean forceManifestSyncs)175 public Builder setForceManifestSyncsWithoutETag(boolean forceManifestSyncs) { 176 this.forceManifestSyncs = forceManifestSyncs; 177 return this; 178 } 179 180 /** Sets the context. */ 181 @CanIgnoreReturnValue setContext(Context context)182 public Builder setContext(Context context) { 183 this.context = context.getApplicationContext(); 184 return this; 185 } 186 187 /** Sets the manifest file flag. */ 188 @CanIgnoreReturnValue setManifestFileFlagSupplier( Supplier<ManifestFileFlag> manifestFileFlagSupplier)189 public Builder setManifestFileFlagSupplier( 190 Supplier<ManifestFileFlag> manifestFileFlagSupplier) { 191 this.manifestFileFlagSupplier = manifestFileFlagSupplier; 192 return this; 193 } 194 195 /** Sets the file downloader. */ 196 @CanIgnoreReturnValue setFileDownloader(Supplier<FileDownloader> fileDownloader)197 public Builder setFileDownloader(Supplier<FileDownloader> fileDownloader) { 198 this.fileDownloader = fileDownloader; 199 return this; 200 } 201 202 /** Sets the manifest config parser that takes file uri and returns {@link ManifestConfig}. */ 203 @CanIgnoreReturnValue setManifestConfigParser(ManifestConfigParser manifestConfigParser)204 public Builder setManifestConfigParser(ManifestConfigParser manifestConfigParser) { 205 this.manifestConfigParser = manifestConfigParser; 206 return this; 207 } 208 209 /** Sets the mobstore file storage. Mobstore file storage must be singleton. */ 210 @CanIgnoreReturnValue setFileStorage(SynchronousFileStorage fileStorage)211 public Builder setFileStorage(SynchronousFileStorage fileStorage) { 212 this.fileStorage = fileStorage; 213 return this; 214 } 215 216 /** Sets the background executor that executes populator's tasks sequentially. */ 217 @CanIgnoreReturnValue setBackgroundExecutor(Executor backgroundExecutor)218 public Builder setBackgroundExecutor(Executor backgroundExecutor) { 219 this.backgroundExecutor = backgroundExecutor; 220 return this; 221 } 222 223 /** 224 * Sets the ManifestFileMetadataStore. 225 * 226 * <p> 227 */ 228 @CanIgnoreReturnValue setMetadataStore(ManifestFileMetadataStore manifestFileMetadataStore)229 public Builder setMetadataStore(ManifestFileMetadataStore manifestFileMetadataStore) { 230 this.manifestFileMetadataStore = manifestFileMetadataStore; 231 return this; 232 } 233 234 /** Sets the MDD logger. */ 235 @CanIgnoreReturnValue setLogger(Logger logger)236 public Builder setLogger(Logger logger) { 237 this.logger = logger; 238 return this; 239 } 240 241 /** Sets the optional manifest config overrider. */ 242 @CanIgnoreReturnValue setOverriderOptional(Optional<ManifestConfigOverrider> overriderOptional)243 public Builder setOverriderOptional(Optional<ManifestConfigOverrider> overriderOptional) { 244 this.overriderOptional = overriderOptional; 245 return this; 246 } 247 248 /** Sets the optional instance ID. */ 249 @CanIgnoreReturnValue setInstanceIdOptional(Optional<String> instanceIdOptional)250 public Builder setInstanceIdOptional(Optional<String> instanceIdOptional) { 251 this.instanceIdOptional = instanceIdOptional; 252 return this; 253 } 254 255 @CanIgnoreReturnValue setFlags(Flags flags)256 public Builder setFlags(Flags flags) { 257 this.flags = flags; 258 return this; 259 } 260 261 /** 262 * Sets the condition to check whether the populator should be enabled. If the value, returned 263 * by the condition is {@code false}, {@code refreshFileGroups} should do nothing. 264 */ setEnabledSupplier(EnabledSupplier enabledSupplier)265 public Builder setEnabledSupplier(EnabledSupplier enabledSupplier) { 266 this.enabledSupplier = enabledSupplier; 267 return this; 268 } 269 build()270 public ManifestFileGroupPopulator build() { 271 Preconditions.checkNotNull(context, "Must call setContext() before build()."); 272 Preconditions.checkNotNull( 273 manifestFileFlagSupplier, "Must call setManifestFileFlagSupplier() before build()."); 274 Preconditions.checkNotNull(fileDownloader, "Must call setFileDownloader() before build()."); 275 Preconditions.checkNotNull( 276 manifestConfigParser, "Must call setManifestConfigParser() before build()."); 277 Preconditions.checkNotNull(fileStorage, "Must call setFileStorage() before build()."); 278 Preconditions.checkNotNull( 279 backgroundExecutor, "Must call setBackgroundExecutor() before build()."); 280 Preconditions.checkNotNull( 281 manifestFileMetadataStore, "Must call manifestFileMetadataStore() before build()."); 282 Preconditions.checkNotNull(logger, "Must call setLogger() before build()."); 283 return new ManifestFileGroupPopulator(this); 284 } 285 } 286 287 private final boolean allowsInsecureHttp; 288 private final boolean dedupDownloadWithEtag; 289 private final boolean forceManifestSyncs; 290 private final Context context; 291 private final Uri manifestDirectoryUri; 292 private final Supplier<ManifestFileFlag> manifestFileFlagSupplier; 293 private final Supplier<FileDownloader> fileDownloader; 294 private final ManifestConfigParser manifestConfigParser; 295 private final SynchronousFileStorage fileStorage; 296 private final Executor backgroundExecutor; 297 private final Optional<ManifestConfigOverrider> overriderOptional; 298 private final ManifestFileMetadataStore manifestFileMetadataStore; 299 private final FileGroupPopulatorLogger eventLogger; 300 // We use futureSerializer for synchronization. 301 private final PropagatedExecutionSequencer futureSerializer = 302 PropagatedExecutionSequencer.create(); 303 private final EnabledSupplier enabledSupplier; 304 305 306 /** Returns a Builder for {@link ManifestFileGroupPopulator}. */ builder()307 public static Builder builder() { 308 return new Builder(); 309 } 310 ManifestFileGroupPopulator(Builder builder)311 private ManifestFileGroupPopulator(Builder builder) { 312 this.allowsInsecureHttp = builder.allowsInsecureHttp; 313 this.dedupDownloadWithEtag = builder.dedupDownloadWithEtag; 314 this.forceManifestSyncs = builder.forceManifestSyncs; 315 this.context = builder.context; 316 this.manifestDirectoryUri = 317 DirectoryUtil.getManifestDirectory(builder.context, builder.instanceIdOptional); 318 this.manifestFileFlagSupplier = builder.manifestFileFlagSupplier; 319 this.fileDownloader = builder.fileDownloader; 320 this.manifestConfigParser = builder.manifestConfigParser; 321 this.fileStorage = builder.fileStorage; 322 this.backgroundExecutor = builder.backgroundExecutor; 323 this.overriderOptional = builder.overriderOptional; 324 this.eventLogger = new FileGroupPopulatorLogger(builder.logger, builder.flags); 325 this.manifestFileMetadataStore = builder.manifestFileMetadataStore; 326 this.enabledSupplier = builder.enabledSupplier; 327 } 328 329 @Override refreshFileGroups(MobileDataDownload mobileDataDownload)330 public ListenableFuture<Void> refreshFileGroups(MobileDataDownload mobileDataDownload) { 331 return futureSerializer.submitAsync( 332 propagateAsyncCallable( 333 () -> { 334 LogUtil.d("%s: Add groups from ManifestFileFlag to MDD.", TAG); 335 336 // We will return immediately if the flag is null or empty. This could happen if P/H 337 // has not synced the flag or we fail to parse the flag. 338 ManifestFileFlag manifestFileFlag = manifestFileFlagSupplier.get(); 339 if (manifestFileFlag == null 340 || manifestFileFlag.equals(ManifestFileFlag.getDefaultInstance())) { 341 LogUtil.w("%s: The ManifestFileFlag is empty.", TAG); 342 logRefreshResult( 343 MddDownloadResult.Code.SUCCESS, ManifestFileFlag.getDefaultInstance()); 344 return immediateVoidFuture(); 345 } 346 347 return refreshFileGroups(mobileDataDownload, manifestFileFlag); 348 }), 349 backgroundExecutor); 350 } 351 refreshFileGroups( MobileDataDownload mobileDataDownload, ManifestFileFlag manifestFileFlag)352 private ListenableFuture<Void> refreshFileGroups( 353 MobileDataDownload mobileDataDownload, ManifestFileFlag manifestFileFlag) { 354 if(!enabledSupplier.isEnabled()){ 355 LogUtil.d("%s: The populator was disabled by enabledSupplier", TAG); 356 return immediateVoidFuture(); 357 } 358 359 if (!validate(manifestFileFlag)) { 360 logRefreshResult( 361 MddDownloadResult.Code.MANIFEST_FILE_GROUP_POPULATOR_INVALID_FLAG_ERROR, 362 manifestFileFlag); 363 LogUtil.e("%s: Invalid manifest config from manifest flag.", TAG); 364 return immediateFailedFuture(new IllegalArgumentException("Invalid manifest flag.")); 365 } 366 367 String manifestFileUrl = manifestFileFlag.getManifestFileUrl(); 368 369 // Manifest files are named and identified with their manifest ID. 370 Uri manifestFileUri = 371 manifestDirectoryUri.buildUpon().appendPath(manifestFileFlag.getManifestId()).build(); 372 373 // Represents the internal state of the metadata. Using AtomicReference here because the 374 // variable captured by lambda needs to be final. 375 final AtomicReference<ManifestFileBookkeeping> bookkeepingRef = 376 new AtomicReference<>(createDefaultManifestFileBookkeeping(manifestFileUrl)); 377 378 ListenableFuture<Void> checkFuture = 379 PropagatedFluentFuture.from(readBookeeping(manifestFileFlag.getManifestId())) 380 .transform( 381 (final Optional<ManifestFileBookkeeping> bookkeepingOptional) -> { 382 if (bookkeepingOptional.isPresent()) { 383 bookkeepingRef.set(bookkeepingOptional.get()); 384 } 385 return (Void) null; 386 }, 387 backgroundExecutor) 388 .transformAsync( 389 voidArg -> 390 // We need to call checkForContentChangeBeforeDownload to sync back the latest 391 // ETag, even when there is no entry for bookkeeping. 392 checkForContentChangeBeforeDownload( 393 manifestFileUrl, manifestFileUri, bookkeepingRef), 394 backgroundExecutor); 395 396 ListenableFuture<Optional<Throwable>> transformCheckFuture = 397 PropagatedFluentFuture.from(checkFuture) 398 .transform(voidArg -> Optional.<Throwable>absent(), backgroundExecutor) 399 .catching(Throwable.class, Optional::of, backgroundExecutor); 400 401 ListenableFuture<Void> processFuture = 402 PropagatedFluentFuture.from(transformCheckFuture) 403 .transformAsync( 404 (final Optional<Throwable> throwableOptional) -> { 405 // We do not want to proceed if transformCheckFuture contains failures, so return 406 // early. 407 if (throwableOptional.isPresent()) { 408 return immediateVoidFuture(); 409 } 410 411 ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); 412 413 if (bookkeeping.getStatus() == Status.COMMITTED) { 414 LogUtil.d("%s: Manifest file was committed.", TAG); 415 if (!overriderOptional.isPresent()) { 416 return immediateVoidFuture(); 417 } 418 419 // When the overrider is present, it may produce different configs each time the 420 // caller triggers refresh. Therefore, we need to recommit to MDD. 421 LogUtil.d("%s: Overrider is present, commit again.", TAG); 422 return parseAndCommitManifestFile( 423 mobileDataDownload, manifestFileUri, bookkeepingRef); 424 } 425 426 if (bookkeeping.getStatus() == Status.DOWNLOADED) { 427 LogUtil.d("%s: Manifest file was downloaded.", TAG); 428 return parseAndCommitManifestFile( 429 mobileDataDownload, manifestFileUri, bookkeepingRef); 430 } 431 432 return PropagatedFluentFuture.from( 433 downloadManifestFile(manifestFileUrl, manifestFileUri)) 434 .transformAsync( 435 voidArgInner -> 436 checkForContentChangeAfterDownload( 437 manifestFileUrl, manifestFileUri, bookkeepingRef), 438 backgroundExecutor) 439 .transformAsync( 440 voidArgInner -> 441 parseAndCommitManifestFile( 442 mobileDataDownload, manifestFileUri, bookkeepingRef), 443 backgroundExecutor); 444 }, 445 backgroundExecutor); 446 447 ListenableFuture<Void> catchingProcessFuture = 448 PropagatedFutures.catchingAsync( 449 processFuture, 450 Throwable.class, 451 (Throwable unused) -> { 452 ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); 453 bookkeepingRef.set(bookkeeping.toBuilder().setStatus(Status.PENDING).build()); 454 deleteManifestFileChecked(manifestFileUri); 455 return immediateVoidFuture(); 456 }, 457 backgroundExecutor); 458 459 ListenableFuture<Void> updateFuture = 460 PropagatedFutures.transformAsync( 461 catchingProcessFuture, 462 voidArg -> writeBookkeeping(manifestFileFlag.getManifestId(), bookkeepingRef.get()), 463 backgroundExecutor); 464 465 return PropagatedFutures.transformAsync( 466 updateFuture, 467 voidArg -> { 468 logAndThrowIfFailed( 469 ImmutableList.of(checkFuture, processFuture, updateFuture), 470 "Failed to refresh file groups", 471 manifestFileFlag); 472 // If there is any failure, it should have been thrown already. Therefore, we log refresh 473 // success here. 474 logRefreshResult(MddDownloadResult.Code.SUCCESS, manifestFileFlag); 475 return immediateVoidFuture(); 476 }, 477 backgroundExecutor); 478 } 479 480 private boolean validate(@Nullable ManifestFileFlag manifestFileFlag) { 481 if (manifestFileFlag == null) { 482 return false; 483 } 484 if (!manifestFileFlag.hasManifestId() || manifestFileFlag.getManifestId().isEmpty()) { 485 return false; 486 } 487 if (!manifestFileFlag.hasManifestFileUrl() 488 || (!allowsInsecureHttp && !manifestFileFlag.getManifestFileUrl().startsWith("https"))) { 489 return false; 490 } 491 return true; 492 } 493 494 private ListenableFuture<Void> parseAndCommitManifestFile( 495 MobileDataDownload mobileDataDownload, 496 Uri manifestFileUri, 497 AtomicReference<ManifestFileBookkeeping> bookkeepingRef) { 498 return PropagatedFluentFuture.from(parseManifestFile(manifestFileUri)) 499 .transformAsync( 500 (final ManifestConfig manifestConfig) -> 501 ManifestConfigHelper.refreshFromManifestConfig( 502 mobileDataDownload, 503 manifestConfig, 504 overriderOptional, 505 /* accounts= */ ImmutableList.of(), 506 /* addGroupsWithVariantId= */ false), 507 backgroundExecutor) 508 .transformAsync( 509 voidArg -> { 510 ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); 511 bookkeepingRef.set(bookkeeping.toBuilder().setStatus(Status.COMMITTED).build()); 512 return immediateVoidFuture(); 513 }, 514 backgroundExecutor); 515 } 516 517 private ListenableFuture<Void> downloadManifestFile(String urlToDownload, Uri destinationUri) { 518 LogUtil.d( 519 "%s: Start downloading the manifest file from %s to %s.", 520 TAG, urlToDownload, destinationUri.toString()); 521 522 // We now download manifest file on any network (similar to P/H). In the future, we may want to 523 // restrict the download only on WiFi, and need to introduce network policy. (However, some 524 // users are never on WiFi) 525 // 526 // Note: Right now, if the download of manifest config file is set to WiFi only but this 527 // populator is triggered in CELLULAR_CHARGING task, then the downloading will be blocked. 528 DownloadConstraints downloadConstraints = DownloadConstraints.NETWORK_CONNECTED; 529 530 return fileDownloader 531 .get() 532 .startDownloading( 533 DownloadRequest.newBuilder() 534 .setUrlToDownload(urlToDownload) 535 .setFileUri(destinationUri) 536 .setDownloadConstraints(downloadConstraints) 537 .build()); 538 } 539 540 private ListenableFuture<ManifestConfig> parseManifestFile(Uri manifestFileUri) { 541 LogUtil.d("%s: Parse the manifest file at %s.", TAG, manifestFileUri); 542 543 ListenableFuture<ManifestConfig> parseFuture = manifestConfigParser.parse(manifestFileUri); 544 return DownloadException.wrapIfFailed( 545 parseFuture, 546 DownloadResultCode.MANIFEST_FILE_GROUP_POPULATOR_PARSE_MANIFEST_FILE_ERROR, 547 "Failed to parse the manifest file."); 548 } 549 550 private ListenableFuture<Void> checkForContentChangeBeforeDownload( 551 String urlToDownload, 552 Uri manifestFileUri, 553 AtomicReference<ManifestFileBookkeeping> bookkeepingRef) { 554 LogUtil.d("%s: Prepare for downloading manifest file.", TAG); 555 556 if (!dedupDownloadWithEtag) { 557 return handleManifestDedupWithoutETag(urlToDownload, manifestFileUri, bookkeepingRef); 558 } 559 560 ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); 561 562 ListenableFuture<CheckContentChangeResponse> isContentChangedFuture = 563 fileDownloader 564 .get() 565 .isContentChanged( 566 CheckContentChangeRequest.newBuilder() 567 .setUrl(urlToDownload) 568 .setCachedETagOptional(getCachedETag(bookkeeping)) 569 .build()); 570 571 return PropagatedFutures.transformAsync( 572 isContentChangedFuture, 573 (final CheckContentChangeResponse response) -> { 574 Status currentStatus = bookkeepingRef.get().getStatus(); 575 576 // If the manifest file on server side has been modified since last download, then the 577 // manifest file previously downloaded is now stale. We need to delete it and re-download 578 // the latest version. 579 // 580 // In case of url changes, we still want to send the network request to fetch the ETag. 581 boolean urlUpdated = !urlToDownload.equals(bookkeeping.getManifestFileUrl()); 582 if (urlUpdated || response.contentChanged()) { 583 LogUtil.d( 584 "%s: Manifest file on server updated, will re-download; urlToDownload = %s;" 585 + " manifestFileUri = %s", 586 TAG, urlToDownload, manifestFileUri); 587 currentStatus = Status.PENDING; 588 deleteManifestFileChecked(manifestFileUri); 589 } 590 591 bookkeepingRef.set( 592 createManifestFileBookkeeping( 593 urlToDownload, currentStatus, response.freshETagOptional())); 594 595 return immediateVoidFuture(); 596 }, 597 backgroundExecutor); 598 } 599 600 /** 601 * Handle Manifest Bookkeeping when ETag check should be bypassed. 602 * 603 * <p>If forced syncs are enabled, the existing manifest file will be deleted and the bookkeeping 604 * reference will be updated to a default value. This forces the manifest to be redownloaded. 605 * 606 * <p>If forced syncs are disabled, this is a no-op and existing bookkeeping will be used. This 607 * reuses a downloaded manifest if one exists, or continues a download of a pending manifest. 608 */ 609 private ListenableFuture<Void> handleManifestDedupWithoutETag( 610 String urlToDownload, 611 Uri manifestFileUri, 612 AtomicReference<ManifestFileBookkeeping> bookkeepingRef) { 613 LogUtil.d( 614 "%s: Not relying on etag to dedup manifest -- checking if manifest should be force" 615 + " downloaded", 616 TAG); 617 if (forceManifestSyncs) { 618 LogUtil.d( 619 "%s: forcing re-download; urlToDownload = %s;" + " manifestFileUri = %s", 620 TAG, urlToDownload, manifestFileUri); 621 try { 622 deleteManifestFileChecked(manifestFileUri); 623 } catch (DownloadException e) { 624 return immediateFailedFuture(e); 625 } 626 bookkeepingRef.set(createDefaultManifestFileBookkeeping(urlToDownload)); 627 } else { 628 LogUtil.d( 629 "%s: not forcing re-download; urlToDownload = %s;" + " manifestFileUri =%s", 630 TAG, urlToDownload, manifestFileUri); 631 } 632 return immediateVoidFuture(); 633 } 634 635 private ListenableFuture<Void> checkForContentChangeAfterDownload( 636 String urlToDownload, 637 Uri manifestFileUri, 638 AtomicReference<ManifestFileBookkeeping> bookkeepingRef) { 639 LogUtil.d("%s: Finalize for downloading manifest file.", TAG); 640 641 if (!dedupDownloadWithEtag) { 642 LogUtil.d( 643 "%s: Not relying on etag to dedup manifest, so the downloaded manifest is" 644 + " assumed to be the latest; urlToDownload = %s, manifestFileUri = %s", 645 TAG, urlToDownload, manifestFileUri); 646 return immediateVoidFuture(); 647 } 648 649 ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); 650 651 ListenableFuture<CheckContentChangeResponse> isContentChangedFuture = 652 fileDownloader 653 .get() 654 .isContentChanged( 655 CheckContentChangeRequest.newBuilder() 656 .setUrl(urlToDownload) 657 .setCachedETagOptional(getCachedETag(bookkeeping)) 658 .build()); 659 660 return PropagatedFutures.transformAsync( 661 isContentChangedFuture, 662 (final CheckContentChangeResponse response) -> { 663 // If the manifest file on server has changed during download. The manifest file we just 664 // downloaded is stale during the download. 665 if (response.contentChanged()) { 666 LogUtil.e( 667 "%s: Manifest file on server changed during download, download failed;" 668 + " urlToDownload = %s; manifestFileUri = %s", 669 TAG, urlToDownload, manifestFileUri); 670 return immediateFailedFuture( 671 DownloadException.builder() 672 .setDownloadResultCode( 673 DownloadResultCode 674 .MANIFEST_FILE_GROUP_POPULATOR_CONTENT_CHANGED_DURING_DOWNLOAD_ERROR) 675 .setMessage("Manifest file on server changed during download.") 676 .build()); 677 } 678 679 bookkeepingRef.set( 680 createManifestFileBookkeeping( 681 urlToDownload, Status.DOWNLOADED, response.freshETagOptional())); 682 683 return immediateVoidFuture(); 684 }, 685 backgroundExecutor); 686 } 687 688 private ListenableFuture<Optional<ManifestFileBookkeeping>> readBookeeping(String manifestId) { 689 return DownloadException.wrapIfFailed( 690 manifestFileMetadataStore.read(manifestId), 691 DownloadResultCode.MANIFEST_FILE_GROUP_POPULATOR_METADATA_IO_ERROR, 692 "Failed to read bookkeeping."); 693 } 694 695 private ListenableFuture<Void> writeBookkeeping( 696 String manifestId, ManifestFileBookkeeping value) { 697 return DownloadException.wrapIfFailed( 698 manifestFileMetadataStore.upsert(manifestId, value), 699 DownloadResultCode.MANIFEST_FILE_GROUP_POPULATOR_METADATA_IO_ERROR, 700 "Failed to write bookkeeping."); 701 } 702 703 private void deleteManifestFileChecked(Uri manifestFileUri) throws DownloadException { 704 try { 705 deleteManifestFile(manifestFileUri); 706 } catch (IOException e) { 707 throw DownloadException.builder() 708 .setCause(e) 709 .setDownloadResultCode( 710 DownloadResultCode.MANIFEST_FILE_GROUP_POPULATOR_DELETE_MANIFEST_FILE_ERROR) 711 .setMessage("Failed to delete manifest file.") 712 .build(); 713 } 714 } 715 716 private void deleteManifestFile(Uri manifestFileUri) throws IOException { 717 if (fileStorage.exists(manifestFileUri)) { 718 LogUtil.d("%s: Removing manifest file at: %s", TAG, manifestFileUri); 719 fileStorage.deleteFile(manifestFileUri); 720 } else { 721 LogUtil.d("%s: Manifest file doesn't exist: %s", TAG, manifestFileUri); 722 } 723 } 724 725 // incompatible argument for parameter code of logManifestFileGroupPopulatorRefreshResult. 726 @SuppressWarnings("nullness:argument.type.incompatible") 727 private void logRefreshResult(DownloadException e, ManifestFileFlag manifestFileFlag) { 728 eventLogger.logManifestFileGroupPopulatorRefreshResult( 729 MddDownloadResult.Code.forNumber(e.getDownloadResultCode().getCode()), 730 manifestFileFlag.getManifestId(), 731 context.getPackageName(), 732 manifestFileFlag.getManifestFileUrl()); 733 } 734 735 private void logRefreshResult(MddDownloadResult.Code code, ManifestFileFlag manifestFileFlag) { 736 eventLogger.logManifestFileGroupPopulatorRefreshResult( 737 code, 738 manifestFileFlag.getManifestId(), 739 context.getPackageName(), 740 manifestFileFlag.getManifestFileUrl()); 741 } 742 743 private void logAndThrowIfFailed( 744 ImmutableList<ListenableFuture<Void>> futures, 745 String message, 746 ManifestFileFlag manifestFileFlag) 747 throws AggregateException { 748 FutureCallback<Void> logRefreshResultCallback = 749 new FutureCallback<Void>() { 750 @Override 751 public void onSuccess(Void unused) {} 752 753 @Override 754 public void onFailure(Throwable t) { 755 if (t instanceof DownloadException) { 756 logRefreshResult((DownloadException) t, manifestFileFlag); 757 } else { 758 // Here, we encountered an error that is unchecked. If UNKNOWN_ERROR is observed, we 759 // will need to investigate the cause and have it checked. 760 logRefreshResult( 761 DownloadException.builder() 762 .setCause(t) 763 .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) 764 .setMessage("Refresh failed.") 765 .build(), 766 manifestFileFlag); 767 } 768 } 769 }; 770 AggregateException.throwIfFailed(futures, Optional.of(logRefreshResultCallback), message); 771 } 772 773 private static ManifestFileBookkeeping createDefaultManifestFileBookkeeping( 774 String manifestFileUrl) { 775 return createManifestFileBookkeeping( 776 manifestFileUrl, Status.PENDING, /* eTagOptional= */ Optional.absent()); 777 } 778 779 private static ManifestFileBookkeeping createManifestFileBookkeeping( 780 String manifestFileUrl, Status status, Optional<String> eTagOptional) { 781 ManifestFileBookkeeping.Builder bookkeeping = 782 ManifestFileBookkeeping.newBuilder().setManifestFileUrl(manifestFileUrl).setStatus(status); 783 if (eTagOptional.isPresent()) { 784 bookkeeping.setCachedEtag(eTagOptional.get()); 785 } 786 return bookkeeping.build(); 787 } 788 789 private static Optional<String> getCachedETag(ManifestFileBookkeeping bookkeeping) { 790 return bookkeeping.hasCachedEtag() 791 ? Optional.of(bookkeeping.getCachedEtag()) 792 : Optional.absent(); 793 } 794 } 795