1 /* 2 * Copyright 2022 Google LLC 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.libraries.mobiledatadownload.internal; 17 18 import static com.google.common.base.Preconditions.checkNotNull; 19 import static com.google.common.util.concurrent.Futures.getDone; 20 import static com.google.common.util.concurrent.Futures.immediateFailedFuture; 21 import static com.google.common.util.concurrent.Futures.immediateFuture; 22 import static com.google.common.util.concurrent.Futures.immediateVoidFuture; 23 import static com.google.common.util.concurrent.MoreExecutors.directExecutor; 24 25 import android.content.Context; 26 import android.content.SharedPreferences; 27 import android.net.Uri; 28 import androidx.annotation.VisibleForTesting; 29 import com.google.android.libraries.mobiledatadownload.FileSource; 30 import com.google.android.libraries.mobiledatadownload.Flags; 31 import com.google.android.libraries.mobiledatadownload.SilentFeedback; 32 import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; 33 import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; 34 import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; 35 import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; 36 import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; 37 import com.google.android.libraries.mobiledatadownload.internal.downloader.FileValidator; 38 import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; 39 import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger; 40 import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; 41 import com.google.android.libraries.mobiledatadownload.internal.logging.FileGroupStatsLogger; 42 import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; 43 import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; 44 import com.google.android.libraries.mobiledatadownload.internal.logging.NetworkLogger; 45 import com.google.android.libraries.mobiledatadownload.internal.logging.StorageLogger; 46 import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; 47 import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; 48 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; 49 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; 50 import com.google.common.base.Optional; 51 import com.google.common.collect.ImmutableList; 52 import com.google.common.collect.ImmutableMap; 53 import com.google.common.util.concurrent.AsyncFunction; 54 import com.google.common.util.concurrent.ListenableFuture; 55 import com.google.errorprone.annotations.CheckReturnValue; 56 import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; 57 import com.google.mobiledatadownload.TransformProto.Transforms; 58 import com.google.mobiledatadownload.internal.MetadataProto.DataFile; 59 import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; 60 import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; 61 import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; 62 import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; 63 import com.google.protobuf.Any; 64 import java.io.IOException; 65 import java.io.PrintWriter; 66 import java.util.ArrayList; 67 import java.util.List; 68 import java.util.Map.Entry; 69 import java.util.concurrent.Executor; 70 import javax.annotation.concurrent.NotThreadSafe; 71 import javax.inject.Inject; 72 import org.checkerframework.checker.nullness.compatqual.NullableType; 73 74 /** 75 * Mobile Data Download Manager is a wrapper over all MDD functions and provides methods for the 76 * public API of MDD as well as internal periodic tasks that handle things like downloading and 77 * garbage collection of data. 78 * 79 * <p>This class is not thread safe, and all calls to it are currently channeled through {@link 80 * com.google.android.gms.mdi.download.service.DataDownloadChimeraService}, running operations in a 81 * single thread. 82 */ 83 @NotThreadSafe 84 @CheckReturnValue 85 public class MobileDataDownloadManager { 86 87 private static final String TAG = "MDDManager"; 88 89 @VisibleForTesting static final String MDD_MANAGER_METADATA = "gms_icing_mdd_manager_metadata"; 90 91 private static final String MDD_PH_CONFIG_VERSION = "gms_icing_mdd_manager_ph_config_version"; 92 93 private static final String MDD_PH_CONFIG_VERSION_TS = 94 "gms_icing_mdd_manager_ph_config_version_timestamp"; 95 96 @VisibleForTesting static final String MDD_MIGRATED_TO_OFFROAD = "mdd_migrated_to_offroad"; 97 98 @VisibleForTesting static final String RESET_TRIGGER = "gms_icing_mdd_reset_trigger"; 99 100 private static final int DEFAULT_DAYS_SINCE_LAST_MAINTENANCE = -1; 101 102 private static volatile boolean isInitialized = false; 103 104 private final Context context; 105 private final EventLogger eventLogger; 106 private final FileGroupManager fileGroupManager; 107 private final FileGroupsMetadata fileGroupsMetadata; 108 private final SharedFileManager sharedFileManager; 109 private final SharedFilesMetadata sharedFilesMetadata; 110 private final ExpirationHandler expirationHandler; 111 private final SilentFeedback silentFeedback; 112 private final StorageLogger storageLogger; 113 private final FileGroupStatsLogger fileGroupStatsLogger; 114 private final NetworkLogger networkLogger; 115 private final Optional<String> instanceId; 116 private final Executor sequentialControlExecutor; 117 private final Flags flags; 118 private final LoggingStateStore loggingStateStore; 119 private final DownloadStageManager downloadStageManager; 120 121 @Inject 122 // TODO: Create a delegateLogger for all logging instead of adding separate logger for 123 // each type. MobileDataDownloadManager( @pplicationContext Context context, EventLogger eventLogger, SharedFileManager sharedFileManager, SharedFilesMetadata sharedFilesMetadata, FileGroupManager fileGroupManager, FileGroupsMetadata fileGroupsMetadata, ExpirationHandler expirationHandler, SilentFeedback silentFeedback, StorageLogger storageLogger, FileGroupStatsLogger fileGroupStatsLogger, NetworkLogger networkLogger, @InstanceId Optional<String> instanceId, @SequentialControlExecutor Executor sequentialControlExecutor, Flags flags, LoggingStateStore loggingStateStore, DownloadStageManager downloadStageManager)124 public MobileDataDownloadManager( 125 @ApplicationContext Context context, 126 EventLogger eventLogger, 127 SharedFileManager sharedFileManager, 128 SharedFilesMetadata sharedFilesMetadata, 129 FileGroupManager fileGroupManager, 130 FileGroupsMetadata fileGroupsMetadata, 131 ExpirationHandler expirationHandler, 132 SilentFeedback silentFeedback, 133 StorageLogger storageLogger, 134 FileGroupStatsLogger fileGroupStatsLogger, 135 NetworkLogger networkLogger, 136 @InstanceId Optional<String> instanceId, 137 @SequentialControlExecutor Executor sequentialControlExecutor, 138 Flags flags, 139 LoggingStateStore loggingStateStore, 140 DownloadStageManager downloadStageManager) { 141 this.context = context; 142 this.eventLogger = eventLogger; 143 this.sharedFileManager = sharedFileManager; 144 this.sharedFilesMetadata = sharedFilesMetadata; 145 this.fileGroupManager = fileGroupManager; 146 this.fileGroupsMetadata = fileGroupsMetadata; 147 this.expirationHandler = expirationHandler; 148 this.silentFeedback = silentFeedback; 149 this.storageLogger = storageLogger; 150 this.fileGroupStatsLogger = fileGroupStatsLogger; 151 this.networkLogger = networkLogger; 152 this.instanceId = instanceId; 153 this.sequentialControlExecutor = sequentialControlExecutor; 154 this.flags = flags; 155 this.loggingStateStore = loggingStateStore; 156 this.downloadStageManager = downloadStageManager; 157 } 158 159 /** 160 * Makes the MDDManager ready for use by performing any upgrades that should be done before using 161 * MDDManager. It is also responsible for initializing all classes underneath, and clears MDD 162 * internal storage if any class init fails. 163 * 164 * <p>This should be the first call in any public method in this class, other than {@link 165 * #clear()}. 166 */ 167 @SuppressWarnings("nullness") init()168 public ListenableFuture<Void> init() { 169 if (isInitialized) { 170 return immediateVoidFuture(); 171 } 172 return PropagatedFluentFuture.from(immediateVoidFuture()) 173 .transformAsync( 174 voidArg -> { 175 SharedPreferences prefs = 176 SharedPreferencesUtil.getSharedPreferences( 177 context, MDD_MANAGER_METADATA, instanceId); 178 // Offroad downloader migration. Since the migration has been enabled in gms 179 // v18, most devices have migrated. For the remaining, we will clear MDD 180 // storage. 181 if (!prefs.getBoolean(MDD_MIGRATED_TO_OFFROAD, false)) { 182 LogUtil.d("%s Clearing MDD as device isn't migrated to offroad.", TAG); 183 return PropagatedFutures.transform( 184 clearForInit(), 185 voidArg1 -> { 186 prefs.edit().putBoolean(MDD_MIGRATED_TO_OFFROAD, true).commit(); 187 return null; 188 }, 189 sequentialControlExecutor); 190 } 191 return immediateVoidFuture(); 192 }, 193 sequentialControlExecutor) 194 .transformAsync( 195 voidArg -> 196 PropagatedFutures.transformAsync( 197 sharedFileManager.init(), 198 initSuccess -> { 199 if (!initSuccess) { 200 // This should be init before the shared file metadata. 201 LogUtil.w( 202 "%s Clearing MDD since FileManager failed or needs migration.", TAG); 203 return clearForInit(); 204 } 205 return immediateVoidFuture(); 206 }, 207 sequentialControlExecutor), 208 sequentialControlExecutor) 209 .transformAsync( 210 voidArg -> 211 PropagatedFutures.transformAsync( 212 sharedFilesMetadata.init(), 213 initSuccess -> { 214 if (!initSuccess) { 215 LogUtil.w( 216 "%s Clearing MDD since FilesMetadata failed or needs migration.", TAG); 217 return clearForInit(); 218 } 219 return immediateVoidFuture(); 220 }, 221 sequentialControlExecutor), 222 sequentialControlExecutor) 223 .transformAsync(voidArg -> fileGroupsMetadata.init(), sequentialControlExecutor) 224 .transform( 225 voidArg -> { 226 isInitialized = true; 227 return null; 228 }, 229 sequentialControlExecutor); 230 } 231 232 /** 233 * Adds the given data file group for download, after doing some sanity testing on the group. 234 * 235 * <p>This doesn't start the download right away. The data is downloaded later when the device has 236 * wifi available, by calling {@link #downloadAllPendingGroups}. 237 * 238 * <p>Calling this api with the exact same file group multiple times is a no op. 239 * 240 * @param groupKey The key for the data to be returned. This is a combination of many parameters 241 * like group name, user account. 242 * @param dataFileGroup The File group that needs to be downloaded. 243 * @return A future that resolves to true if the group was successfully added for download, or the 244 * exact group was already added earlier; false if the group being added was invalid or an I/O 245 * error occurs. 246 */ 247 // TODO(b/143572409): addGroupForDownload() call-chain should return void and use exceptions 248 // instead of boolean for failure 249 public ListenableFuture<Boolean> addGroupForDownload( 250 GroupKey groupKey, DataFileGroupInternal dataFileGroup) { 251 return addGroupForDownloadInternal(groupKey, dataFileGroup, unused -> immediateFuture(true)); 252 } 253 254 public ListenableFuture<Boolean> addGroupForDownloadInternal( 255 GroupKey groupKey, 256 DataFileGroupInternal dataFileGroup, 257 AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { 258 LogUtil.d("%s addGroupForDownload %s", TAG, groupKey.getGroupName()); 259 return PropagatedFutures.transformAsync( 260 init(), 261 voidArg -> { 262 // Check if the group we received is a valid group. 263 if (!DataFileGroupValidator.isValidGroup(dataFileGroup, context, flags)) { 264 eventLogger.logEventSampled( 265 MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, 266 dataFileGroup.getGroupName(), 267 dataFileGroup.getFileGroupVersionNumber(), 268 dataFileGroup.getBuildId(), 269 dataFileGroup.getVariantId()); 270 return immediateFuture(false); 271 } 272 273 DataFileGroupInternal populatedDataFileGroup = mayPopulateChecksum(dataFileGroup); 274 try { 275 return PropagatedFluentFuture.from( 276 fileGroupManager.addGroupForDownload(groupKey, populatedDataFileGroup)) 277 .transformAsync( 278 addGroupForDownloadResult -> { 279 if (addGroupForDownloadResult) { 280 return maybeMarkPendingGroupAsDownloadedImmediately( 281 groupKey, customFileGroupValidator); 282 } 283 return immediateVoidFuture(); 284 }, 285 sequentialControlExecutor) 286 .transform(unused -> true, sequentialControlExecutor); 287 } catch (ExpiredFileGroupException 288 | UninstalledAppException 289 | ActivationRequiredForGroupException e) { 290 LogUtil.w("%s %s", TAG, e.getClass()); 291 return immediateFailedFuture(e); 292 } catch (IOException e) { 293 LogUtil.e("%s %s", TAG, e.getClass()); 294 silentFeedback.send(e, "Failed to add group to MDD"); 295 return immediateFailedFuture(e); 296 } 297 }, 298 sequentialControlExecutor); 299 } 300 301 /** 302 * Helper method to mark a group as downloaded immediately. 303 * 304 * <p>This method checks if a pending group is already downloaded and updates its state in MDD's 305 * metadata if it is downloaded. Additionally, a download complete immediate event is logged for 306 * this case. 307 * 308 * <p>If no pending version of the group is available, this method is a no-op. 309 * 310 * <p>NOTE: This method is only meant to be called during addFileGroup, where it makes sense to 311 * log the immediate download complete event. 312 */ 313 private ListenableFuture<Void> maybeMarkPendingGroupAsDownloadedImmediately( 314 GroupKey groupKey, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { 315 ListenableFuture<@NullableType DataFileGroupInternal> pendingGroupFuture = 316 fileGroupManager.getFileGroup(groupKey, /* downloaded= */ false); 317 return PropagatedFluentFuture.from(pendingGroupFuture) 318 .transformAsync( 319 pendingGroup -> { 320 if (pendingGroup == null) { 321 // send pending state to skip logging the event 322 return immediateFuture(GroupDownloadStatus.PENDING); 323 } 324 // Verify the group is downloaded (and commit this to metadata). 325 return fileGroupManager.verifyGroupDownloaded( 326 groupKey, 327 pendingGroup, 328 /* removePendingVersion= */ true, 329 customFileGroupValidator, 330 DownloadStateLogger.forDownload(eventLogger)); 331 }, 332 sequentialControlExecutor) 333 .transformAsync( 334 verifyPendingGroupDownloadedResult -> { 335 if (verifyPendingGroupDownloadedResult == GroupDownloadStatus.DOWNLOADED) { 336 // Use checkNotNull to satisfy nullness checker -- if the group status is 337 // downloaded, pendingGroup must be non-null. 338 DataFileGroupInternal group = checkNotNull(getDone(pendingGroupFuture)); 339 eventLogger.logEventSampled( 340 MddClientEvent.Code.DATA_DOWNLOAD_COMPLETE_IMMEDIATE, 341 group.getGroupName(), 342 group.getFileGroupVersionNumber(), 343 group.getBuildId(), 344 group.getVariantId()); 345 } 346 return immediateVoidFuture(); 347 }, 348 sequentialControlExecutor); 349 } 350 351 /** 352 * Removes the file group from MDD with the given group key. This will cancel any ongoing download 353 * of the file group. 354 * 355 * @param groupKey The key for the file group to be removed from MDD. This is a combination of 356 * many parameters like group name, user account. 357 * @param pendingOnly When true, only remove the pending version of this file group. 358 * @return ListenableFuture that may throw an IOException if some error is encountered when 359 * removing from metadata or a SharedFileMissingException if some of the shared file metadata 360 * is missing. 361 */ 362 public ListenableFuture<Void> removeFileGroup(GroupKey groupKey, boolean pendingOnly) 363 throws SharedFileMissingException, IOException { 364 LogUtil.d("%s removeFileGroup %s", TAG, groupKey.getGroupName()); 365 366 return PropagatedFutures.transformAsync( 367 init(), 368 voidArg -> fileGroupManager.removeFileGroup(groupKey, pendingOnly), 369 sequentialControlExecutor); 370 } 371 372 /** 373 * Removes the file groups from MDD with the given group keys. 374 * 375 * <p>This will cancel any ongoing downloads of file groups that should be removed. 376 * 377 * @param groupKeys The keys of file groups that should be removed from MDD. 378 * @return ListenableFuture that resolves when file groups have been deleted, or fails if some 379 * error is encountered when removing metadata. 380 */ 381 public ListenableFuture<Void> removeFileGroups(List<GroupKey> groupKeys) { 382 LogUtil.d("%s removeFileGroups for %d groups", TAG, groupKeys.size()); 383 384 return PropagatedFutures.transformAsync( 385 init(), voidArg -> fileGroupManager.removeFileGroups(groupKeys), sequentialControlExecutor); 386 } 387 388 /** 389 * Returns the latest data that we have for the given client key. 390 * 391 * @param groupKey The key for the data to be returned. This is a combination of many parameters 392 * like group name, user account. 393 * @param downloaded Whether to return a downloaded version or a pending version of the group. 394 * @return A ListenableFuture that resolves to the requested data file group for the given group 395 * name, if it exists, null otherwise. 396 */ 397 public ListenableFuture<@NullableType DataFileGroupInternal> getFileGroup( 398 GroupKey groupKey, boolean downloaded) { 399 LogUtil.d("%s getFileGroup %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); 400 401 return PropagatedFutures.transformAsync( 402 init(), 403 voidArg -> fileGroupManager.getFileGroup(groupKey, downloaded), 404 sequentialControlExecutor); 405 } 406 407 /** Returns a future resolving to a list of all pending and downloaded groups in MDD. */ 408 public ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups() { 409 LogUtil.d("%s getAllFreshGroups", TAG); 410 411 return PropagatedFutures.transformAsync( 412 init(), voidArg -> fileGroupsMetadata.getAllFreshGroups(), sequentialControlExecutor); 413 } 414 415 /** 416 * Returns a map of on-device URIs for the requested {@link DataFileGroupInternal}. 417 * 418 * <p>If a DataFile does not have an on-device URI (e.g. the download for the file is not 419 * completed), The returned map will not contain an entry for that DataFile. 420 * 421 * <p>If the group supports isolated structures, verification of the isolated structure can be 422 * controlled. If a file fails the verification (either the symlink is not created, or does not 423 * point to the correct location), it will be omitted from the map. 424 * 425 * <p>NOTE: Verification should only be turned off on critical access paths where latency must be 426 * minimized. This may lead to an edge case where the isolated structure becomes broken and/or 427 * corrupted until MDD can fix the structure in its daily maintenance task. 428 */ 429 public ListenableFuture<ImmutableMap<DataFile, Uri>> getDataFileUris( 430 DataFileGroupInternal dataFileGroup, boolean verifyIsolatedStructure) { 431 LogUtil.d("%s: getDataFileUris %s", TAG, dataFileGroup.getGroupName()); 432 433 boolean useIsolatedStructure = FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup); 434 435 // If isolated structure is supported, get the isolated uris (symlinks which point to the 436 // on-device location). These can be calculated synchronously and before init since they only 437 // require the file group metadata. 438 ImmutableMap.Builder<DataFile, Uri> isolatedUriMapBuilder = ImmutableMap.builder(); 439 if (useIsolatedStructure) { 440 isolatedUriMapBuilder.putAll(fileGroupManager.getIsolatedFileUris(dataFileGroup)); 441 } 442 ImmutableMap<DataFile, Uri> isolatedUriMap = isolatedUriMapBuilder.buildKeepingLast(); 443 444 return PropagatedFluentFuture.from(init()) 445 .transformAsync( 446 unused -> { 447 // Lookup on-device uris only if required to reduce latency. On-device lookups happen 448 // asynchronously since we need to access the latest underlying file metadata. 449 // 1. The group does not support an isolated structure 450 // 2. The group supports an isolated structure AND verification of that structure 451 // should occur. 452 if (!useIsolatedStructure || verifyIsolatedStructure) { 453 return fileGroupManager.getOnDeviceUris(dataFileGroup); 454 } 455 456 // Return an empty map here since we won't be using the on-device uris. 457 return immediateFuture(ImmutableMap.of()); 458 }, 459 sequentialControlExecutor) 460 .transform( 461 onDeviceUriMap -> { 462 if (useIsolatedStructure) { 463 if (verifyIsolatedStructure) { 464 // Return verified map of isolated uris. 465 return fileGroupManager.verifyIsolatedFileUris(isolatedUriMap, onDeviceUriMap); 466 } 467 468 // Verification not required, return isolated uris. 469 return isolatedUriMap; 470 } 471 472 // Isolated structure are not in use, return on-device uris. 473 return onDeviceUriMap; 474 }, 475 sequentialControlExecutor) 476 .transform( 477 selectedUriMap -> { 478 // Before returning uri map, apply read transforms if required. 479 ImmutableMap.Builder<DataFile, Uri> finalUriMapBuilder = ImmutableMap.builder(); 480 for (Entry<DataFile, Uri> entry : selectedUriMap.entrySet()) { 481 DataFile dataFile = entry.getKey(); 482 // Skip entries which have a null uri value. 483 if (entry.getValue() == null) { 484 continue; 485 } 486 if (dataFile.hasReadTransforms()) { 487 finalUriMapBuilder.put( 488 dataFile, 489 applyTransformsToFileUri(entry.getValue(), dataFile.getReadTransforms())); 490 } else { 491 finalUriMapBuilder.put(entry); 492 } 493 } 494 return finalUriMapBuilder.buildKeepingLast(); 495 }, 496 sequentialControlExecutor); 497 } 498 499 /** 500 * Convenience method for {@link #getDataFileUris(DataFileGroupInternal, boolean)} when only a 501 * single data file is required. 502 */ 503 public ListenableFuture<@NullableType Uri> getDataFileUri( 504 DataFile dataFile, DataFileGroupInternal dataFileGroup, boolean verifyIsolatedStructure) { 505 LogUtil.d("%s getDataFileUri %s %s", TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); 506 return PropagatedFutures.transform( 507 getDataFileUris(dataFileGroup, verifyIsolatedStructure), 508 dataFileUris -> dataFileUris.get(dataFile), 509 directExecutor()); 510 } 511 512 private Uri applyTransformsToFileUri(Uri fileUri, Transforms transforms) { 513 if (!flags.enableCompressedFile() || transforms.getTransformCount() == 0) { 514 return fileUri; 515 } 516 return fileUri 517 .buildUpon() 518 .encodedFragment(TransformProtos.toEncodedFragment(transforms)) 519 .build(); 520 } 521 522 /** 523 * Import inline files into an existing DataFileGroup and update its metadata accordingly. 524 * 525 * @param groupKey The key of file group to update 526 * @param buildId build id to identify the file group to update 527 * @param variantId variant id to identify the file group to update 528 * @param updatedDataFileList list of DataFiles to import into the file group 529 * @param inlineFileMap Map of inline file sources to import 530 * @param customPropertyOptional Optional custom property used to identify the file group to 531 * update 532 * @return A ListenableFuture that resolves when inline files have successfully imported 533 */ 534 public ListenableFuture<Void> importFiles( 535 GroupKey groupKey, 536 long buildId, 537 String variantId, 538 ImmutableList<DataFile> updatedDataFileList, 539 ImmutableMap<String, FileSource> inlineFileMap, 540 Optional<Any> customPropertyOptional, 541 AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { 542 LogUtil.d("%s: importFiles %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); 543 return PropagatedFutures.transformAsync( 544 init(), 545 voidArg -> 546 fileGroupManager.importFilesIntoFileGroup( 547 groupKey, 548 buildId, 549 variantId, 550 mayPopulateChecksum(updatedDataFileList), 551 inlineFileMap, 552 customPropertyOptional, 553 customFileGroupValidator), 554 sequentialControlExecutor); 555 } 556 557 /** 558 * Download the pending group that we have for the given group key. 559 * 560 * @param groupKey The key of file group to be downloaded. 561 * @param downloadConditionsOptional The conditions for the download. If absent, MDD will use the 562 * config from server. 563 * @return The ListenableFuture that download the file group. 564 */ 565 public ListenableFuture<DataFileGroupInternal> downloadFileGroup( 566 GroupKey groupKey, 567 Optional<DownloadConditions> downloadConditionsOptional, 568 AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { 569 LogUtil.d( 570 "%s downloadFileGroup %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); 571 return PropagatedFutures.transformAsync( 572 init(), 573 voidArg -> 574 fileGroupManager.downloadFileGroup( 575 groupKey, downloadConditionsOptional.orNull(), customFileGroupValidator), 576 sequentialControlExecutor); 577 } 578 579 /** 580 * Set the activation status for the group. 581 * 582 * @param groupKey The key for which the activation is to be set. 583 * @param activation Whether the group should be activated or deactivated. 584 * @return future resolving to whether the activation was successful. 585 */ 586 public ListenableFuture<Boolean> setGroupActivation(GroupKey groupKey, boolean activation) { 587 LogUtil.d( 588 "%s setGroupActivation %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); 589 return PropagatedFutures.transformAsync( 590 init(), 591 voidArg -> fileGroupManager.setGroupActivation(groupKey, activation), 592 sequentialControlExecutor); 593 } 594 595 /** 596 * Tries to download all pending file groups, which contains at least one file that isn't yet 597 * downloaded. 598 * 599 * @param onWifi whether the device is on wifi at the moment. 600 */ 601 public ListenableFuture<Void> downloadAllPendingGroups( 602 boolean onWifi, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { 603 LogUtil.d("%s downloadAllPendingGroups on wifi = %s", TAG, onWifi); 604 return PropagatedFutures.transformAsync( 605 init(), 606 voidArg -> { 607 if (flags.mddEnableDownloadPendingGroups()) { 608 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 609 return fileGroupManager.scheduleAllPendingGroupsForDownload( 610 onWifi, customFileGroupValidator); 611 } 612 return immediateVoidFuture(); 613 }, 614 sequentialControlExecutor); 615 } 616 617 /** 618 * Tries to verify all pending file groups, which contains at least one file that isn't yet 619 * downloaded. 620 */ 621 public ListenableFuture<Void> verifyAllPendingGroups( 622 AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { 623 LogUtil.d("%s verifyAllPendingGroups", TAG); 624 return PropagatedFutures.transformAsync( 625 init(), 626 voidArg -> { 627 if (flags.mddEnableVerifyPendingGroups()) { 628 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 629 return fileGroupManager.verifyAllPendingGroupsDownloaded(customFileGroupValidator); 630 } 631 return immediateVoidFuture(); 632 }, 633 sequentialControlExecutor); 634 } 635 636 /** 637 * Performs periodic maintenance. This includes: 638 * 639 * <ol> 640 * <li>Check if any of the pending groups were downloaded. 641 * <li>Garbage collect all old data mdd has. 642 * </ol> 643 */ 644 public ListenableFuture<Void> maintenance() { 645 LogUtil.d("%s Running maintenance", TAG); 646 647 return PropagatedFluentFuture.from(init()) 648 .transformAsync(voidArg -> getAndResetDaysSinceLastMaintenance(), directExecutor()) 649 .transformAsync( 650 daysSinceLastLog -> { 651 List<ListenableFuture<Void>> maintenanceFutures = new ArrayList<>(); 652 653 // It's possible that we missed the flag change notification for mdd reset before. 654 // Check now to be sure. 655 maintenanceFutures.add(checkResetTrigger()); 656 657 if (flags.logFileGroupsWithFilesMissing()) { 658 maintenanceFutures.add(fileGroupManager.logAndDeleteForMissingSharedFiles()); 659 } 660 661 // Remove all groups belonging to apps that were uninstalled. 662 if (flags.mddDeleteUninstalledApps()) { 663 maintenanceFutures.add(fileGroupManager.deleteUninstalledAppGroups()); 664 } 665 666 // Remove all groups belonging to accounts that were removed. 667 if (flags.mddDeleteGroupsRemovedAccounts()) { 668 maintenanceFutures.add(fileGroupManager.deleteRemovedAccountGroups()); 669 } 670 671 if (flags.enableIsolatedStructureVerification()) { 672 maintenanceFutures.add(fileGroupManager.verifyAndAttemptToRepairIsolatedFiles()); 673 } 674 675 if (flags.mddEnableGarbageCollection()) { 676 maintenanceFutures.add(expirationHandler.updateExpiration()); 677 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 678 } 679 680 // Log daily file group stats. 681 maintenanceFutures.add(fileGroupStatsLogger.log(daysSinceLastLog)); 682 683 // Log storage stats. 684 maintenanceFutures.add(storageLogger.logStorageStats(daysSinceLastLog)); 685 686 // Log network usage stats. 687 maintenanceFutures.add(networkLogger.log()); 688 689 // Clear checkPhenotypeFreshness settings from Shared Prefs as the feature was 690 // deleted. 691 SharedPreferences prefs = 692 SharedPreferencesUtil.getSharedPreferences( 693 context, MDD_MANAGER_METADATA, instanceId); 694 prefs.edit().remove(MDD_PH_CONFIG_VERSION).remove(MDD_PH_CONFIG_VERSION_TS).commit(); 695 696 return PropagatedFutures.whenAllComplete(maintenanceFutures) 697 .call(() -> null, sequentialControlExecutor); 698 }, 699 sequentialControlExecutor); 700 } 701 702 /** 703 * Removes expired FileGroups (whether active or stale) and deletes files no longer referenced by 704 * a FileGroup. 705 */ 706 public ListenableFuture<Void> removeExpiredGroupsAndFiles() { 707 return PropagatedFluentFuture.from(init()) 708 .transformAsync(voidArg -> expirationHandler.updateExpiration(), sequentialControlExecutor); 709 } 710 711 /** Dumps the current internal state of the MDD manager. */ 712 public ListenableFuture<Void> dump(final PrintWriter writer) { 713 return PropagatedFutures.transformAsync( 714 init(), 715 voidArg -> 716 PropagatedFutures.transformAsync( 717 fileGroupManager.dump(writer), 718 voidParam -> sharedFileManager.dump(writer), 719 sequentialControlExecutor), 720 sequentialControlExecutor); 721 } 722 723 /** Checks to see if a flag change requires MDD to clear its data. */ 724 public ListenableFuture<Void> checkResetTrigger() { 725 LogUtil.d("%s checkResetTrigger", TAG); 726 return PropagatedFutures.transformAsync( 727 init(), 728 voidArg -> { 729 SharedPreferences prefs = 730 SharedPreferencesUtil.getSharedPreferences(context, MDD_MANAGER_METADATA, instanceId); 731 if (!prefs.contains(RESET_TRIGGER)) { 732 prefs.edit().putInt(RESET_TRIGGER, flags.mddResetTrigger()).commit(); 733 } 734 int savedResetValue = prefs.getInt(RESET_TRIGGER, 0); 735 int currentResetValue = flags.mddResetTrigger(); 736 // If the flag has changed since we last saw it, save the new value in shared prefs and 737 // clear. 738 if (savedResetValue < currentResetValue) { 739 prefs.edit().putInt(RESET_TRIGGER, currentResetValue).commit(); 740 LogUtil.d("%s Received reset trigger. Clearing all Mdd data.", TAG); 741 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 742 return clearAllFilesAndMetadata(); 743 } 744 return immediateVoidFuture(); 745 }, 746 sequentialControlExecutor); 747 } 748 749 /** Clears the internal state of MDD and deletes all downloaded files. */ 750 @SuppressWarnings("ApplySharedPref") 751 public ListenableFuture<Void> clear() { 752 LogUtil.d("%s Clearing MDD internal storage", TAG); 753 754 // Delete all of the bookkeeping files used by MDD Manager's internal classes. 755 // Clear downloadStageManager first since it needs to know which builds to delete from 756 // SharedFilesMetadata. 757 return PropagatedFluentFuture.from(downloadStageManager.clearAll()) 758 .transformAsync(voidArg -> clearAllFilesAndMetadata(), sequentialControlExecutor) 759 .transformAsync( 760 voidArg -> { 761 // Clear all migration status. 762 Migrations.clear(context); 763 SharedPreferencesUtil.getSharedPreferences(context, MDD_MANAGER_METADATA, instanceId) 764 .edit() 765 .clear() 766 .commit(); 767 768 isInitialized = false; 769 return immediateVoidFuture(); 770 }, 771 sequentialControlExecutor) 772 .transformAsync(voidArg -> loggingStateStore.clear(), sequentialControlExecutor); 773 } 774 775 @VisibleForTesting 776 public static void resetForTest() { 777 isInitialized = false; 778 } 779 780 /** Clear during MDD init */ 781 private ListenableFuture<Void> clearForInit() { 782 return PropagatedFutures.transformAsync( 783 // Clear only, no need to cancel download. 784 sharedFileManager.clear(), 785 voidArg0 -> 786 // The metadata files should be cleared after the classes have been cleared. 787 PropagatedFutures.transformAsync( 788 sharedFilesMetadata.clear(), 789 voidArg1 -> fileGroupsMetadata.clear(), 790 sequentialControlExecutor), 791 sequentialControlExecutor); 792 } 793 794 /* Clear all metadata and files, also cancel pending download. */ 795 private ListenableFuture<Void> clearAllFilesAndMetadata() { 796 return PropagatedFutures.transformAsync( 797 // Need to cancel download after MDD is already initialized. 798 sharedFileManager.cancelDownloadAndClear(), 799 voidArg1 -> 800 // The metadata files should be cleared after the classes have been cleared. 801 PropagatedFutures.transformAsync( 802 sharedFilesMetadata.clear(), 803 voidArg2 -> fileGroupsMetadata.clear(), 804 sequentialControlExecutor), 805 sequentialControlExecutor); 806 } 807 808 // Convenience method to populate checksums for a DataFileGroup 809 private static DataFileGroupInternal mayPopulateChecksum(DataFileGroupInternal dataFileGroup) { 810 List<DataFile> dataFileList = dataFileGroup.getFileList(); 811 ImmutableList<DataFile> updatedDataFileList = mayPopulateChecksum(dataFileList); 812 return dataFileGroup.toBuilder().clearFile().addAllFile(updatedDataFileList).build(); 813 } 814 815 private static ImmutableList<DataFile> mayPopulateChecksum(List<DataFile> dataFileList) { 816 boolean hasChecksumTypeNone = false; 817 818 for (DataFile dataFile : dataFileList) { 819 if (dataFile.getChecksumType() == ChecksumType.NONE) { 820 hasChecksumTypeNone = true; 821 break; 822 } 823 } 824 825 if (!hasChecksumTypeNone) { 826 return ImmutableList.copyOf(dataFileList); 827 } 828 829 // Check if any file does not have checksum, replace the checksum with the checksum of 830 // download url. 831 ImmutableList.Builder<DataFile> dataFileListBuilder = 832 ImmutableList.builderWithExpectedSize(dataFileList.size()); 833 for (DataFile dataFile : dataFileList) { 834 switch (dataFile.getChecksumType()) { 835 // Default stands for SHA1. 836 case DEFAULT: 837 dataFileListBuilder.add(dataFile); 838 break; 839 case NONE: 840 // Since internally we use checksum as a key, it can't be empty. We will generate the 841 // checksum using the urlToDownload if it's not set. 842 DataFile.Builder dataFileBuilder = dataFile.toBuilder(); 843 String checksum = FileValidator.computeSha1Digest(dataFile.getUrlToDownload()); 844 // When a data file has zip transforms, downloaded file checksum is used for identifying 845 // the data file; otherwise, checksum is used. 846 if (FileGroupUtil.hasZipDownloadTransform(dataFile)) { 847 dataFileBuilder.setDownloadedFileChecksum(checksum); 848 } else { 849 dataFileBuilder.setChecksum(checksum); 850 } 851 LogUtil.d( 852 "FileId %s does not have checksum. Generated checksum from url %s", 853 dataFileBuilder.getFileId(), dataFileBuilder.getChecksum()); 854 855 dataFileListBuilder.add(dataFileBuilder.build()); 856 break; 857 // continue below. 858 } 859 } 860 861 return dataFileListBuilder.build(); 862 } 863 864 /** 865 * Gets and resets the number of days since last maintenance from {@link loggingStateStore}. If 866 * loggingStateStore fails to provide a value (if it throws an exception or the value was not set) 867 * this handles that by returning -1. clear 868 * 869 * <p>If {@link Flags.enableDaysSinceLastMaintenanceTracking} is not enabled, this returns -1. 870 */ 871 private ListenableFuture<Integer> getAndResetDaysSinceLastMaintenance() { 872 if (!flags.enableDaysSinceLastMaintenanceTracking()) { 873 return immediateFuture(DEFAULT_DAYS_SINCE_LAST_MAINTENANCE); 874 } 875 876 return PropagatedFluentFuture.from(loggingStateStore.getAndResetDaysSinceLastMaintenance()) 877 .catching( 878 IOException.class, 879 exception -> { 880 LogUtil.d(exception, "Failed to update days since last maintenance"); 881 // If we failed to read or update the days since last maintenance, just set the value 882 // to -1. 883 return Optional.of(DEFAULT_DAYS_SINCE_LAST_MAINTENANCE); 884 }, 885 directExecutor()) 886 .transform( 887 daysSinceLastMaintenanceOptional -> { 888 if (!daysSinceLastMaintenanceOptional.isPresent()) { 889 return DEFAULT_DAYS_SINCE_LAST_MAINTENANCE; 890 } 891 Integer daysSinceLastMaintenance = daysSinceLastMaintenanceOptional.get(); 892 if (daysSinceLastMaintenance < 0) { 893 return DEFAULT_DAYS_SINCE_LAST_MAINTENANCE; 894 } 895 // TODO(b/191042900): should we add an upper bound here? 896 return daysSinceLastMaintenance; 897 }, 898 directExecutor()); 899 } 900 } 901