1 /* 2 * Copyright 2022 Google LLC 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.libraries.mobiledatadownload.internal; 17 18 import static com.google.common.util.concurrent.Futures.immediateVoidFuture; 19 import static java.lang.Math.min; 20 21 import android.content.Context; 22 import android.net.Uri; 23 import androidx.annotation.VisibleForTesting; 24 import com.google.android.libraries.mobiledatadownload.Flags; 25 import com.google.android.libraries.mobiledatadownload.SilentFeedback; 26 import com.google.android.libraries.mobiledatadownload.TimeSource; 27 import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; 28 import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; 29 import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; 30 import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; 31 import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; 32 import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; 33 import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; 34 import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; 35 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; 36 import com.google.common.base.Optional; 37 import com.google.common.util.concurrent.ListenableFuture; 38 import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; 39 import com.google.mobiledatadownload.internal.MetadataProto.DataFile; 40 import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; 41 import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; 42 import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; 43 import java.io.IOException; 44 import java.util.ArrayList; 45 import java.util.HashSet; 46 import java.util.List; 47 import java.util.Set; 48 import java.util.concurrent.Executor; 49 import java.util.concurrent.atomic.AtomicInteger; 50 import javax.inject.Inject; 51 52 /** 53 * A class that handles of the logic for file group expiration and file expiration. Expiration is 54 * determined by two sources: 1) when the active_expiration_date (set server-side by the client) has 55 * passed 2) when stale_lifetime_secs has passed since the group became stale. 56 */ 57 public class ExpirationHandler { 58 59 private static final String TAG = "ExpirationHandler"; 60 61 @VisibleForTesting 62 static final String MDD_EXPIRATION_HANDLER = "gms_icing_mdd_expiration_handler"; 63 64 private final Context context; 65 private final FileGroupsMetadata fileGroupsMetadata; 66 private final SharedFileManager sharedFileManager; 67 private final SharedFilesMetadata sharedFilesMetadata; 68 private final EventLogger eventLogger; 69 private final TimeSource timeSource; 70 private final SynchronousFileStorage fileStorage; 71 private final Optional<String> instanceId; 72 private final SilentFeedback silentFeedback; 73 private final Executor sequentialControlExecutor; 74 private final Flags flags; 75 76 @Inject ExpirationHandler( @pplicationContext Context context, FileGroupsMetadata fileGroupsMetadata, SharedFileManager sharedFileManager, SharedFilesMetadata sharedFilesMetadata, EventLogger eventLogger, TimeSource timeSource, SynchronousFileStorage fileStorage, @InstanceId Optional<String> instanceId, SilentFeedback silentFeedback, @SequentialControlExecutor Executor sequentialControlExecutor, Flags flags)77 public ExpirationHandler( 78 @ApplicationContext Context context, 79 FileGroupsMetadata fileGroupsMetadata, 80 SharedFileManager sharedFileManager, 81 SharedFilesMetadata sharedFilesMetadata, 82 EventLogger eventLogger, 83 TimeSource timeSource, 84 SynchronousFileStorage fileStorage, 85 @InstanceId Optional<String> instanceId, 86 SilentFeedback silentFeedback, 87 @SequentialControlExecutor Executor sequentialControlExecutor, 88 Flags flags) { 89 this.context = context; 90 this.fileGroupsMetadata = fileGroupsMetadata; 91 this.sharedFileManager = sharedFileManager; 92 this.sharedFilesMetadata = sharedFilesMetadata; 93 this.eventLogger = eventLogger; 94 this.timeSource = timeSource; 95 this.fileStorage = fileStorage; 96 this.instanceId = instanceId; 97 this.silentFeedback = silentFeedback; 98 this.sequentialControlExecutor = sequentialControlExecutor; 99 this.flags = flags; 100 } 101 updateExpiration()102 ListenableFuture<Void> updateExpiration() { 103 return PropagatedFutures.transformAsync( 104 removeExpiredStaleGroups(), 105 voidArg0 -> 106 PropagatedFutures.transformAsync( 107 removeExpiredFreshGroups(), 108 voidArg1 -> removeUnaccountedFiles(), 109 sequentialControlExecutor), 110 sequentialControlExecutor); 111 } 112 113 /** Returns a future that checks all File Groups and remove expired ones from FileGroupManager */ removeExpiredFreshGroups()114 private ListenableFuture<Void> removeExpiredFreshGroups() { 115 return PropagatedFutures.transformAsync( 116 fileGroupsMetadata.getAllFreshGroups(), 117 groups -> { 118 List<GroupKey> expiredGroupKeys = new ArrayList<>(); 119 for (GroupKeyAndGroup pair : groups) { 120 GroupKey groupKey = pair.groupKey(); 121 DataFileGroupInternal dataFileGroup = pair.dataFileGroup(); 122 Long groupExpirationDateMillis = FileGroupUtil.getExpirationDateMillis(dataFileGroup); 123 LogUtil.d( 124 "%s: Checking group %s with expiration date %s", 125 TAG, dataFileGroup.getGroupName(), groupExpirationDateMillis); 126 if (FileGroupUtil.isExpired(groupExpirationDateMillis, timeSource)) { 127 eventLogger.logEventSampled( 128 MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, 129 dataFileGroup.getGroupName(), 130 dataFileGroup.getFileGroupVersionNumber(), 131 dataFileGroup.getBuildId(), 132 dataFileGroup.getVariantId()); 133 LogUtil.d( 134 "%s: Expired group %s with expiration date %s", 135 TAG, dataFileGroup.getGroupName(), groupExpirationDateMillis); 136 expiredGroupKeys.add(groupKey); 137 138 // Remove Isolated structure if necessary. 139 if (FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup)) { 140 FileGroupUtil.removeIsolatedFileStructure( 141 context, instanceId, dataFileGroup, fileStorage); 142 } 143 } 144 } 145 146 return PropagatedFutures.transform( 147 fileGroupsMetadata.removeAllGroupsWithKeys(expiredGroupKeys), 148 removeSuccess -> { 149 if (!removeSuccess) { 150 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 151 LogUtil.e("%s: Failed to remove expired groups!", TAG); 152 } 153 return null; 154 }, 155 sequentialControlExecutor); 156 }, 157 sequentialControlExecutor); 158 } 159 160 /** Check and update all stale File Groups; remove staled ones */ 161 private ListenableFuture<Void> removeExpiredStaleGroups() { 162 return PropagatedFutures.transformAsync( 163 fileGroupsMetadata.getAllStaleGroups(), 164 staleGroups -> { 165 List<DataFileGroupInternal> nonExpiredStaleGroups = new ArrayList<>(); 166 for (DataFileGroupInternal staleGroup : staleGroups) { 167 long groupStaleExpirationDateMillis = 168 FileGroupUtil.getStaleExpirationDateMillis(staleGroup); 169 long groupExpirationDateMillis = FileGroupUtil.getExpirationDateMillis(staleGroup); 170 long actualExpirationDateMillis = 171 min(groupStaleExpirationDateMillis, groupExpirationDateMillis); 172 173 // Remove the group from this list if its expired. 174 if (FileGroupUtil.isExpired(actualExpirationDateMillis, timeSource)) { 175 eventLogger.logEventSampled( 176 MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, 177 staleGroup.getGroupName(), 178 staleGroup.getFileGroupVersionNumber(), 179 staleGroup.getBuildId(), 180 staleGroup.getVariantId()); 181 182 // Remove Isolated structure if necessary. 183 if (FileGroupUtil.isIsolatedStructureAllowed(staleGroup)) { 184 FileGroupUtil.removeIsolatedFileStructure( 185 context, instanceId, staleGroup, fileStorage); 186 } 187 } else { 188 nonExpiredStaleGroups.add(staleGroup); 189 } 190 } 191 192 // Empty the list of stale groups in the FGGC and write only the non-expired stale groups. 193 return PropagatedFutures.transformAsync( 194 fileGroupsMetadata.removeAllStaleGroups(), 195 voidArg -> 196 PropagatedFutures.transformAsync( 197 fileGroupsMetadata.writeStaleGroups(nonExpiredStaleGroups), 198 writeSuccess -> { 199 if (!writeSuccess) { 200 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 201 LogUtil.e("%s: Failed to write back stale groups!", TAG); 202 } 203 return immediateVoidFuture(); 204 }, 205 sequentialControlExecutor), 206 sequentialControlExecutor); 207 }, 208 sequentialControlExecutor); 209 } 210 211 private ListenableFuture<Void> removeUnaccountedFiles() { 212 return PropagatedFutures.transformAsync( 213 getFileKeysReferencedByAnyGroup(), 214 // Remove all shared file metadata that are not referenced by any group. 215 fileKeysReferencedByAnyGroup -> 216 PropagatedFutures.transformAsync( 217 sharedFilesMetadata.getAllFileKeys(), 218 allFileKeys -> { 219 List<Uri> filesRequiredByMdd = new ArrayList<>(); 220 List<Uri> androidSharedFilesToBeReleased = new ArrayList<>(); 221 // Use AtomicInteger because variables captured by lambdas must be effectively 222 // final. 223 AtomicInteger removedMetadataCount = new AtomicInteger(0); 224 List<ListenableFuture<Void>> futures = new ArrayList<>(); 225 for (NewFileKey newFileKey : allFileKeys) { 226 if (!fileKeysReferencedByAnyGroup.contains(newFileKey)) { 227 ListenableFuture<Void> removeEntryFuture = 228 PropagatedFutures.transformAsync( 229 sharedFilesMetadata.read(newFileKey), 230 sharedFile -> { 231 if (sharedFile != null && sharedFile.getAndroidShared()) { 232 androidSharedFilesToBeReleased.add( 233 DirectoryUtil.getBlobUri( 234 context, sharedFile.getAndroidSharingChecksum())); 235 } 236 return PropagatedFutures.transform( 237 sharedFileManager.removeFileEntry(newFileKey), 238 success -> { 239 if (success) { 240 removedMetadataCount.getAndIncrement(); 241 } else { 242 eventLogger.logEventSampled( 243 MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 244 LogUtil.e( 245 "%s: Unsubscribe from file %s failed!", 246 TAG, newFileKey); 247 } 248 return null; 249 }, 250 sequentialControlExecutor); 251 }, 252 sequentialControlExecutor); 253 futures.add(removeEntryFuture); 254 } else { 255 futures.add( 256 PropagatedFutures.transform( 257 sharedFileManager.getOnDeviceUri(newFileKey), 258 uri -> { 259 if (uri != null) { 260 filesRequiredByMdd.add(uri); 261 } 262 return null; 263 }, 264 sequentialControlExecutor)); 265 } 266 } 267 268 // If isolated structure verification is enabled, include all individual isolated 269 // file uris referenced by fresh groups. This ensures any unaccounted isolated 270 // file uris are removed (i.e. verification is performed). 271 if (flags.enableIsolatedStructureVerification()) { 272 futures.add( 273 PropagatedFutures.transform( 274 getIsolatedFileUrisReferencedByFreshGroups(), 275 referencedIsolatedFileUris -> { 276 filesRequiredByMdd.addAll(referencedIsolatedFileUris); 277 return null; 278 }, 279 sequentialControlExecutor)); 280 } else { 281 // Isolated structure verification is disabled, include the base symlink 282 // directory as required so all isolated file uris under this directory are 283 // _not_ removed (i.e. verification is not performed). 284 filesRequiredByMdd.add( 285 DirectoryUtil.getBaseDownloadSymlinkDirectory(context, instanceId)); 286 } 287 return PropagatedFutures.whenAllComplete(futures) 288 .call( 289 () -> { 290 if (removedMetadataCount.get() > 0) { 291 eventLogger.logMddDataDownloadFileExpirationEvent( 292 0, removedMetadataCount.get()); 293 } 294 Uri parentDirectory = 295 DirectoryUtil.getBaseDownloadDirectory(context, instanceId); 296 int releasedFiles = 297 releaseUnaccountedAndroidSharedFiles( 298 androidSharedFilesToBeReleased); 299 LogUtil.d( 300 "%s: Total %d unaccounted file released. ", TAG, releasedFiles); 301 302 int unaccountedFileCount = 303 deleteUnaccountedFilesRecursively( 304 parentDirectory, filesRequiredByMdd); 305 LogUtil.d( 306 "%s: Total %d unaccounted file deleted. ", 307 TAG, unaccountedFileCount); 308 if (unaccountedFileCount > 0) { 309 eventLogger.logMddDataDownloadFileExpirationEvent( 310 0, unaccountedFileCount); 311 } 312 if (releasedFiles > 0) { 313 eventLogger.logMddDataDownloadFileExpirationEvent(0, releasedFiles); 314 } 315 return null; 316 }, 317 sequentialControlExecutor); 318 }, 319 sequentialControlExecutor), 320 sequentialControlExecutor); 321 } 322 323 private ListenableFuture<Set<NewFileKey>> getFileKeysReferencedByAnyGroup() { 324 return PropagatedFutures.transformAsync( 325 fileGroupsMetadata.getAllFreshGroups(), 326 allGroupsByKey -> { 327 Set<NewFileKey> fileKeysReferencedByAnyGroup = new HashSet<>(); 328 List<DataFileGroupInternal> dataFileGroups = new ArrayList<>(); 329 for (GroupKeyAndGroup dataFileGroupPair : allGroupsByKey) { 330 dataFileGroups.add(dataFileGroupPair.dataFileGroup()); 331 } 332 return PropagatedFutures.transform( 333 fileGroupsMetadata.getAllStaleGroups(), 334 staleGroups -> { 335 dataFileGroups.addAll(staleGroups); 336 for (DataFileGroupInternal dataFileGroup : dataFileGroups) { 337 for (DataFile dataFile : dataFileGroup.getFileList()) { 338 fileKeysReferencedByAnyGroup.add( 339 SharedFilesMetadata.createKeyFromDataFileForCurrentVersion( 340 context, 341 dataFile, 342 dataFileGroup.getAllowedReadersEnum(), 343 silentFeedback)); 344 } 345 } 346 return fileKeysReferencedByAnyGroup; 347 }, 348 sequentialControlExecutor); 349 }, 350 sequentialControlExecutor); 351 } 352 353 /** 354 * Get all isolated file uris that are referenced by any fresh groups. 355 * 356 * <p>Fresh groups are active/pending groups. Isolated file uris are expected when 1) the OS 357 * version supports symlinks (at least Lollipop (21)); and 2) The file group enables file 358 * isolation. 359 * 360 * @return ListenableFuture that resolves with List of isolated uris that are referenced by 361 * active/pending groups 362 */ 363 private ListenableFuture<List<Uri>> getIsolatedFileUrisReferencedByFreshGroups() { 364 List<Uri> referencedIsolatedFileUris = new ArrayList<>(); 365 return PropagatedFutures.transform( 366 fileGroupsMetadata.getAllFreshGroups(), 367 groupKeyAndGroupList -> { 368 for (GroupKeyAndGroup groupKeyAndGroup : groupKeyAndGroupList) { 369 DataFileGroupInternal freshGroup = groupKeyAndGroup.dataFileGroup(); 370 // Skip any groups that don't support isolated structures 371 if (!FileGroupUtil.isIsolatedStructureAllowed(freshGroup)) { 372 continue; 373 } 374 375 // Add the expected isolated file uris for each file 376 for (DataFile file : freshGroup.getFileList()) { 377 Uri isolatedFileUri = 378 FileGroupUtil.getIsolatedFileUri(context, instanceId, file, freshGroup); 379 referencedIsolatedFileUris.add(isolatedFileUri); 380 } 381 } 382 383 return referencedIsolatedFileUris; 384 }, 385 sequentialControlExecutor); 386 } 387 388 private int releaseUnaccountedAndroidSharedFiles(List<Uri> androidSharedFilesToBeReleased) { 389 int releasedFiles = 0; 390 for (Uri sharedFile : androidSharedFilesToBeReleased) { 391 try { 392 fileStorage.deleteFile(sharedFile); 393 releasedFiles += 1; 394 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 395 } catch (IOException e) { 396 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 397 LogUtil.e(e, "%s: Failed to release unaccounted file!", TAG); 398 } 399 } 400 return releasedFiles; 401 } 402 403 // TODO(b/119622504) Fix nullness violation: incompatible types in argument. 404 @SuppressWarnings("nullness:argument") 405 private int deleteUnaccountedFilesRecursively(Uri directory, List<Uri> filesRequiredByMdd) { 406 int unaccountedFileCount = 0; 407 try { 408 if (!fileStorage.exists(directory)) { 409 return unaccountedFileCount; 410 } 411 412 for (Uri uri : fileStorage.children(directory)) { 413 try { 414 if (isContainedInUriList(uri, filesRequiredByMdd)) { 415 continue; 416 } 417 if (fileStorage.isDirectory(uri)) { 418 unaccountedFileCount += deleteUnaccountedFilesRecursively(uri, filesRequiredByMdd); 419 } else { 420 LogUtil.d("%s: Deleted unaccounted file with uri %s!", TAG, uri.getPath()); 421 fileStorage.deleteFile(uri); 422 unaccountedFileCount++; 423 } 424 425 } catch (IOException e) { 426 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 427 LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); 428 } 429 } 430 431 } catch (IOException e) { 432 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); 433 LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); 434 } 435 return unaccountedFileCount; 436 } 437 438 /** 439 * Returns true if given uri is within the given uri list or is a child of any uri in the list. 440 * 441 * <p>Used by MDD's unaccounted file logic to filter out files that shouldn't be deleted. This is 442 * used in two cases: 443 * 444 * <ul> 445 * <li>files referred by any active MDD files. This includes internal MDD files, such as delta 446 * files of a full active file, which are stored using the active file name and a checksum 447 * suffix. 448 * <li>symlinks created for an isolated file structure. These symlinks will reference active 449 * files and their lifecycle is managed on the file group level, rather than as individual 450 * files. 451 * </ul> 452 */ 453 private boolean isContainedInUriList(Uri uri, List<Uri> uriList) { 454 for (Uri activeUri : uriList) { 455 if (uri.toString().startsWith(activeUri.toString())) { 456 return true; 457 } 458 } 459 return false; 460 } 461 } 462