• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2022 Google LLC
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.google.android.libraries.mobiledatadownload.internal.downloader;
17 
18 import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
19 import static com.google.common.util.concurrent.Futures.immediateFuture;
20 import static com.google.common.util.concurrent.Futures.immediateVoidFuture;
21 
22 import android.net.Uri;
23 import android.os.Build.VERSION;
24 import android.os.Build.VERSION_CODES;
25 import androidx.annotation.VisibleForTesting;
26 import com.google.android.libraries.mobiledatadownload.DownloadException;
27 import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode;
28 import com.google.android.libraries.mobiledatadownload.Flags;
29 import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage;
30 import com.google.android.libraries.mobiledatadownload.file.openers.ReadStreamOpener;
31 import com.google.android.libraries.mobiledatadownload.file.openers.RecursiveSizeOpener;
32 import com.google.android.libraries.mobiledatadownload.file.openers.WriteStreamOpener;
33 import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos;
34 import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata;
35 import com.google.android.libraries.mobiledatadownload.internal.downloader.MddFileDownloader.DownloaderCallback;
36 import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger;
37 import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil;
38 import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil;
39 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture;
40 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures;
41 import com.google.common.io.ByteStreams;
42 import com.google.common.util.concurrent.ListenableFuture;
43 import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent;
44 import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats;
45 import com.google.mobiledatadownload.internal.MetadataProto.DataFile;
46 import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders;
47 import com.google.mobiledatadownload.internal.MetadataProto.FileStatus;
48 import com.google.mobiledatadownload.internal.MetadataProto.GroupKey;
49 import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey;
50 import com.google.mobiledatadownload.internal.MetadataProto.SharedFile;
51 import java.io.IOException;
52 import java.io.InputStream;
53 import java.io.OutputStream;
54 import java.util.concurrent.Executor;
55 
56 /**
57  * Impl for {@link DownloaderCallback}, that is called by the file downloader on download complete
58  * or failed events
59  */
60 public class DownloaderCallbackImpl implements DownloaderCallback {
61 
62   private static final String TAG = "DownloaderCallbackImpl";
63 
64   private final SharedFilesMetadata sharedFilesMetadata;
65   private final SynchronousFileStorage fileStorage;
66   private final DataFile dataFile;
67   private final AllowedReaders allowedReaders;
68   private final String checksum;
69   private final EventLogger eventLogger;
70   private final GroupKey groupKey;
71   private final int fileGroupVersionNumber;
72   private final long buildId;
73   private final String variantId;
74   private final Flags flags;
75   private final Executor sequentialControlExecutor;
76 
DownloaderCallbackImpl( SharedFilesMetadata sharedFilesMetadata, SynchronousFileStorage fileStorage, DataFile dataFile, AllowedReaders allowedReaders, EventLogger eventLogger, GroupKey groupKey, int fileGroupVersionNumber, long buildId, String variantId, Flags flags, Executor sequentialControlExecutor)77   public DownloaderCallbackImpl(
78       SharedFilesMetadata sharedFilesMetadata,
79       SynchronousFileStorage fileStorage,
80       DataFile dataFile,
81       AllowedReaders allowedReaders,
82       EventLogger eventLogger,
83       GroupKey groupKey,
84       int fileGroupVersionNumber,
85       long buildId,
86       String variantId,
87       Flags flags,
88       Executor sequentialControlExecutor) {
89     this.sharedFilesMetadata = sharedFilesMetadata;
90     this.fileStorage = fileStorage;
91     this.dataFile = dataFile;
92     this.allowedReaders = allowedReaders;
93     checksum = FileGroupUtil.getFileChecksum(dataFile);
94     this.eventLogger = eventLogger;
95     this.groupKey = groupKey;
96     this.fileGroupVersionNumber = fileGroupVersionNumber;
97     this.buildId = buildId;
98     this.variantId = variantId;
99     this.flags = flags;
100     this.sequentialControlExecutor = sequentialControlExecutor;
101   }
102 
103   @Override
onDownloadComplete(Uri fileUri)104   public ListenableFuture<Void> onDownloadComplete(Uri fileUri) {
105     LogUtil.d("%s: Successfully downloaded file %s", TAG, checksum);
106 
107     // Use DownloadedFileChecksum to verify downloaded file integrity if the file has Download
108     // Transforms
109     String downloadedFileChecksum =
110         dataFile.hasDownloadTransforms()
111             ? dataFile.getDownloadedFileChecksum()
112             : dataFile.getChecksum();
113 
114     try {
115       FileValidator.validateDownloadedFile(fileStorage, dataFile, fileUri, downloadedFileChecksum);
116 
117       if (dataFile.hasDownloadTransforms()) {
118         handleDownloadTransform(fileUri);
119       }
120     } catch (DownloadException exception) {
121       if (exception
122           .getDownloadResultCode()
123           .equals(DownloadResultCode.DOWNLOADED_FILE_CHECKSUM_MISMATCH_ERROR)) {
124         // File was downloaded successfully, but failed checksum mismatch error. Attempt to delete
125         // the file, then fail with the given exception.
126         return PropagatedFluentFuture.from(
127                 maybeDeleteFileOnChecksumMismatch(
128                     sharedFilesMetadata,
129                     dataFile,
130                     allowedReaders,
131                     fileStorage,
132                     fileUri,
133                     checksum,
134                     eventLogger,
135                     flags,
136                     sequentialControlExecutor))
137             .catchingAsync(
138                 IOException.class,
139                 ioException -> {
140                   // Delete on checksum failed, add it as a suppressed exception if supported (API
141                   // level 19 or higher).
142                   if (VERSION.SDK_INT >= VERSION_CODES.KITKAT) {
143                     exception.addSuppressed(ioException);
144                   }
145                   return immediateVoidFuture();
146                 },
147                 sequentialControlExecutor)
148             .transformAsync(unused -> immediateFailedFuture(exception), sequentialControlExecutor);
149       }
150       return immediateFailedFuture(exception);
151     }
152 
153     return updateFileStatus(
154         FileStatus.DOWNLOAD_COMPLETE,
155         dataFile,
156         allowedReaders,
157         sharedFilesMetadata,
158         sequentialControlExecutor);
159   }
160 
161   @Override
onDownloadFailed(DownloadException exception)162   public ListenableFuture<Void> onDownloadFailed(DownloadException exception) {
163     LogUtil.d("%s: Failed to download file %s", TAG, checksum);
164     if (exception
165         .getDownloadResultCode()
166         .equals(DownloadResultCode.DOWNLOADED_FILE_CHECKSUM_MISMATCH_ERROR)) {
167       return updateFileStatus(
168           FileStatus.CORRUPTED,
169           dataFile,
170           allowedReaders,
171           sharedFilesMetadata,
172           sequentialControlExecutor);
173     }
174     return updateFileStatus(
175         FileStatus.DOWNLOAD_FAILED,
176         dataFile,
177         allowedReaders,
178         sharedFilesMetadata,
179         sequentialControlExecutor);
180   }
181 
handleDownloadTransform(Uri downloadedFileUri)182   private void handleDownloadTransform(Uri downloadedFileUri) throws DownloadException {
183     if (!dataFile.hasDownloadTransforms()) {
184       return;
185     }
186     Uri finalFileUri = FileNameUtil.getFinalFileUriWithTempDownloadedFile(downloadedFileUri);
187     if (FileGroupUtil.hasZipDownloadTransform(dataFile)) {
188       applyZipDownloadTransforms(
189           eventLogger,
190           fileStorage,
191           downloadedFileUri,
192           finalFileUri,
193           groupKey,
194           fileGroupVersionNumber,
195           buildId,
196           variantId,
197           dataFile.getFileId());
198     } else {
199       handleNonZipDownloadTransform(downloadedFileUri, finalFileUri);
200     }
201   }
202 
handleNonZipDownloadTransform(Uri downloadedFileUri, Uri finalFileUri)203   private void handleNonZipDownloadTransform(Uri downloadedFileUri, Uri finalFileUri)
204       throws DownloadException {
205     Uri downloadFileUriWithTransform;
206     try {
207       downloadFileUriWithTransform =
208           downloadedFileUri
209               .buildUpon()
210               .encodedFragment(TransformProtos.toEncodedFragment(dataFile.getDownloadTransforms()))
211               .build();
212     } catch (IllegalArgumentException e) {
213       LogUtil.e(e, "%s: Exception while trying to serialize download transform", TAG);
214       throw DownloadException.builder()
215           .setDownloadResultCode(DownloadResultCode.UNABLE_TO_SERIALIZE_DOWNLOAD_TRANSFORM_ERROR)
216           .setCause(e)
217           .build();
218     }
219     applyDownloadTransforms(
220         eventLogger,
221         fileStorage,
222         downloadFileUriWithTransform,
223         finalFileUri,
224         groupKey,
225         fileGroupVersionNumber,
226         buildId,
227         variantId,
228         dataFile);
229     // Verify original checksum if provided.
230     if (dataFile.getChecksumType() != DataFile.ChecksumType.NONE
231         && !FileValidator.verifyChecksum(fileStorage, finalFileUri, checksum)) {
232       LogUtil.e("%s: Final file checksum verification failed. %s.", TAG, finalFileUri);
233       throw DownloadException.builder()
234           .setDownloadResultCode(DownloadResultCode.FINAL_FILE_CHECKSUM_MISMATCH_ERROR)
235           .build();
236     }
237   }
238 
239   @VisibleForTesting
applyDownloadTransforms( EventLogger eventLogger, SynchronousFileStorage fileStorage, Uri source, Uri target, GroupKey groupKey, int fileGroupVersionNumber, long buildId, String variantId, DataFile dataFile)240   static void applyDownloadTransforms(
241       EventLogger eventLogger,
242       SynchronousFileStorage fileStorage,
243       Uri source,
244       Uri target,
245       GroupKey groupKey,
246       int fileGroupVersionNumber,
247       long buildId,
248       String variantId,
249       DataFile dataFile)
250       throws DownloadException {
251 
252     try (InputStream in = fileStorage.open(source, ReadStreamOpener.create());
253         OutputStream out = fileStorage.open(target, WriteStreamOpener.create())) {
254       ByteStreams.copy(in, out);
255     } catch (IOException ioe) {
256       LogUtil.e(ioe, "%s: Failed to apply download transform for file %s.", TAG, source);
257       throw DownloadException.builder()
258           .setDownloadResultCode(DownloadResultCode.DOWNLOAD_TRANSFORM_IO_ERROR)
259           .setCause(ioe)
260           .build();
261     }
262     try {
263       if (FileGroupUtil.hasCompressDownloadTransform(dataFile)) {
264         long fullFileSize = fileStorage.fileSize(target);
265         long downloadedFileSize = fileStorage.fileSize(source);
266         if (fullFileSize > downloadedFileSize) {
267           DataDownloadFileGroupStats fileGroupStats =
268               DataDownloadFileGroupStats.newBuilder()
269                   .setFileGroupName(groupKey.getGroupName())
270                   .setFileGroupVersionNumber(fileGroupVersionNumber)
271                   .setBuildId(buildId)
272                   .setVariantId(variantId)
273                   .setOwnerPackage(groupKey.getOwnerPackage())
274                   .build();
275           eventLogger.logMddNetworkSavings(
276               fileGroupStats,
277               0,
278               fullFileSize,
279               downloadedFileSize,
280               dataFile.getFileId(),
281               /* deltaIndex= */ 0);
282         }
283       }
284       fileStorage.deleteFile(source);
285     } catch (IOException ioe) {
286       // Ignore if fails to log file size or delete the temp compress file, as it will eventually
287       // be garbage collected.
288       LogUtil.d(ioe, "%s: Failed to get file size or delete compress file %s.", TAG, source);
289     }
290   }
291 
292   @VisibleForTesting
applyZipDownloadTransforms( EventLogger eventLogger, SynchronousFileStorage fileStorage, Uri source, Uri target, GroupKey groupKey, int fileGroupVersionNumber, long buildId, String variantId, String fileId)293   static void applyZipDownloadTransforms(
294       EventLogger eventLogger,
295       SynchronousFileStorage fileStorage,
296       Uri source,
297       Uri target,
298       GroupKey groupKey,
299       int fileGroupVersionNumber,
300       long buildId,
301       String variantId,
302       String fileId)
303       throws DownloadException {
304 
305     try {
306       fileStorage.open(source, ZipFolderOpener.create(target));
307     } catch (IOException ioe) {
308       LogUtil.e(ioe, "%s: Failed to apply zip download transform for file %s.", TAG, source);
309       throw DownloadException.builder()
310           .setDownloadResultCode(DownloadResultCode.DOWNLOAD_TRANSFORM_IO_ERROR)
311           .setCause(ioe)
312           .build();
313     }
314     try {
315       DataDownloadFileGroupStats fileGroupStats =
316           DataDownloadFileGroupStats.newBuilder()
317               .setFileGroupName(groupKey.getGroupName())
318               .setFileGroupVersionNumber(fileGroupVersionNumber)
319               .setBuildId(buildId)
320               .setVariantId(variantId)
321               .setOwnerPackage(groupKey.getOwnerPackage())
322               .build();
323       eventLogger.logMddNetworkSavings(
324           fileGroupStats,
325           0,
326           getFileOrDirectorySize(fileStorage, target),
327           fileStorage.fileSize(source),
328           fileId,
329           0);
330       // Delete the zip file only if unzip successfully to avoid re-download
331       fileStorage.deleteFile(source);
332     } catch (IOException ioe) {
333       // Ignore if fails to log file size or delete the temp zip file, as it will eventually be
334       // garbage collected.
335       LogUtil.d(ioe, "%s: Failed to get file size or delete zip file %s.", TAG, source);
336     }
337   }
338 
getFileOrDirectorySize(SynchronousFileStorage fileStorage, Uri uri)339   private static long getFileOrDirectorySize(SynchronousFileStorage fileStorage, Uri uri)
340       throws IOException {
341     return fileStorage.open(uri, RecursiveSizeOpener.create());
342   }
343 
344   /** Get {@link SharedFile} or fail with {@link DownloadException}. */
getSharedFileOrFail( SharedFilesMetadata sharedFilesMetadata, NewFileKey newFileKey, Executor sequentialControlExecutor)345   private static ListenableFuture<SharedFile> getSharedFileOrFail(
346       SharedFilesMetadata sharedFilesMetadata,
347       NewFileKey newFileKey,
348       Executor sequentialControlExecutor) {
349     return PropagatedFutures.transformAsync(
350         sharedFilesMetadata.read(newFileKey),
351         sharedFile -> {
352           // Cannot find the file metadata, fail to update the file status.
353           if (sharedFile == null) {
354             // TODO(b/131166925): MDD dump should not use lite proto toString.
355             LogUtil.e("%s: Shared file not found, newFileKey = %s", TAG, newFileKey);
356             return immediateFailedFuture(
357                 DownloadException.builder()
358                     .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR)
359                     .build());
360           }
361 
362           return immediateFuture(sharedFile);
363         },
364         sequentialControlExecutor);
365   }
366 
367   /**
368    * Maybe delete on-device file after a completed download when a checksum mismatch occurs.
369    *
370    * <p>When a checksum mismatch occurs after a completed download, it's possible that the data has
371    * been corrupted on-disk. In this event, we should delete the on-disk file so it can be
372    * redownloaded again in a non-corrupted state.
373    *
374    * <p>However, it's also possible that a bad config was sent with a wrong checksum. In this event,
375    * the on-disk file may not be corrupted, so deleting it could lead to an increase in network
376    * bandwidth usage.
377    *
378    * <p>In order to balance the two cases, MDD will start to delete the on-disk file, but after a
379    * certain number of retries, this deletion will be skipped to prevent unnecessary network
380    * bandwidth usage.
381    *
382    * <p>This future may return a failed future with an IOException if attempting to delete the file
383    * fails.
384    */
385   static ListenableFuture<Void> maybeDeleteFileOnChecksumMismatch(
386       SharedFilesMetadata sharedFilesMetadata,
387       DataFile dataFile,
388       AllowedReaders allowedReaders,
389       SynchronousFileStorage fileStorage,
390       Uri fileUri,
391       String checksum,
392       EventLogger eventLogger,
393       Flags flags,
394       Executor sequentialControlExecutor) {
395     NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile(dataFile, allowedReaders);
396     return PropagatedFluentFuture.from(
397             getSharedFileOrFail(sharedFilesMetadata, newFileKey, sequentialControlExecutor))
398         .transformAsync(
399             sharedFile -> {
400               if (sharedFile.getChecksumMismatchRetryDownloadCount()
401                   >= flags.downloaderMaxRetryOnChecksumMismatchCount()) {
402                 LogUtil.d(
403                     "%s: Checksum mismatch detected but the has already reached retry limit!"
404                         + " Skipping removal for file %s",
405                     TAG, checksum);
406                 eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED);
407               } else {
408                 LogUtil.d(
409                     "%s: Removing file and marking as corrupted due to checksum mismatch", TAG);
410                 try {
411                   fileStorage.deleteFile(fileUri);
412                 } catch (IOException e) {
413                   // Deleting the corrupted file is best effort, the next time MDD attempts to
414                   // download, we will try again to delete the file. For now, just log this error.
415                   LogUtil.e(e, "%s: Failed to remove corrupted file %s", TAG, checksum);
416                   return immediateFailedFuture(e);
417                 }
418               }
419               return immediateVoidFuture();
420             },
421             sequentialControlExecutor);
422   }
423 
424   /**
425    * Find the file metadata and update the file status. Throws {@link DownloadException} if the file
426    * status failed to be updated.
427    */
428   static ListenableFuture<Void> updateFileStatus(
429       FileStatus fileStatus,
430       DataFile dataFile,
431       AllowedReaders allowedReaders,
432       SharedFilesMetadata sharedFilesMetadata,
433       Executor sequentialControlExecutor) {
434     NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile(dataFile, allowedReaders);
435 
436     return PropagatedFluentFuture.from(
437             getSharedFileOrFail(sharedFilesMetadata, newFileKey, sequentialControlExecutor))
438         .transformAsync(
439             sharedFile -> {
440               SharedFile.Builder sharedFileBuilder =
441                   sharedFile.toBuilder().setFileStatus(fileStatus);
442               if (fileStatus.equals(FileStatus.CORRUPTED)) {
443                 // Corrupted state indicates a checksum mismatch failure, so increment the retry
444                 // download count.
445                 sharedFileBuilder.setChecksumMismatchRetryDownloadCount(
446                     sharedFile.getChecksumMismatchRetryDownloadCount() + 1);
447               }
448               return sharedFilesMetadata.write(newFileKey, sharedFileBuilder.build());
449             },
450             sequentialControlExecutor)
451         .transformAsync(
452             writeSuccess -> {
453               if (!writeSuccess) {
454                 // TODO(b/131166925): MDD dump should not use lite proto toString.
455                 LogUtil.e(
456                     "%s: Unable to write back download info for file entry with %s",
457                     TAG, newFileKey);
458                 return immediateFailedFuture(
459                     DownloadException.builder()
460                         .setDownloadResultCode(DownloadResultCode.UNABLE_TO_UPDATE_FILE_STATE_ERROR)
461                         .build());
462               }
463               return immediateVoidFuture();
464             },
465             sequentialControlExecutor);
466   }
467 }
468