• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2022 Google LLC
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.google.android.libraries.mobiledatadownload.internal.logging;
17 
18 import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR;
19 import static com.google.common.util.concurrent.Futures.immediateFuture;
20 
21 import android.content.Context;
22 import android.net.Uri;
23 import com.google.android.libraries.mobiledatadownload.SilentFeedback;
24 import com.google.android.libraries.mobiledatadownload.annotations.InstanceId;
25 import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage;
26 import com.google.android.libraries.mobiledatadownload.file.openers.RecursiveSizeOpener;
27 import com.google.android.libraries.mobiledatadownload.internal.ApplicationContext;
28 import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata;
29 import com.google.android.libraries.mobiledatadownload.internal.MddConstants;
30 import com.google.android.libraries.mobiledatadownload.internal.SharedFileManager;
31 import com.google.android.libraries.mobiledatadownload.internal.SharedFileMissingException;
32 import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata;
33 import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor;
34 import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup;
35 import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil;
36 import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil;
37 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture;
38 import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures;
39 import com.google.common.base.Optional;
40 import com.google.common.base.Preconditions;
41 import com.google.common.base.Splitter;
42 import com.google.common.util.concurrent.ListenableFuture;
43 import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats;
44 import com.google.mobiledatadownload.LogProto.MddStorageStats;
45 import com.google.mobiledatadownload.internal.MetadataProto.DataFile;
46 import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal;
47 import com.google.mobiledatadownload.internal.MetadataProto.GroupKey;
48 import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey;
49 import java.io.IOException;
50 import java.util.ArrayList;
51 import java.util.HashMap;
52 import java.util.HashSet;
53 import java.util.List;
54 import java.util.Map;
55 import java.util.Set;
56 import java.util.concurrent.Executor;
57 import java.util.concurrent.atomic.AtomicLong;
58 import javax.inject.Inject;
59 
60 /**
61  * Log MDD storage stats at daily maintenance. For each file group, it will log the total bytes used
62  * on disk for that file group and the bytes used by the downloaded group.
63  */
64 public class StorageLogger {
65   private static final String TAG = "StorageLogger";
66   private final FileGroupsMetadata fileGroupsMetadata;
67   private final SharedFileManager sharedFileManager;
68   private final SynchronousFileStorage fileStorage;
69   private final EventLogger eventLogger;
70   private final Context context;
71   private final SilentFeedback silentFeedback;
72   private final Optional<String> instanceId;
73   private final Executor sequentialControlExecutor;
74 
75   /** Store the storage stats for a file group. */
76   static class GroupStorage {
77     // The sum of all on-disk file sizes of the files belonging to this file group, in bytes.
78     public long totalBytesUsed;
79 
80     // The sum of all on-disk inline file sizes of the files belonging to this file group, in bytes.
81     public long totalInlineBytesUsed;
82 
83     // The sum of all on-disk file sizes of this downloaded file group in bytes.
84     public long downloadedGroupBytesUsed;
85 
86     // The sum of all on-disk inline files sizes of this downloaded file group in bytes.
87     public long downloadedGroupInlineBytesUsed;
88 
89     // The total number of files in the group.
90     public int totalFileCount;
91 
92     // The number of inline files in the group.
93     public int totalInlineFileCount;
94   }
95 
96   @Inject
StorageLogger( @pplicationContext Context context, FileGroupsMetadata fileGroupsMetadata, SharedFileManager sharedFileManager, SynchronousFileStorage fileStorage, EventLogger eventLogger, SilentFeedback silentFeedback, @InstanceId Optional<String> instanceId, @SequentialControlExecutor Executor sequentialControlExecutor)97   public StorageLogger(
98       @ApplicationContext Context context,
99       FileGroupsMetadata fileGroupsMetadata,
100       SharedFileManager sharedFileManager,
101       SynchronousFileStorage fileStorage,
102       EventLogger eventLogger,
103       SilentFeedback silentFeedback,
104       @InstanceId Optional<String> instanceId,
105       @SequentialControlExecutor Executor sequentialControlExecutor) {
106     this.context = context;
107     this.fileGroupsMetadata = fileGroupsMetadata;
108     this.sharedFileManager = sharedFileManager;
109     this.fileStorage = fileStorage;
110     this.eventLogger = eventLogger;
111     this.silentFeedback = silentFeedback;
112     this.instanceId = instanceId;
113     this.sequentialControlExecutor = sequentialControlExecutor;
114   }
115 
116   // TODO(b/64764648): Combine this with MobileDataDownloadManager.createGroupKey
createGroupKey(DataFileGroupInternal fileGroup)117   private static GroupKey createGroupKey(DataFileGroupInternal fileGroup) {
118     GroupKey.Builder groupKey = GroupKey.newBuilder().setGroupName(fileGroup.getGroupName());
119 
120     if (fileGroup.getOwnerPackage().isEmpty()) {
121       groupKey.setOwnerPackage(MddConstants.GMS_PACKAGE);
122     } else {
123       groupKey.setOwnerPackage(fileGroup.getOwnerPackage());
124     }
125 
126     return groupKey.build();
127   }
128 
logStorageStats(int daysSinceLastLog)129   public ListenableFuture<Void> logStorageStats(int daysSinceLastLog) {
130     return eventLogger.logMddStorageStats(() -> buildStorageStatsLogData(daysSinceLastLog));
131   }
132 
buildStorageStatsLogData(int daysSinceLastLog)133   private ListenableFuture<MddStorageStats> buildStorageStatsLogData(int daysSinceLastLog) {
134     return PropagatedFluentFuture.from(fileGroupsMetadata.getAllFreshGroups())
135         .transformAsync(
136             allGroups ->
137                 PropagatedFutures.transformAsync(
138                     fileGroupsMetadata.getAllStaleGroups(),
139                     staleGroups ->
140                         buildStorageStatsInternal(allGroups, staleGroups, daysSinceLastLog),
141                     sequentialControlExecutor),
142             sequentialControlExecutor);
143   }
144 
buildStorageStatsInternal( List<GroupKeyAndGroup> allKeysAndGroupPairs, List<DataFileGroupInternal> staleGroups, int daysSinceLastLog)145   private ListenableFuture<MddStorageStats> buildStorageStatsInternal(
146       List<GroupKeyAndGroup> allKeysAndGroupPairs,
147       List<DataFileGroupInternal> staleGroups,
148       int daysSinceLastLog) {
149 
150     List<GroupKeyAndGroup> allKeysAndGroups = new ArrayList<>();
151     for (GroupKeyAndGroup groupKeyAndGroup : allKeysAndGroupPairs) {
152       allKeysAndGroups.add(groupKeyAndGroup);
153     }
154 
155     // Adding staleGroups to allGroups.
156     for (DataFileGroupInternal fileGroup : staleGroups) {
157       allKeysAndGroups.add(GroupKeyAndGroup.create(createGroupKey(fileGroup), fileGroup));
158     }
159 
160     Map<String, GroupStorage> groupKeyToGroupStorage = new HashMap<>();
161     Map<String, Set<NewFileKey>> groupKeyToFileKeys = new HashMap<>();
162     Map<String, Set<NewFileKey>> downloadedGroupKeyToFileKeys = new HashMap<>();
163     Map<String, DataFileGroupInternal> downloadedGroupKeyToDataFileGroup = new HashMap<>();
164 
165     Set<NewFileKey> allFileKeys = new HashSet<>();
166     // Our bytes counter has to be wrapped in an Object because variables captured by lambda
167     // expressions need to be "effectively final" - meaning they never appear on the left-hand side
168     // of an assignment statement. As such, we use AtomicLong.
169     AtomicLong totalMddBytesUsed = new AtomicLong(0L);
170 
171     List<ListenableFuture<Void>> futures = new ArrayList<>();
172     for (GroupKeyAndGroup groupKeyAndGroup : allKeysAndGroups) {
173 
174       Set<NewFileKey> fileKeys =
175           safeGetFileKeys(
176               groupKeyToFileKeys, getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey()));
177 
178       GroupStorage groupStorage =
179           safeGetGroupStorage(
180               groupKeyToGroupStorage, getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey()));
181 
182       Set<NewFileKey> downloadedFileKeysInit = null;
183 
184       if (groupKeyAndGroup.groupKey().getDownloaded()) {
185         downloadedFileKeysInit =
186             safeGetFileKeys(
187                 downloadedGroupKeyToFileKeys,
188                 getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey()));
189         downloadedGroupKeyToDataFileGroup.put(
190             getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey()),
191             groupKeyAndGroup.dataFileGroup());
192       }
193 
194       // Variables captured by lambdas must be effectively final.
195       Set<NewFileKey> downloadedFileKeys = downloadedFileKeysInit;
196       int totalFileCount = groupKeyAndGroup.dataFileGroup().getFileCount();
197       for (DataFile dataFile : groupKeyAndGroup.dataFileGroup().getFileList()) {
198         boolean isInlineFile = FileGroupUtil.isInlineFile(dataFile);
199 
200         NewFileKey fileKey =
201             SharedFilesMetadata.createKeyFromDataFile(
202                 dataFile, groupKeyAndGroup.dataFileGroup().getAllowedReadersEnum());
203         futures.add(
204             PropagatedFutures.transform(
205                 computeFileSize(fileKey),
206                 fileSize -> {
207                   if (!allFileKeys.contains(fileKey)) {
208                     totalMddBytesUsed.getAndAdd(fileSize);
209                     allFileKeys.add(fileKey);
210                   }
211 
212                   // Check if we have processed this fileKey before.
213                   if (!fileKeys.contains(fileKey)) {
214                     if (isInlineFile) {
215                       groupStorage.totalInlineBytesUsed += fileSize;
216                     }
217 
218                     groupStorage.totalBytesUsed += fileSize;
219                     fileKeys.add(fileKey);
220                   }
221 
222                   if (groupKeyAndGroup.groupKey().getDownloaded()) {
223                     // Note: Nullness checker is not smart enough to figure out that
224                     // downloadedFileKeys is never null.
225                     Preconditions.checkNotNull(downloadedFileKeys);
226                     // Check if we have processed this fileKey before.
227                     if (!downloadedFileKeys.contains(fileKey)) {
228                       if (isInlineFile) {
229                         groupStorage.downloadedGroupInlineBytesUsed += fileSize;
230                         groupStorage.totalInlineFileCount += 1;
231                       }
232 
233                       groupStorage.downloadedGroupBytesUsed += fileSize;
234                       downloadedFileKeys.add(fileKey);
235                     }
236                   }
237                   return null;
238                 },
239                 sequentialControlExecutor));
240       }
241       groupStorage.totalFileCount = totalFileCount;
242     }
243 
244     return PropagatedFutures.whenAllComplete(futures)
245         .call(
246             () -> {
247               MddStorageStats.Builder storageStatsBuilder = MddStorageStats.newBuilder();
248               for (String groupName : groupKeyToGroupStorage.keySet()) {
249                 GroupStorage groupStorage = groupKeyToGroupStorage.get(groupName);
250                 List<String> groupNameAndOwnerPackage =
251                     Splitter.on(SPLIT_CHAR).splitToList(groupName);
252 
253                 DataDownloadFileGroupStats.Builder fileGroupDetailsBuilder =
254                     DataDownloadFileGroupStats.newBuilder()
255                         .setFileGroupName(groupNameAndOwnerPackage.get(0))
256                         .setOwnerPackage(groupNameAndOwnerPackage.get(1))
257                         .setFileCount(groupStorage.totalFileCount)
258                         .setInlineFileCount(groupStorage.totalInlineFileCount);
259 
260                 DataFileGroupInternal dataFileGroup =
261                     downloadedGroupKeyToDataFileGroup.get(groupName);
262 
263                 if (dataFileGroup == null) {
264                   fileGroupDetailsBuilder.setFileGroupVersionNumber(-1);
265                 } else {
266                   fileGroupDetailsBuilder
267                       .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber())
268                       .setBuildId(dataFileGroup.getBuildId())
269                       .setVariantId(dataFileGroup.getVariantId());
270                 }
271 
272                 storageStatsBuilder.addDataDownloadFileGroupStats(fileGroupDetailsBuilder.build());
273 
274                 storageStatsBuilder.addTotalBytesUsed(groupStorage.totalBytesUsed);
275                 storageStatsBuilder.addTotalInlineBytesUsed(groupStorage.totalInlineBytesUsed);
276                 storageStatsBuilder.addDownloadedGroupBytesUsed(
277                     groupStorage.downloadedGroupBytesUsed);
278                 storageStatsBuilder.addDownloadedGroupInlineBytesUsed(
279                     groupStorage.downloadedGroupInlineBytesUsed);
280               }
281 
282               storageStatsBuilder.setTotalMddBytesUsed(totalMddBytesUsed.get());
283 
284               long mddDirectoryBytesUsed = 0;
285               try {
286                 Uri uri = DirectoryUtil.getBaseDownloadDirectory(context, instanceId);
287                 if (fileStorage.exists(uri)) {
288                   mddDirectoryBytesUsed = fileStorage.open(uri, RecursiveSizeOpener.create());
289                 }
290               } catch (IOException e) {
291                 mddDirectoryBytesUsed = 0;
292                 LogUtil.e(
293                     e, "%s: Failed to call Mobstore to compute MDD Directory bytes used!", TAG);
294                 silentFeedback.send(
295                     e, "Failed to call Mobstore to compute MDD Directory bytes used!");
296               }
297 
298               storageStatsBuilder
299                   .setTotalMddDirectoryBytesUsed(mddDirectoryBytesUsed)
300                   .setDaysSinceLastLog(daysSinceLastLog);
301 
302               return storageStatsBuilder.build();
303             },
304             sequentialControlExecutor);
305   }
306 
getGroupWithOwnerPackageKey(GroupKey groupKey)307   private String getGroupWithOwnerPackageKey(GroupKey groupKey) {
308     return new StringBuilder(groupKey.getGroupName())
309         .append(SPLIT_CHAR)
310         .append(groupKey.getOwnerPackage())
311         .toString();
312   }
313 
safeGetFileKeys( Map<String, Set<NewFileKey>> groupNameToFileKeys, String groupName)314   private Set<NewFileKey> safeGetFileKeys(
315       Map<String, Set<NewFileKey>> groupNameToFileKeys, String groupName) {
316     Set<NewFileKey> fileKeys = groupNameToFileKeys.get(groupName);
317     if (fileKeys == null) {
318       groupNameToFileKeys.put(groupName, new HashSet<>());
319       fileKeys = groupNameToFileKeys.get(groupName);
320     }
321     return fileKeys;
322   }
323 
safeGetGroupStorage( Map<String, GroupStorage> groupNameToStats, String groupName)324   private GroupStorage safeGetGroupStorage(
325       Map<String, GroupStorage> groupNameToStats, String groupName) {
326     GroupStorage groupStorage = groupNameToStats.get(groupName);
327     if (groupStorage == null) {
328       groupNameToStats.put(groupName, new GroupStorage());
329       groupStorage = groupNameToStats.get(groupName);
330     }
331     return groupStorage;
332   }
333 
computeFileSize(NewFileKey newFileKey)334   private ListenableFuture<Long> computeFileSize(NewFileKey newFileKey) {
335     return PropagatedFluentFuture.from(sharedFileManager.getOnDeviceUri(newFileKey))
336         .catchingAsync(
337             SharedFileMissingException.class, e -> immediateFuture(null), sequentialControlExecutor)
338         .transform(
339             fileUri -> {
340               if (fileUri != null) {
341                 try {
342                   return fileStorage.fileSize(fileUri);
343                 } catch (IOException e) {
344                   LogUtil.e(e, "%s: Failed to call mobstore fileSize on uri %s!", TAG, fileUri);
345                 }
346               }
347               return 0L;
348             },
349             sequentialControlExecutor);
350   }
351 }
352