1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy 6 * of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations 14 * under the License. 15 */ 16 17 package com.android.server.usage; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.app.usage.UsageEvents; 22 import android.app.usage.UsageStats; 23 import android.app.usage.UsageStatsManager; 24 import android.os.Build; 25 import android.os.SystemClock; 26 import android.os.SystemProperties; 27 import android.util.ArrayMap; 28 import android.util.ArraySet; 29 import android.util.AtomicFile; 30 import android.util.LongSparseArray; 31 import android.util.Slog; 32 import android.util.SparseArray; 33 import android.util.TimeUtils; 34 35 import com.android.internal.annotations.VisibleForTesting; 36 import com.android.internal.annotations.GuardedBy; 37 import com.android.internal.util.ArrayUtils; 38 import com.android.internal.util.IndentingPrintWriter; 39 40 import libcore.io.IoUtils; 41 42 import java.io.BufferedReader; 43 import java.io.BufferedWriter; 44 import java.io.ByteArrayInputStream; 45 import java.io.ByteArrayOutputStream; 46 import java.io.DataInputStream; 47 import java.io.DataOutputStream; 48 import java.io.File; 49 import java.io.FileInputStream; 50 import java.io.FileNotFoundException; 51 import java.io.FileOutputStream; 52 import java.io.FileReader; 53 import java.io.FileWriter; 54 import java.io.FilenameFilter; 55 import java.io.IOException; 56 import java.io.InputStream; 57 import java.io.OutputStream; 58 import java.nio.file.Files; 59 import java.nio.file.StandardCopyOption; 60 import java.util.ArrayList; 61 import java.util.Arrays; 62 import java.util.Collections; 63 import java.util.HashMap; 64 import java.util.List; 65 import java.util.Set; 66 import java.util.concurrent.TimeUnit; 67 68 /** 69 * Provides an interface to query for UsageStat data from a Protocol Buffer database. 70 * 71 * Prior to version 4, UsageStatsDatabase used XML to store Usage Stats data to disk. 72 * When the UsageStatsDatabase version is upgraded, the files on disk are migrated to the new 73 * version on init. The steps of migration are as follows: 74 * 1) Check if version upgrade breadcrumb exists on disk, if so skip to step 4. 75 * 2) Move current files to a timestamped backup directory. 76 * 3) Write a temporary breadcrumb file with some info about the backup directory. 77 * 4) Deserialize the backup files in the timestamped backup folder referenced by the breadcrumb. 78 * 5) Reserialize the data read from the file with the new version format and replace the old files 79 * 6) Repeat Step 3 and 4 for each file in the backup folder. 80 * 7) Update the version file with the new version and build fingerprint. 81 * 8) Delete the time stamped backup folder (unless flagged to be kept). 82 * 9) Delete the breadcrumb file. 83 * 84 * Performing the upgrade steps in this order, protects against unexpected shutdowns mid upgrade 85 * 86 * The backup directory will contain directories with timestamp names. If the upgrade breadcrumb 87 * exists on disk, it will contain a timestamp which will match one of the backup directories. The 88 * breadcrumb will also contain a version number which will denote how the files in the backup 89 * directory should be deserialized. 90 */ 91 public class UsageStatsDatabase { 92 private static final int DEFAULT_CURRENT_VERSION = 5; 93 /** 94 * Current version of the backup schema 95 * 96 * @hide 97 */ 98 @VisibleForTesting 99 public static final int BACKUP_VERSION = 4; 100 101 @VisibleForTesting 102 static final int[] MAX_FILES_PER_INTERVAL_TYPE = new int[]{100, 50, 12, 10}; 103 104 // Key under which the payload blob is stored 105 // same as UsageStatsBackupHelper.KEY_USAGE_STATS 106 static final String KEY_USAGE_STATS = "usage_stats"; 107 108 // Persist versioned backup files. 109 // Should be false, except when testing new versions 110 static final boolean KEEP_BACKUP_DIR = false; 111 112 private static final String TAG = "UsageStatsDatabase"; 113 private static final boolean DEBUG = UsageStatsService.DEBUG; 114 private static final String BAK_SUFFIX = ".bak"; 115 private static final String CHECKED_IN_SUFFIX = UsageStatsXml.CHECKED_IN_SUFFIX; 116 private static final String RETENTION_LEN_KEY = "ro.usagestats.chooser.retention"; 117 private static final int SELECTION_LOG_RETENTION_LEN = 118 SystemProperties.getInt(RETENTION_LEN_KEY, 14); 119 120 private final Object mLock = new Object(); 121 private final File[] mIntervalDirs; 122 @VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE) 123 final LongSparseArray<AtomicFile>[] mSortedStatFiles; 124 private final UnixCalendar mCal; 125 private final File mVersionFile; 126 private final File mBackupsDir; 127 // If this file exists on disk, UsageStatsDatabase is in the middle of migrating files to a new 128 // version. If this file exists on boot, the upgrade was interrupted and needs to be picked up 129 // where it left off. 130 private final File mUpdateBreadcrumb; 131 // Current version of the database files schema 132 private int mCurrentVersion; 133 private boolean mFirstUpdate; 134 private boolean mNewUpdate; 135 private boolean mUpgradePerformed; 136 137 // The obfuscated packages to tokens mappings file 138 private final File mPackageMappingsFile; 139 // Holds all of the data related to the obfuscated packages and their token mappings. 140 @GuardedBy("mLock") 141 final PackagesTokenData mPackagesTokenData = new PackagesTokenData(); 142 143 /** 144 * UsageStatsDatabase constructor that allows setting the version number. 145 * This should only be used for testing. 146 * 147 * @hide 148 */ 149 @VisibleForTesting UsageStatsDatabase(File dir, int version)150 public UsageStatsDatabase(File dir, int version) { 151 mIntervalDirs = new File[]{ 152 new File(dir, "daily"), 153 new File(dir, "weekly"), 154 new File(dir, "monthly"), 155 new File(dir, "yearly"), 156 }; 157 mCurrentVersion = version; 158 mVersionFile = new File(dir, "version"); 159 mBackupsDir = new File(dir, "backups"); 160 mUpdateBreadcrumb = new File(dir, "breadcrumb"); 161 mSortedStatFiles = new LongSparseArray[mIntervalDirs.length]; 162 mPackageMappingsFile = new File(dir, "mappings"); 163 mCal = new UnixCalendar(0); 164 } 165 UsageStatsDatabase(File dir)166 public UsageStatsDatabase(File dir) { 167 this(dir, DEFAULT_CURRENT_VERSION); 168 } 169 170 /** 171 * Initialize any directories required and index what stats are available. 172 */ init(long currentTimeMillis)173 public void init(long currentTimeMillis) { 174 synchronized (mLock) { 175 for (File f : mIntervalDirs) { 176 f.mkdirs(); 177 if (!f.exists()) { 178 throw new IllegalStateException("Failed to create directory " 179 + f.getAbsolutePath()); 180 } 181 } 182 183 checkVersionAndBuildLocked(); 184 // Perform a pruning of older files if there was an upgrade, otherwise do indexing. 185 if (mUpgradePerformed) { 186 prune(currentTimeMillis); // prune performs an indexing when done 187 } else { 188 indexFilesLocked(); 189 } 190 191 // Delete files that are in the future. 192 for (LongSparseArray<AtomicFile> files : mSortedStatFiles) { 193 final int startIndex = files.firstIndexOnOrAfter(currentTimeMillis); 194 if (startIndex < 0) { 195 continue; 196 } 197 198 final int fileCount = files.size(); 199 for (int i = startIndex; i < fileCount; i++) { 200 files.valueAt(i).delete(); 201 } 202 203 // Remove in a separate loop because any accesses (valueAt) 204 // will cause a gc in the SparseArray and mess up the order. 205 for (int i = startIndex; i < fileCount; i++) { 206 files.removeAt(i); 207 } 208 } 209 } 210 } 211 212 public interface CheckinAction { checkin(IntervalStats stats)213 boolean checkin(IntervalStats stats); 214 } 215 216 /** 217 * Calls {@link CheckinAction#checkin(IntervalStats)} on the given {@link CheckinAction} 218 * for all {@link IntervalStats} that haven't been checked-in. 219 * If any of the calls to {@link CheckinAction#checkin(IntervalStats)} returns false or throws 220 * an exception, the check-in will be aborted. 221 * 222 * @param checkinAction The callback to run when checking-in {@link IntervalStats}. 223 * @return true if the check-in succeeded. 224 */ checkinDailyFiles(CheckinAction checkinAction)225 public boolean checkinDailyFiles(CheckinAction checkinAction) { 226 synchronized (mLock) { 227 final LongSparseArray<AtomicFile> files = 228 mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY]; 229 final int fileCount = files.size(); 230 231 // We may have holes in the checkin (if there was an error) 232 // so find the last checked-in file and go from there. 233 int lastCheckin = -1; 234 for (int i = 0; i < fileCount - 1; i++) { 235 if (files.valueAt(i).getBaseFile().getPath().endsWith(CHECKED_IN_SUFFIX)) { 236 lastCheckin = i; 237 } 238 } 239 240 final int start = lastCheckin + 1; 241 if (start == fileCount - 1) { 242 return true; 243 } 244 245 try { 246 for (int i = start; i < fileCount - 1; i++) { 247 final IntervalStats stats = new IntervalStats(); 248 readLocked(files.valueAt(i), stats, false); 249 if (!checkinAction.checkin(stats)) { 250 return false; 251 } 252 } 253 } catch (Exception e) { 254 Slog.e(TAG, "Failed to check-in", e); 255 return false; 256 } 257 258 // We have successfully checked-in the stats, so rename the files so that they 259 // are marked as checked-in. 260 for (int i = start; i < fileCount - 1; i++) { 261 final AtomicFile file = files.valueAt(i); 262 final File checkedInFile = new File( 263 file.getBaseFile().getPath() + CHECKED_IN_SUFFIX); 264 if (!file.getBaseFile().renameTo(checkedInFile)) { 265 // We must return success, as we've already marked some files as checked-in. 266 // It's better to repeat ourselves than to lose data. 267 Slog.e(TAG, "Failed to mark file " + file.getBaseFile().getPath() 268 + " as checked-in"); 269 return true; 270 } 271 272 // AtomicFile needs to set a new backup path with the same -c extension, so 273 // we replace the old AtomicFile with the updated one. 274 files.setValueAt(i, new AtomicFile(checkedInFile)); 275 } 276 } 277 return true; 278 } 279 280 /** @hide */ 281 @VisibleForTesting forceIndexFiles()282 void forceIndexFiles() { 283 synchronized (mLock) { 284 indexFilesLocked(); 285 } 286 } 287 indexFilesLocked()288 private void indexFilesLocked() { 289 final FilenameFilter backupFileFilter = new FilenameFilter() { 290 @Override 291 public boolean accept(File dir, String name) { 292 return !name.endsWith(BAK_SUFFIX); 293 } 294 }; 295 // Index the available usage stat files on disk. 296 for (int i = 0; i < mSortedStatFiles.length; i++) { 297 if (mSortedStatFiles[i] == null) { 298 mSortedStatFiles[i] = new LongSparseArray<>(); 299 } else { 300 mSortedStatFiles[i].clear(); 301 } 302 File[] files = mIntervalDirs[i].listFiles(backupFileFilter); 303 if (files != null) { 304 if (DEBUG) { 305 Slog.d(TAG, "Found " + files.length + " stat files for interval " + i); 306 } 307 final int len = files.length; 308 for (int j = 0; j < len; j++) { 309 final File f = files[j]; 310 final AtomicFile af = new AtomicFile(f); 311 try { 312 mSortedStatFiles[i].put(parseBeginTime(af), af); 313 } catch (IOException e) { 314 Slog.e(TAG, "failed to index file: " + f, e); 315 } 316 } 317 318 // only keep the max allowed number of files for each interval type. 319 final int toDelete = mSortedStatFiles[i].size() - MAX_FILES_PER_INTERVAL_TYPE[i]; 320 if (toDelete > 0) { 321 for (int j = 0; j < toDelete; j++) { 322 mSortedStatFiles[i].valueAt(0).delete(); 323 mSortedStatFiles[i].removeAt(0); 324 } 325 Slog.d(TAG, "Deleted " + toDelete + " stat files for interval " + i); 326 } 327 } 328 } 329 } 330 331 /** 332 * Is this the first update to the system from L to M? 333 */ isFirstUpdate()334 boolean isFirstUpdate() { 335 return mFirstUpdate; 336 } 337 338 /** 339 * Is this a system update since we started tracking build fingerprint in the version file? 340 */ isNewUpdate()341 boolean isNewUpdate() { 342 return mNewUpdate; 343 } 344 345 /** 346 * Was an upgrade performed when this database was initialized? 347 */ wasUpgradePerformed()348 boolean wasUpgradePerformed() { 349 return mUpgradePerformed; 350 } 351 checkVersionAndBuildLocked()352 private void checkVersionAndBuildLocked() { 353 int version; 354 String buildFingerprint; 355 String currentFingerprint = getBuildFingerprint(); 356 mFirstUpdate = true; 357 mNewUpdate = true; 358 try (BufferedReader reader = new BufferedReader(new FileReader(mVersionFile))) { 359 version = Integer.parseInt(reader.readLine()); 360 buildFingerprint = reader.readLine(); 361 if (buildFingerprint != null) { 362 mFirstUpdate = false; 363 } 364 if (currentFingerprint.equals(buildFingerprint)) { 365 mNewUpdate = false; 366 } 367 } catch (NumberFormatException | IOException e) { 368 version = 0; 369 } 370 371 if (version != mCurrentVersion) { 372 Slog.i(TAG, "Upgrading from version " + version + " to " + mCurrentVersion); 373 if (!mUpdateBreadcrumb.exists()) { 374 try { 375 doUpgradeLocked(version); 376 } catch (Exception e) { 377 Slog.e(TAG, 378 "Failed to upgrade from version " + version + " to " + mCurrentVersion, 379 e); 380 // Fallback to previous version. 381 mCurrentVersion = version; 382 return; 383 } 384 } else { 385 Slog.i(TAG, "Version upgrade breadcrumb found on disk! Continuing version upgrade"); 386 } 387 } 388 389 if (mUpdateBreadcrumb.exists()) { 390 int previousVersion; 391 long token; 392 try (BufferedReader reader = new BufferedReader( 393 new FileReader(mUpdateBreadcrumb))) { 394 token = Long.parseLong(reader.readLine()); 395 previousVersion = Integer.parseInt(reader.readLine()); 396 } catch (NumberFormatException | IOException e) { 397 Slog.e(TAG, "Failed read version upgrade breadcrumb"); 398 throw new RuntimeException(e); 399 } 400 if (mCurrentVersion >= 4) { 401 continueUpgradeLocked(previousVersion, token); 402 } else { 403 Slog.wtf(TAG, "Attempting to upgrade to an unsupported version: " 404 + mCurrentVersion); 405 } 406 } 407 408 if (version != mCurrentVersion || mNewUpdate) { 409 try (BufferedWriter writer = new BufferedWriter(new FileWriter(mVersionFile))) { 410 writer.write(Integer.toString(mCurrentVersion)); 411 writer.write("\n"); 412 writer.write(currentFingerprint); 413 writer.write("\n"); 414 writer.flush(); 415 } catch (IOException e) { 416 Slog.e(TAG, "Failed to write new version"); 417 throw new RuntimeException(e); 418 } 419 } 420 421 if (mUpdateBreadcrumb.exists()) { 422 // Files should be up to date with current version. Clear the version update breadcrumb 423 mUpdateBreadcrumb.delete(); 424 // update mUpgradePerformed after breadcrumb is deleted to indicate a successful upgrade 425 mUpgradePerformed = true; 426 } 427 428 if (mBackupsDir.exists() && !KEEP_BACKUP_DIR) { 429 mUpgradePerformed = true; // updated here to ensure that data is cleaned up 430 deleteDirectory(mBackupsDir); 431 } 432 } 433 getBuildFingerprint()434 private String getBuildFingerprint() { 435 return Build.VERSION.RELEASE + ";" 436 + Build.VERSION.CODENAME + ";" 437 + Build.VERSION.INCREMENTAL; 438 } 439 doUpgradeLocked(int thisVersion)440 private void doUpgradeLocked(int thisVersion) { 441 if (thisVersion < 2) { 442 // Delete all files if we are version 0. This is a pre-release version, 443 // so this is fine. 444 Slog.i(TAG, "Deleting all usage stats files"); 445 for (int i = 0; i < mIntervalDirs.length; i++) { 446 File[] files = mIntervalDirs[i].listFiles(); 447 if (files != null) { 448 for (File f : files) { 449 f.delete(); 450 } 451 } 452 } 453 } else { 454 // Create a dir in backups based on current timestamp 455 final long token = System.currentTimeMillis(); 456 final File backupDir = new File(mBackupsDir, Long.toString(token)); 457 backupDir.mkdirs(); 458 if (!backupDir.exists()) { 459 throw new IllegalStateException( 460 "Failed to create backup directory " + backupDir.getAbsolutePath()); 461 } 462 try { 463 Files.copy(mVersionFile.toPath(), 464 new File(backupDir, mVersionFile.getName()).toPath(), 465 StandardCopyOption.REPLACE_EXISTING); 466 } catch (IOException e) { 467 Slog.e(TAG, "Failed to back up version file : " + mVersionFile.toString()); 468 throw new RuntimeException(e); 469 } 470 471 for (int i = 0; i < mIntervalDirs.length; i++) { 472 final File backupIntervalDir = new File(backupDir, mIntervalDirs[i].getName()); 473 backupIntervalDir.mkdir(); 474 475 if (!backupIntervalDir.exists()) { 476 throw new IllegalStateException( 477 "Failed to create interval backup directory " 478 + backupIntervalDir.getAbsolutePath()); 479 } 480 File[] files = mIntervalDirs[i].listFiles(); 481 if (files != null) { 482 for (int j = 0; j < files.length; j++) { 483 final File backupFile = new File(backupIntervalDir, files[j].getName()); 484 if (DEBUG) { 485 Slog.d(TAG, "Creating versioned (" + Integer.toString(thisVersion) 486 + ") backup of " + files[j].toString() 487 + " stat files for interval " 488 + i + " to " + backupFile.toString()); 489 } 490 491 try { 492 // Backup file should not already exist, but make sure it doesn't 493 Files.move(files[j].toPath(), backupFile.toPath(), 494 StandardCopyOption.REPLACE_EXISTING); 495 } catch (IOException e) { 496 Slog.e(TAG, "Failed to back up file : " + files[j].toString()); 497 throw new RuntimeException(e); 498 } 499 } 500 } 501 } 502 503 // Leave a breadcrumb behind noting that all the usage stats have been moved to a backup 504 BufferedWriter writer = null; 505 try { 506 writer = new BufferedWriter(new FileWriter(mUpdateBreadcrumb)); 507 writer.write(Long.toString(token)); 508 writer.write("\n"); 509 writer.write(Integer.toString(thisVersion)); 510 writer.write("\n"); 511 writer.flush(); 512 } catch (IOException e) { 513 Slog.e(TAG, "Failed to write new version upgrade breadcrumb"); 514 throw new RuntimeException(e); 515 } finally { 516 IoUtils.closeQuietly(writer); 517 } 518 } 519 } 520 continueUpgradeLocked(int version, long token)521 private void continueUpgradeLocked(int version, long token) { 522 if (version <= 3) { 523 Slog.w(TAG, "Reading UsageStats as XML; current database version: " + mCurrentVersion); 524 } 525 final File backupDir = new File(mBackupsDir, Long.toString(token)); 526 527 // Upgrade step logic for the entire usage stats directory, not individual interval dirs. 528 if (version >= 5) { 529 readMappingsLocked(); 530 } 531 532 // Read each file in the backup according to the version and write to the interval 533 // directories in the current versions format 534 for (int i = 0; i < mIntervalDirs.length; i++) { 535 final File backedUpInterval = new File(backupDir, mIntervalDirs[i].getName()); 536 File[] files = backedUpInterval.listFiles(); 537 if (files != null) { 538 for (int j = 0; j < files.length; j++) { 539 if (DEBUG) { 540 Slog.d(TAG, 541 "Upgrading " + files[j].toString() + " to version (" 542 + Integer.toString( 543 mCurrentVersion) + ") for interval " + i); 544 } 545 try { 546 IntervalStats stats = new IntervalStats(); 547 readLocked(new AtomicFile(files[j]), stats, version, mPackagesTokenData, 548 false); 549 // Upgrade to version 5+. 550 // Future version upgrades should add additional logic here to upgrade. 551 if (mCurrentVersion >= 5) { 552 // Create the initial obfuscated packages map. 553 stats.obfuscateData(mPackagesTokenData); 554 } 555 writeLocked(new AtomicFile(new File(mIntervalDirs[i], 556 Long.toString(stats.beginTime))), stats, mCurrentVersion, 557 mPackagesTokenData); 558 } catch (Exception e) { 559 // This method is called on boot, log the exception and move on 560 Slog.e(TAG, "Failed to upgrade backup file : " + files[j].toString()); 561 } 562 } 563 } 564 } 565 566 // Upgrade step logic for the entire usage stats directory, not individual interval dirs. 567 if (mCurrentVersion >= 5) { 568 try { 569 writeMappingsLocked(); 570 } catch (IOException e) { 571 Slog.e(TAG, "Failed to write the tokens mappings file."); 572 } 573 } 574 } 575 576 /** 577 * Returns the token mapped to the package removed or {@code PackagesTokenData.UNASSIGNED_TOKEN} 578 * if not mapped. 579 */ onPackageRemoved(String packageName, long timeRemoved)580 int onPackageRemoved(String packageName, long timeRemoved) { 581 synchronized (mLock) { 582 final int tokenRemoved = mPackagesTokenData.removePackage(packageName, timeRemoved); 583 try { 584 writeMappingsLocked(); 585 } catch (Exception e) { 586 Slog.w(TAG, "Unable to update package mappings on disk after removing token " 587 + tokenRemoved); 588 } 589 return tokenRemoved; 590 } 591 } 592 593 /** 594 * Reads all the usage stats data on disk and rewrites it with any data related to uninstalled 595 * packages omitted. Returns {@code true} on success, {@code false} otherwise. 596 */ pruneUninstalledPackagesData()597 boolean pruneUninstalledPackagesData() { 598 synchronized (mLock) { 599 for (int i = 0; i < mIntervalDirs.length; i++) { 600 final File[] files = mIntervalDirs[i].listFiles(); 601 if (files == null) { 602 continue; 603 } 604 for (int j = 0; j < files.length; j++) { 605 try { 606 final IntervalStats stats = new IntervalStats(); 607 final AtomicFile atomicFile = new AtomicFile(files[j]); 608 if (!readLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData, 609 false)) { 610 continue; // no data was omitted when read so no need to rewrite 611 } 612 // Any data related to packages that have been removed would have failed 613 // the deobfuscation step on read so the IntervalStats object here only 614 // contains data for packages that are currently installed - all we need 615 // to do here is write the data back to disk. 616 writeLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData); 617 } catch (Exception e) { 618 Slog.e(TAG, "Failed to prune data from: " + files[j].toString()); 619 return false; 620 } 621 } 622 } 623 624 try { 625 writeMappingsLocked(); 626 } catch (IOException e) { 627 Slog.e(TAG, "Failed to write package mappings after pruning data."); 628 return false; 629 } 630 return true; 631 } 632 } 633 634 /** 635 * Iterates through all the files on disk and prunes any data that belongs to packages that have 636 * been uninstalled (packages that are not in the given list). 637 * Note: this should only be called once, when there has been a database upgrade. 638 * 639 * @param installedPackages map of installed packages (package_name:package_install_time) 640 */ prunePackagesDataOnUpgrade(HashMap<String, Long> installedPackages)641 void prunePackagesDataOnUpgrade(HashMap<String, Long> installedPackages) { 642 if (ArrayUtils.isEmpty(installedPackages)) { 643 return; 644 } 645 synchronized (mLock) { 646 for (int i = 0; i < mIntervalDirs.length; i++) { 647 final File[] files = mIntervalDirs[i].listFiles(); 648 if (files == null) { 649 continue; 650 } 651 for (int j = 0; j < files.length; j++) { 652 try { 653 final IntervalStats stats = new IntervalStats(); 654 final AtomicFile atomicFile = new AtomicFile(files[j]); 655 readLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData, false); 656 if (!pruneStats(installedPackages, stats)) { 657 continue; // no stats were pruned so no need to rewrite 658 } 659 writeLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData); 660 } catch (Exception e) { 661 Slog.e(TAG, "Failed to prune data from: " + files[j].toString()); 662 } 663 } 664 } 665 } 666 } 667 pruneStats(HashMap<String, Long> installedPackages, IntervalStats stats)668 private boolean pruneStats(HashMap<String, Long> installedPackages, IntervalStats stats) { 669 boolean dataPruned = false; 670 671 // prune old package usage stats 672 for (int i = stats.packageStats.size() - 1; i >= 0; i--) { 673 final UsageStats usageStats = stats.packageStats.valueAt(i); 674 final Long timeInstalled = installedPackages.get(usageStats.mPackageName); 675 if (timeInstalled == null || timeInstalled > usageStats.mEndTimeStamp) { 676 stats.packageStats.removeAt(i); 677 dataPruned = true; 678 } 679 } 680 if (dataPruned) { 681 // ensure old stats don't linger around during the obfuscation step on write 682 stats.packageStatsObfuscated.clear(); 683 } 684 685 // prune old events 686 for (int i = stats.events.size() - 1; i >= 0; i--) { 687 final UsageEvents.Event event = stats.events.get(i); 688 final Long timeInstalled = installedPackages.get(event.mPackage); 689 if (timeInstalled == null || timeInstalled > event.mTimeStamp) { 690 stats.events.remove(i); 691 dataPruned = true; 692 } 693 } 694 695 return dataPruned; 696 } 697 onTimeChanged(long timeDiffMillis)698 public void onTimeChanged(long timeDiffMillis) { 699 synchronized (mLock) { 700 StringBuilder logBuilder = new StringBuilder(); 701 logBuilder.append("Time changed by "); 702 TimeUtils.formatDuration(timeDiffMillis, logBuilder); 703 logBuilder.append("."); 704 705 int filesDeleted = 0; 706 int filesMoved = 0; 707 708 for (LongSparseArray<AtomicFile> files : mSortedStatFiles) { 709 final int fileCount = files.size(); 710 for (int i = 0; i < fileCount; i++) { 711 final AtomicFile file = files.valueAt(i); 712 final long newTime = files.keyAt(i) + timeDiffMillis; 713 if (newTime < 0) { 714 filesDeleted++; 715 file.delete(); 716 } else { 717 try { 718 file.openRead().close(); 719 } catch (IOException e) { 720 // Ignore, this is just to make sure there are no backups. 721 } 722 723 String newName = Long.toString(newTime); 724 if (file.getBaseFile().getName().endsWith(CHECKED_IN_SUFFIX)) { 725 newName = newName + CHECKED_IN_SUFFIX; 726 } 727 728 final File newFile = new File(file.getBaseFile().getParentFile(), newName); 729 filesMoved++; 730 file.getBaseFile().renameTo(newFile); 731 } 732 } 733 files.clear(); 734 } 735 736 logBuilder.append(" files deleted: ").append(filesDeleted); 737 logBuilder.append(" files moved: ").append(filesMoved); 738 Slog.i(TAG, logBuilder.toString()); 739 740 // Now re-index the new files. 741 indexFilesLocked(); 742 } 743 } 744 745 /** 746 * Get the latest stats that exist for this interval type. 747 */ getLatestUsageStats(int intervalType)748 public IntervalStats getLatestUsageStats(int intervalType) { 749 synchronized (mLock) { 750 if (intervalType < 0 || intervalType >= mIntervalDirs.length) { 751 throw new IllegalArgumentException("Bad interval type " + intervalType); 752 } 753 754 final int fileCount = mSortedStatFiles[intervalType].size(); 755 if (fileCount == 0) { 756 return null; 757 } 758 759 try { 760 final AtomicFile f = mSortedStatFiles[intervalType].valueAt(fileCount - 1); 761 IntervalStats stats = new IntervalStats(); 762 readLocked(f, stats, false); 763 return stats; 764 } catch (Exception e) { 765 Slog.e(TAG, "Failed to read usage stats file", e); 766 } 767 } 768 return null; 769 } 770 771 /** 772 * Filter out those stats from the given stats that belong to removed packages. Filtering out 773 * all of the stats at once has an amortized cost for future calls. 774 */ filterStats(IntervalStats stats)775 void filterStats(IntervalStats stats) { 776 synchronized (mLock) { 777 if (mPackagesTokenData.removedPackagesMap.isEmpty()) { 778 return; 779 } 780 final ArrayMap<String, Long> removedPackagesMap = mPackagesTokenData.removedPackagesMap; 781 782 // filter out package usage stats 783 final int removedPackagesSize = removedPackagesMap.size(); 784 for (int i = 0; i < removedPackagesSize; i++) { 785 final String removedPackage = removedPackagesMap.keyAt(i); 786 final UsageStats usageStats = stats.packageStats.get(removedPackage); 787 if (usageStats != null && 788 usageStats.mEndTimeStamp < removedPackagesMap.valueAt(i)) { 789 stats.packageStats.remove(removedPackage); 790 } 791 } 792 793 // filter out events 794 for (int i = stats.events.size() - 1; i >= 0; i--) { 795 final UsageEvents.Event event = stats.events.get(i); 796 final Long timeRemoved = removedPackagesMap.get(event.mPackage); 797 if (timeRemoved != null && timeRemoved > event.mTimeStamp) { 798 stats.events.remove(i); 799 } 800 } 801 } 802 } 803 804 /** 805 * Figures out what to extract from the given IntervalStats object. 806 */ 807 public interface StatCombiner<T> { 808 809 /** 810 * Implementations should extract interesting information from <code>stats</code> and add it 811 * to the <code>accumulatedResult</code> list. 812 * 813 * If the <code>stats</code> object is mutable, <code>mutable</code> will be true, 814 * which means you should make a copy of the data before adding it to the 815 * <code>accumulatedResult</code> list. 816 * 817 * @param stats The {@link IntervalStats} object selected. 818 * @param mutable Whether or not the data inside the stats object is mutable. 819 * @param accumulatedResult The list to which to add extracted data. 820 * @return Whether or not to continue providing new stats to this combiner. If {@code false} 821 * is returned, then combine will no longer be called. 822 */ combine(IntervalStats stats, boolean mutable, List<T> accumulatedResult)823 boolean combine(IntervalStats stats, boolean mutable, List<T> accumulatedResult); 824 } 825 826 /** 827 * Find all {@link IntervalStats} for the given range and interval type. 828 */ 829 @Nullable queryUsageStats(int intervalType, long beginTime, long endTime, StatCombiner<T> combiner, boolean skipEvents)830 public <T> List<T> queryUsageStats(int intervalType, long beginTime, long endTime, 831 StatCombiner<T> combiner, boolean skipEvents) { 832 // mIntervalDirs is final. Accessing its size without holding the lock should be fine. 833 if (intervalType < 0 || intervalType >= mIntervalDirs.length) { 834 throw new IllegalArgumentException("Bad interval type " + intervalType); 835 } 836 837 if (endTime <= beginTime) { 838 if (DEBUG) { 839 Slog.d(TAG, "endTime(" + endTime + ") <= beginTime(" + beginTime + ")"); 840 } 841 return null; 842 } 843 844 synchronized (mLock) { 845 final LongSparseArray<AtomicFile> intervalStats = mSortedStatFiles[intervalType]; 846 847 int endIndex = intervalStats.lastIndexOnOrBefore(endTime); 848 if (endIndex < 0) { 849 // All the stats start after this range ends, so nothing matches. 850 if (DEBUG) { 851 Slog.d(TAG, "No results for this range. All stats start after."); 852 } 853 return null; 854 } 855 856 if (intervalStats.keyAt(endIndex) == endTime) { 857 // The endTime is exclusive, so if we matched exactly take the one before. 858 endIndex--; 859 if (endIndex < 0) { 860 // All the stats start after this range ends, so nothing matches. 861 if (DEBUG) { 862 Slog.d(TAG, "No results for this range. All stats start after."); 863 } 864 return null; 865 } 866 } 867 868 int startIndex = intervalStats.lastIndexOnOrBefore(beginTime); 869 if (startIndex < 0) { 870 // All the stats available have timestamps after beginTime, which means they all 871 // match. 872 startIndex = 0; 873 } 874 875 final ArrayList<T> results = new ArrayList<>(); 876 for (int i = startIndex; i <= endIndex; i++) { 877 final AtomicFile f = intervalStats.valueAt(i); 878 final IntervalStats stats = new IntervalStats(); 879 880 if (DEBUG) { 881 Slog.d(TAG, "Reading stat file " + f.getBaseFile().getAbsolutePath()); 882 } 883 884 try { 885 readLocked(f, stats, skipEvents); 886 if (beginTime < stats.endTime 887 && !combiner.combine(stats, false, results)) { 888 break; 889 } 890 } catch (Exception e) { 891 Slog.e(TAG, "Failed to read usage stats file", e); 892 // We continue so that we return results that are not 893 // corrupt. 894 } 895 } 896 return results; 897 } 898 } 899 900 /** 901 * Find the interval that best matches this range. 902 * 903 * TODO(adamlesinski): Use endTimeStamp in best fit calculation. 904 */ findBestFitBucket(long beginTimeStamp, long endTimeStamp)905 public int findBestFitBucket(long beginTimeStamp, long endTimeStamp) { 906 synchronized (mLock) { 907 int bestBucket = -1; 908 long smallestDiff = Long.MAX_VALUE; 909 for (int i = mSortedStatFiles.length - 1; i >= 0; i--) { 910 final int index = mSortedStatFiles[i].lastIndexOnOrBefore(beginTimeStamp); 911 int size = mSortedStatFiles[i].size(); 912 if (index >= 0 && index < size) { 913 // We have some results here, check if they are better than our current match. 914 long diff = Math.abs(mSortedStatFiles[i].keyAt(index) - beginTimeStamp); 915 if (diff < smallestDiff) { 916 smallestDiff = diff; 917 bestBucket = i; 918 } 919 } 920 } 921 return bestBucket; 922 } 923 } 924 925 /** 926 * Remove any usage stat files that are too old. 927 */ prune(final long currentTimeMillis)928 public void prune(final long currentTimeMillis) { 929 synchronized (mLock) { 930 // prune all files older than 2 years in the yearly directory 931 mCal.setTimeInMillis(currentTimeMillis); 932 mCal.addYears(-2); 933 pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_YEARLY], 934 mCal.getTimeInMillis()); 935 936 // prune all files older than 6 months in the monthly directory 937 mCal.setTimeInMillis(currentTimeMillis); 938 mCal.addMonths(-6); 939 pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_MONTHLY], 940 mCal.getTimeInMillis()); 941 942 // prune all files older than 4 weeks in the weekly directory 943 mCal.setTimeInMillis(currentTimeMillis); 944 mCal.addWeeks(-4); 945 pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_WEEKLY], 946 mCal.getTimeInMillis()); 947 948 // prune all files older than 10 days in the weekly directory 949 mCal.setTimeInMillis(currentTimeMillis); 950 mCal.addDays(-10); 951 pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_DAILY], 952 mCal.getTimeInMillis()); 953 954 // prune chooser counts for all usage stats older than the defined period 955 mCal.setTimeInMillis(currentTimeMillis); 956 mCal.addDays(-SELECTION_LOG_RETENTION_LEN); 957 for (int i = 0; i < mIntervalDirs.length; ++i) { 958 pruneChooserCountsOlderThan(mIntervalDirs[i], mCal.getTimeInMillis()); 959 } 960 961 // We must re-index our file list or we will be trying to read 962 // deleted files. 963 indexFilesLocked(); 964 } 965 } 966 pruneFilesOlderThan(File dir, long expiryTime)967 private static void pruneFilesOlderThan(File dir, long expiryTime) { 968 File[] files = dir.listFiles(); 969 if (files != null) { 970 for (File f : files) { 971 long beginTime; 972 try { 973 beginTime = parseBeginTime(f); 974 } catch (IOException e) { 975 beginTime = 0; 976 } 977 978 if (beginTime < expiryTime) { 979 new AtomicFile(f).delete(); 980 } 981 } 982 } 983 } 984 pruneChooserCountsOlderThan(File dir, long expiryTime)985 private void pruneChooserCountsOlderThan(File dir, long expiryTime) { 986 File[] files = dir.listFiles(); 987 if (files != null) { 988 for (File f : files) { 989 long beginTime; 990 try { 991 beginTime = parseBeginTime(f); 992 } catch (IOException e) { 993 beginTime = 0; 994 } 995 996 if (beginTime < expiryTime) { 997 try { 998 final AtomicFile af = new AtomicFile(f); 999 final IntervalStats stats = new IntervalStats(); 1000 readLocked(af, stats, false); 1001 final int pkgCount = stats.packageStats.size(); 1002 for (int i = 0; i < pkgCount; i++) { 1003 UsageStats pkgStats = stats.packageStats.valueAt(i); 1004 if (pkgStats.mChooserCounts != null) { 1005 pkgStats.mChooserCounts.clear(); 1006 } 1007 } 1008 writeLocked(af, stats); 1009 } catch (Exception e) { 1010 Slog.e(TAG, "Failed to delete chooser counts from usage stats file", e); 1011 } 1012 } 1013 } 1014 } 1015 } 1016 parseBeginTime(AtomicFile file)1017 private static long parseBeginTime(AtomicFile file) throws IOException { 1018 return parseBeginTime(file.getBaseFile()); 1019 } 1020 parseBeginTime(File file)1021 private static long parseBeginTime(File file) throws IOException { 1022 String name = file.getName(); 1023 1024 // Parse out the digits from the front of the file name 1025 for (int i = 0; i < name.length(); i++) { 1026 final char c = name.charAt(i); 1027 if (c < '0' || c > '9') { 1028 // found first char that is not a digit. 1029 name = name.substring(0, i); 1030 break; 1031 } 1032 } 1033 1034 try { 1035 return Long.parseLong(name); 1036 } catch (NumberFormatException e) { 1037 throw new IOException(e); 1038 } 1039 } 1040 writeLocked(AtomicFile file, IntervalStats stats)1041 private void writeLocked(AtomicFile file, IntervalStats stats) 1042 throws IOException, RuntimeException { 1043 if (mCurrentVersion <= 3) { 1044 Slog.wtf(TAG, "Attempting to write UsageStats as XML with version " + mCurrentVersion); 1045 return; 1046 } 1047 writeLocked(file, stats, mCurrentVersion, mPackagesTokenData); 1048 } 1049 writeLocked(AtomicFile file, IntervalStats stats, int version, PackagesTokenData packagesTokenData)1050 private static void writeLocked(AtomicFile file, IntervalStats stats, int version, 1051 PackagesTokenData packagesTokenData) throws IOException, RuntimeException { 1052 FileOutputStream fos = file.startWrite(); 1053 try { 1054 writeLocked(fos, stats, version, packagesTokenData); 1055 file.finishWrite(fos); 1056 fos = null; 1057 } catch (Exception e) { 1058 // Do nothing. Exception has already been handled. 1059 } finally { 1060 // When fos is null (successful write), this will no-op 1061 file.failWrite(fos); 1062 } 1063 } 1064 writeLocked(OutputStream out, IntervalStats stats, int version, PackagesTokenData packagesTokenData)1065 private static void writeLocked(OutputStream out, IntervalStats stats, int version, 1066 PackagesTokenData packagesTokenData) throws Exception { 1067 switch (version) { 1068 case 1: 1069 case 2: 1070 case 3: 1071 Slog.wtf(TAG, "Attempting to write UsageStats as XML with version " + version); 1072 break; 1073 case 4: 1074 try { 1075 UsageStatsProto.write(out, stats); 1076 } catch (Exception e) { 1077 Slog.e(TAG, "Unable to write interval stats to proto.", e); 1078 throw e; 1079 } 1080 break; 1081 case 5: 1082 stats.obfuscateData(packagesTokenData); 1083 try { 1084 UsageStatsProtoV2.write(out, stats); 1085 } catch (Exception e) { 1086 Slog.e(TAG, "Unable to write interval stats to proto.", e); 1087 throw e; 1088 } 1089 break; 1090 default: 1091 throw new RuntimeException( 1092 "Unhandled UsageStatsDatabase version: " + Integer.toString(version) 1093 + " on write."); 1094 } 1095 } 1096 1097 /** 1098 * Note: the data read from the given file will add to the IntervalStats object passed into this 1099 * method. It is up to the caller to ensure that this is the desired behavior - if not, the 1100 * caller should ensure that the data in the reused object is being cleared. 1101 */ readLocked(AtomicFile file, IntervalStats statsOut, boolean skipEvents)1102 private void readLocked(AtomicFile file, IntervalStats statsOut, boolean skipEvents) 1103 throws IOException, RuntimeException { 1104 if (mCurrentVersion <= 3) { 1105 Slog.wtf(TAG, "Reading UsageStats as XML; current database version: " 1106 + mCurrentVersion); 1107 } 1108 readLocked(file, statsOut, mCurrentVersion, mPackagesTokenData, skipEvents); 1109 } 1110 1111 /** 1112 * Returns {@code true} if any stats were omitted while reading, {@code false} otherwise. 1113 * <p/> 1114 * Note: the data read from the given file will add to the IntervalStats object passed into this 1115 * method. It is up to the caller to ensure that this is the desired behavior - if not, the 1116 * caller should ensure that the data in the reused object is being cleared. 1117 */ readLocked(AtomicFile file, IntervalStats statsOut, int version, PackagesTokenData packagesTokenData, boolean skipEvents)1118 private static boolean readLocked(AtomicFile file, IntervalStats statsOut, int version, 1119 PackagesTokenData packagesTokenData, boolean skipEvents) 1120 throws IOException, RuntimeException { 1121 boolean dataOmitted = false; 1122 try { 1123 FileInputStream in = file.openRead(); 1124 try { 1125 statsOut.beginTime = parseBeginTime(file); 1126 dataOmitted = readLocked(in, statsOut, version, packagesTokenData, skipEvents); 1127 statsOut.lastTimeSaved = file.getLastModifiedTime(); 1128 } finally { 1129 try { 1130 in.close(); 1131 } catch (IOException e) { 1132 // Empty 1133 } 1134 } 1135 } catch (FileNotFoundException e) { 1136 Slog.e(TAG, "UsageStatsDatabase", e); 1137 throw e; 1138 } 1139 return dataOmitted; 1140 } 1141 1142 /** 1143 * Returns {@code true} if any stats were omitted while reading, {@code false} otherwise. 1144 * <p/> 1145 * Note: the data read from the given file will add to the IntervalStats object passed into this 1146 * method. It is up to the caller to ensure that this is the desired behavior - if not, the 1147 * caller should ensure that the data in the reused object is being cleared. 1148 */ readLocked(InputStream in, IntervalStats statsOut, int version, PackagesTokenData packagesTokenData, boolean skipEvents)1149 private static boolean readLocked(InputStream in, IntervalStats statsOut, int version, 1150 PackagesTokenData packagesTokenData, boolean skipEvents) throws RuntimeException { 1151 boolean dataOmitted = false; 1152 switch (version) { 1153 case 1: 1154 case 2: 1155 case 3: 1156 Slog.w(TAG, "Reading UsageStats as XML; database version: " + version); 1157 try { 1158 UsageStatsXml.read(in, statsOut); 1159 } catch (Exception e) { 1160 Slog.e(TAG, "Unable to read interval stats from XML", e); 1161 } 1162 break; 1163 case 4: 1164 try { 1165 UsageStatsProto.read(in, statsOut); 1166 } catch (Exception e) { 1167 Slog.e(TAG, "Unable to read interval stats from proto.", e); 1168 } 1169 break; 1170 case 5: 1171 try { 1172 UsageStatsProtoV2.read(in, statsOut, skipEvents); 1173 } catch (Exception e) { 1174 Slog.e(TAG, "Unable to read interval stats from proto.", e); 1175 } 1176 dataOmitted = statsOut.deobfuscateData(packagesTokenData); 1177 break; 1178 default: 1179 throw new RuntimeException( 1180 "Unhandled UsageStatsDatabase version: " + Integer.toString(version) 1181 + " on read."); 1182 } 1183 return dataOmitted; 1184 } 1185 1186 /** 1187 * Reads the obfuscated data file from disk containing the tokens to packages mappings and 1188 * rebuilds the packages to tokens mappings based on that data. 1189 */ readMappingsLocked()1190 public void readMappingsLocked() { 1191 if (!mPackageMappingsFile.exists()) { 1192 return; // package mappings file is missing - recreate mappings on next write. 1193 } 1194 1195 try (FileInputStream in = new AtomicFile(mPackageMappingsFile).openRead()) { 1196 UsageStatsProtoV2.readObfuscatedData(in, mPackagesTokenData); 1197 } catch (Exception e) { 1198 Slog.e(TAG, "Failed to read the obfuscated packages mapping file.", e); 1199 return; 1200 } 1201 1202 final SparseArray<ArrayList<String>> tokensToPackagesMap = 1203 mPackagesTokenData.tokensToPackagesMap; 1204 final int tokensToPackagesMapSize = tokensToPackagesMap.size(); 1205 for (int i = 0; i < tokensToPackagesMapSize; i++) { 1206 final int packageToken = tokensToPackagesMap.keyAt(i); 1207 final ArrayList<String> tokensMap = tokensToPackagesMap.valueAt(i); 1208 final ArrayMap<String, Integer> packageStringsMap = new ArrayMap<>(); 1209 final int tokensMapSize = tokensMap.size(); 1210 // package name will always be at index 0 but its token should not be 0 1211 packageStringsMap.put(tokensMap.get(0), packageToken); 1212 for (int j = 1; j < tokensMapSize; j++) { 1213 packageStringsMap.put(tokensMap.get(j), j); 1214 } 1215 mPackagesTokenData.packagesToTokensMap.put(tokensMap.get(0), packageStringsMap); 1216 } 1217 } 1218 writeMappingsLocked()1219 void writeMappingsLocked() throws IOException { 1220 final AtomicFile file = new AtomicFile(mPackageMappingsFile); 1221 FileOutputStream fos = file.startWrite(); 1222 try { 1223 UsageStatsProtoV2.writeObfuscatedData(fos, mPackagesTokenData); 1224 file.finishWrite(fos); 1225 fos = null; 1226 } catch (Exception e) { 1227 Slog.e(TAG, "Unable to write obfuscated data to proto.", e); 1228 } finally { 1229 file.failWrite(fos); 1230 } 1231 } 1232 obfuscateCurrentStats(IntervalStats[] currentStats)1233 void obfuscateCurrentStats(IntervalStats[] currentStats) { 1234 synchronized (mLock) { 1235 if (mCurrentVersion < 5) { 1236 return; 1237 } 1238 for (int i = 0; i < currentStats.length; i++) { 1239 final IntervalStats stats = currentStats[i]; 1240 stats.obfuscateData(mPackagesTokenData); 1241 } 1242 } 1243 } 1244 1245 /** 1246 * Update the stats in the database. They may not be written to disk immediately. 1247 */ putUsageStats(int intervalType, IntervalStats stats)1248 public void putUsageStats(int intervalType, IntervalStats stats) throws IOException { 1249 if (stats == null) return; 1250 synchronized (mLock) { 1251 if (intervalType < 0 || intervalType >= mIntervalDirs.length) { 1252 throw new IllegalArgumentException("Bad interval type " + intervalType); 1253 } 1254 1255 AtomicFile f = mSortedStatFiles[intervalType].get(stats.beginTime); 1256 if (f == null) { 1257 f = new AtomicFile(new File(mIntervalDirs[intervalType], 1258 Long.toString(stats.beginTime))); 1259 mSortedStatFiles[intervalType].put(stats.beginTime, f); 1260 } 1261 1262 writeLocked(f, stats); 1263 stats.lastTimeSaved = f.getLastModifiedTime(); 1264 } 1265 } 1266 1267 /* Backup/Restore Code */ getBackupPayload(String key)1268 byte[] getBackupPayload(String key) { 1269 return getBackupPayload(key, BACKUP_VERSION); 1270 } 1271 1272 /** 1273 * @hide 1274 */ 1275 @VisibleForTesting getBackupPayload(String key, int version)1276 public byte[] getBackupPayload(String key, int version) { 1277 if (version >= 1 && version <= 3) { 1278 Slog.wtf(TAG, "Attempting to backup UsageStats as XML with version " + version); 1279 return null; 1280 } 1281 if (version < 1 || version > BACKUP_VERSION) { 1282 Slog.wtf(TAG, "Attempting to backup UsageStats with an unknown version: " + version); 1283 return null; 1284 } 1285 synchronized (mLock) { 1286 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 1287 if (KEY_USAGE_STATS.equals(key)) { 1288 prune(System.currentTimeMillis()); 1289 DataOutputStream out = new DataOutputStream(baos); 1290 try { 1291 out.writeInt(version); 1292 1293 out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY].size()); 1294 1295 for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY].size(); 1296 i++) { 1297 writeIntervalStatsToStream(out, 1298 mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY].valueAt(i), 1299 version); 1300 } 1301 1302 out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_WEEKLY].size()); 1303 for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_WEEKLY].size(); 1304 i++) { 1305 writeIntervalStatsToStream(out, 1306 mSortedStatFiles[UsageStatsManager.INTERVAL_WEEKLY].valueAt(i), 1307 version); 1308 } 1309 1310 out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_MONTHLY].size()); 1311 for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_MONTHLY].size(); 1312 i++) { 1313 writeIntervalStatsToStream(out, 1314 mSortedStatFiles[UsageStatsManager.INTERVAL_MONTHLY].valueAt(i), 1315 version); 1316 } 1317 1318 out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_YEARLY].size()); 1319 for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_YEARLY].size(); 1320 i++) { 1321 writeIntervalStatsToStream(out, 1322 mSortedStatFiles[UsageStatsManager.INTERVAL_YEARLY].valueAt(i), 1323 version); 1324 } 1325 if (DEBUG) Slog.i(TAG, "Written " + baos.size() + " bytes of data"); 1326 } catch (IOException ioe) { 1327 Slog.d(TAG, "Failed to write data to output stream", ioe); 1328 baos.reset(); 1329 } 1330 } 1331 return baos.toByteArray(); 1332 } 1333 } 1334 1335 /** 1336 * Updates the set of packages given to only include those that have been used within the 1337 * given timeframe (as defined by {@link UsageStats#getLastTimePackageUsed()}). 1338 */ calculatePackagesUsedWithinTimeframe( IntervalStats stats, Set<String> packagesList, long timeframeMs)1339 private void calculatePackagesUsedWithinTimeframe( 1340 IntervalStats stats, Set<String> packagesList, long timeframeMs) { 1341 for (UsageStats stat : stats.packageStats.values()) { 1342 if (stat.getLastTimePackageUsed() > timeframeMs) { 1343 packagesList.add(stat.mPackageName); 1344 } 1345 } 1346 } 1347 1348 /** 1349 * @hide 1350 */ 1351 @VisibleForTesting applyRestoredPayload(String key, byte[] payload)1352 public @NonNull Set<String> applyRestoredPayload(String key, byte[] payload) { 1353 synchronized (mLock) { 1354 if (KEY_USAGE_STATS.equals(key)) { 1355 // Read stats files for the current device configs 1356 IntervalStats dailyConfigSource = 1357 getLatestUsageStats(UsageStatsManager.INTERVAL_DAILY); 1358 IntervalStats weeklyConfigSource = 1359 getLatestUsageStats(UsageStatsManager.INTERVAL_WEEKLY); 1360 IntervalStats monthlyConfigSource = 1361 getLatestUsageStats(UsageStatsManager.INTERVAL_MONTHLY); 1362 IntervalStats yearlyConfigSource = 1363 getLatestUsageStats(UsageStatsManager.INTERVAL_YEARLY); 1364 1365 final Set<String> packagesRestored = new ArraySet<>(); 1366 try { 1367 DataInputStream in = new DataInputStream(new ByteArrayInputStream(payload)); 1368 int backupDataVersion = in.readInt(); 1369 1370 // Can't handle this backup set 1371 if (backupDataVersion < 1 || backupDataVersion > BACKUP_VERSION) { 1372 return packagesRestored; 1373 } 1374 1375 // Delete all stats files 1376 // Do this after reading version and before actually restoring 1377 for (int i = 0; i < mIntervalDirs.length; i++) { 1378 deleteDirectoryContents(mIntervalDirs[i]); 1379 } 1380 1381 // 90 days before today in epoch 1382 final long timeframe = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(90); 1383 int fileCount = in.readInt(); 1384 for (int i = 0; i < fileCount; i++) { 1385 IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in), 1386 backupDataVersion); 1387 calculatePackagesUsedWithinTimeframe(stats, packagesRestored, timeframe); 1388 packagesRestored.addAll(stats.packageStats.keySet()); 1389 stats = mergeStats(stats, dailyConfigSource); 1390 putUsageStats(UsageStatsManager.INTERVAL_DAILY, stats); 1391 } 1392 1393 fileCount = in.readInt(); 1394 for (int i = 0; i < fileCount; i++) { 1395 IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in), 1396 backupDataVersion); 1397 calculatePackagesUsedWithinTimeframe(stats, packagesRestored, timeframe); 1398 stats = mergeStats(stats, weeklyConfigSource); 1399 putUsageStats(UsageStatsManager.INTERVAL_WEEKLY, stats); 1400 } 1401 1402 fileCount = in.readInt(); 1403 for (int i = 0; i < fileCount; i++) { 1404 IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in), 1405 backupDataVersion); 1406 calculatePackagesUsedWithinTimeframe(stats, packagesRestored, timeframe); 1407 stats = mergeStats(stats, monthlyConfigSource); 1408 putUsageStats(UsageStatsManager.INTERVAL_MONTHLY, stats); 1409 } 1410 1411 fileCount = in.readInt(); 1412 for (int i = 0; i < fileCount; i++) { 1413 IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in), 1414 backupDataVersion); 1415 calculatePackagesUsedWithinTimeframe(stats, packagesRestored, timeframe); 1416 stats = mergeStats(stats, yearlyConfigSource); 1417 putUsageStats(UsageStatsManager.INTERVAL_YEARLY, stats); 1418 } 1419 if (DEBUG) Slog.i(TAG, "Completed Restoring UsageStats"); 1420 } catch (IOException ioe) { 1421 Slog.d(TAG, "Failed to read data from input stream", ioe); 1422 } finally { 1423 indexFilesLocked(); 1424 } 1425 return packagesRestored; 1426 } 1427 return Collections.EMPTY_SET; 1428 } 1429 } 1430 1431 /** 1432 * Get the Configuration Statistics from the current device statistics and merge them 1433 * with the backed up usage statistics. 1434 */ mergeStats(IntervalStats beingRestored, IntervalStats onDevice)1435 private IntervalStats mergeStats(IntervalStats beingRestored, IntervalStats onDevice) { 1436 if (onDevice == null) return beingRestored; 1437 if (beingRestored == null) return null; 1438 beingRestored.activeConfiguration = onDevice.activeConfiguration; 1439 beingRestored.configurations.putAll(onDevice.configurations); 1440 beingRestored.events.clear(); 1441 beingRestored.events.merge(onDevice.events); 1442 return beingRestored; 1443 } 1444 writeIntervalStatsToStream(DataOutputStream out, AtomicFile statsFile, int version)1445 private void writeIntervalStatsToStream(DataOutputStream out, AtomicFile statsFile, int version) 1446 throws IOException { 1447 IntervalStats stats = new IntervalStats(); 1448 try { 1449 readLocked(statsFile, stats, false); 1450 } catch (IOException e) { 1451 Slog.e(TAG, "Failed to read usage stats file", e); 1452 out.writeInt(0); 1453 return; 1454 } 1455 sanitizeIntervalStatsForBackup(stats); 1456 byte[] data = serializeIntervalStats(stats, version); 1457 out.writeInt(data.length); 1458 out.write(data); 1459 } 1460 getIntervalStatsBytes(DataInputStream in)1461 private static byte[] getIntervalStatsBytes(DataInputStream in) throws IOException { 1462 int length = in.readInt(); 1463 byte[] buffer = new byte[length]; 1464 in.read(buffer, 0, length); 1465 return buffer; 1466 } 1467 sanitizeIntervalStatsForBackup(IntervalStats stats)1468 private static void sanitizeIntervalStatsForBackup(IntervalStats stats) { 1469 if (stats == null) return; 1470 stats.activeConfiguration = null; 1471 stats.configurations.clear(); 1472 stats.events.clear(); 1473 } 1474 serializeIntervalStats(IntervalStats stats, int version)1475 private byte[] serializeIntervalStats(IntervalStats stats, int version) { 1476 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 1477 DataOutputStream out = new DataOutputStream(baos); 1478 try { 1479 out.writeLong(stats.beginTime); 1480 writeLocked(out, stats, version, mPackagesTokenData); 1481 } catch (Exception ioe) { 1482 Slog.d(TAG, "Serializing IntervalStats Failed", ioe); 1483 baos.reset(); 1484 } 1485 return baos.toByteArray(); 1486 } 1487 deserializeIntervalStats(byte[] data, int version)1488 private IntervalStats deserializeIntervalStats(byte[] data, int version) { 1489 ByteArrayInputStream bais = new ByteArrayInputStream(data); 1490 DataInputStream in = new DataInputStream(bais); 1491 IntervalStats stats = new IntervalStats(); 1492 try { 1493 stats.beginTime = in.readLong(); 1494 readLocked(in, stats, version, mPackagesTokenData, false); 1495 } catch (Exception e) { 1496 Slog.d(TAG, "DeSerializing IntervalStats Failed", e); 1497 stats = null; 1498 } 1499 return stats; 1500 } 1501 deleteDirectoryContents(File directory)1502 private static void deleteDirectoryContents(File directory) { 1503 File[] files = directory.listFiles(); 1504 for (File file : files) { 1505 deleteDirectory(file); 1506 } 1507 } 1508 deleteDirectory(File directory)1509 private static void deleteDirectory(File directory) { 1510 File[] files = directory.listFiles(); 1511 if (files != null) { 1512 for (File file : files) { 1513 if (!file.isDirectory()) { 1514 file.delete(); 1515 } else { 1516 deleteDirectory(file); 1517 } 1518 } 1519 } 1520 directory.delete(); 1521 } 1522 1523 /** 1524 * Prints the obfuscated package mappings and a summary of the database files. 1525 * @param pw the print writer to print to 1526 */ dump(IndentingPrintWriter pw, boolean compact)1527 public void dump(IndentingPrintWriter pw, boolean compact) { 1528 synchronized (mLock) { 1529 pw.println(); 1530 pw.println("UsageStatsDatabase:"); 1531 pw.increaseIndent(); 1532 dumpMappings(pw); 1533 pw.decreaseIndent(); 1534 pw.println("Database Summary:"); 1535 pw.increaseIndent(); 1536 for (int i = 0; i < mSortedStatFiles.length; i++) { 1537 final LongSparseArray<AtomicFile> files = mSortedStatFiles[i]; 1538 final int size = files.size(); 1539 pw.print(UserUsageStatsService.intervalToString(i)); 1540 pw.print(" stats files: "); 1541 pw.print(size); 1542 pw.println(", sorted list of files:"); 1543 pw.increaseIndent(); 1544 for (int f = 0; f < size; f++) { 1545 final long fileName = files.keyAt(f); 1546 if (compact) { 1547 pw.print(UserUsageStatsService.formatDateTime(fileName, false)); 1548 } else { 1549 pw.printPair(Long.toString(fileName), 1550 UserUsageStatsService.formatDateTime(fileName, true)); 1551 } 1552 pw.println(); 1553 } 1554 pw.decreaseIndent(); 1555 } 1556 pw.decreaseIndent(); 1557 } 1558 } 1559 dumpMappings(IndentingPrintWriter pw)1560 void dumpMappings(IndentingPrintWriter pw) { 1561 synchronized (mLock) { 1562 pw.println("Obfuscated Packages Mappings:"); 1563 pw.increaseIndent(); 1564 pw.println("Counter: " + mPackagesTokenData.counter); 1565 pw.println("Tokens Map Size: " + mPackagesTokenData.tokensToPackagesMap.size()); 1566 if (!mPackagesTokenData.removedPackageTokens.isEmpty()) { 1567 pw.println("Removed Package Tokens: " 1568 + Arrays.toString(mPackagesTokenData.removedPackageTokens.toArray())); 1569 } 1570 for (int i = 0; i < mPackagesTokenData.tokensToPackagesMap.size(); i++) { 1571 final int packageToken = mPackagesTokenData.tokensToPackagesMap.keyAt(i); 1572 final String packageStrings = String.join(", ", 1573 mPackagesTokenData.tokensToPackagesMap.valueAt(i)); 1574 pw.println("Token " + packageToken + ": [" + packageStrings + "]"); 1575 } 1576 pw.println(); 1577 pw.decreaseIndent(); 1578 } 1579 } 1580 deleteDataFor(String pkg)1581 void deleteDataFor(String pkg) { 1582 // reuse the existing prune method to delete data for the specified package. 1583 // we'll use the current timestamp so that all events before now get pruned. 1584 prunePackagesDataOnUpgrade( 1585 new HashMap<>(Collections.singletonMap(pkg, SystemClock.elapsedRealtime()))); 1586 } 1587 readIntervalStatsForFile(int interval, long fileName)1588 IntervalStats readIntervalStatsForFile(int interval, long fileName) { 1589 synchronized (mLock) { 1590 final IntervalStats stats = new IntervalStats(); 1591 try { 1592 readLocked(mSortedStatFiles[interval].get(fileName, null), stats, false); 1593 return stats; 1594 } catch (Exception e) { 1595 return null; 1596 } 1597 } 1598 } 1599 } 1600