1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.providers.media; 18 19 import static com.android.providers.media.DatabaseHelper.DATA_MEDIA_XATTR_DIRECTORY_PATH; 20 import static com.android.providers.media.DatabaseHelper.EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX; 21 import static com.android.providers.media.DatabaseHelper.EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX; 22 import static com.android.providers.media.DatabaseHelper.INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX; 23 import static com.android.providers.media.DatabaseHelper.INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX; 24 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__BACKUP_MISSING; 25 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__FUSE_DAEMON_TIMEOUT; 26 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__GET_BACKUP_DATA_FAILURE; 27 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__OTHER_ERROR; 28 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__SUCCESS; 29 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__VOLUME_NOT_ATTACHED; 30 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY; 31 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__INTERNAL; 32 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC; 33 import static com.android.providers.media.util.Logging.TAG; 34 import static com.android.providers.media.flags.Flags.enableStableUrisForExternalPrimaryVolume; 35 import static com.android.providers.media.flags.Flags.enableStableUrisForPublicVolume; 36 37 import android.content.ContentValues; 38 import android.database.Cursor; 39 import android.database.sqlite.SQLiteDatabase; 40 import android.os.Build; 41 import android.os.CancellationSignal; 42 import android.os.Environment; 43 import android.os.ParcelFileDescriptor; 44 import android.os.SystemClock; 45 import android.os.SystemProperties; 46 import android.os.UserHandle; 47 import android.provider.MediaStore; 48 import android.system.ErrnoException; 49 import android.system.Os; 50 import android.system.OsConstants; 51 import android.util.Log; 52 import android.util.Pair; 53 54 import androidx.annotation.NonNull; 55 56 import com.android.providers.media.dao.FileRow; 57 import com.android.providers.media.fuse.FuseDaemon; 58 import com.android.providers.media.stableuris.dao.BackupIdRow; 59 import com.android.providers.media.util.StringUtils; 60 61 import com.google.common.base.Strings; 62 63 import java.io.File; 64 import java.io.FileNotFoundException; 65 import java.io.IOException; 66 import java.util.ArrayList; 67 import java.util.Arrays; 68 import java.util.HashSet; 69 import java.util.List; 70 import java.util.Locale; 71 import java.util.Map; 72 import java.util.Optional; 73 import java.util.Set; 74 import java.util.concurrent.ConcurrentHashMap; 75 import java.util.concurrent.TimeoutException; 76 import java.util.concurrent.atomic.AtomicInteger; 77 import java.util.concurrent.locks.ReentrantLock; 78 import java.util.stream.Collectors; 79 80 /** 81 * To ensure that the ids of MediaStore database uris are stable and reliable. 82 */ 83 public class DatabaseBackupAndRecovery { 84 85 private static final String LOWER_FS_RECOVERY_DIRECTORY_PATH = 86 "/data/media/" + UserHandle.myUserId() + "/.transforms/recovery"; 87 88 /** 89 * Path for storing owner id to owner package identifier relation and vice versa. 90 * Lower file system path is used as upper file system does not support xattrs. 91 */ 92 private static final String OWNER_RELATION_LOWER_FS_BACKUP_PATH = 93 "/data/media/" + UserHandle.myUserId() + "/.transforms/recovery/leveldb-ownership"; 94 95 private static final String INTERNAL_VOLUME_LOWER_FS_BACKUP_PATH = 96 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-internal"; 97 98 private static final String EXTERNAL_PRIMARY_VOLUME_LOWER_FS_BACKUP_PATH = 99 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-external_primary"; 100 101 /** 102 * Every LevelDB table name starts with this prefix. 103 */ 104 private static final String LEVEL_DB_PREFIX = "leveldb-"; 105 106 private static final String OWNERSHIP_TABLE_NAME = LEVEL_DB_PREFIX + "ownership"; 107 108 /** 109 * Frequency at which next value of owner id is backed up in the external storage. 110 */ 111 private static final int NEXT_OWNER_ID_BACKUP_FREQUENCY = 50; 112 113 /** 114 * Start value used for next owner id. 115 */ 116 private static final int NEXT_OWNER_ID_DEFAULT_VALUE = 0; 117 118 /** 119 * Key name of xattr used to set next owner id on ownership DB. 120 */ 121 private static final String NEXT_OWNER_ID_XATTR_KEY = "user.nextownerid"; 122 123 /** 124 * Key name of xattr used to store last modified generation number. 125 */ 126 private static final String LAST_BACKEDUP_GENERATION_XATTR_KEY = "user.lastbackedgeneration"; 127 128 /** 129 * Key name of xattr used to store a public volume recovery flag. 130 */ 131 private static final String PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY 132 = "user.publicvolumerecoveryflag"; 133 134 /** 135 * External primary storage root path for given user. 136 */ 137 private static final String EXTERNAL_PRIMARY_ROOT_PATH = 138 "/storage/emulated/" + UserHandle.myUserId(); 139 140 /** 141 * Array of columns backed up in external storage. 142 */ 143 private static final String[] QUERY_COLUMNS = new String[]{ 144 MediaStore.Files.FileColumns._ID, 145 MediaStore.Files.FileColumns.DATA, 146 MediaStore.Files.FileColumns.IS_FAVORITE, 147 MediaStore.Files.FileColumns.IS_PENDING, 148 MediaStore.Files.FileColumns.IS_TRASHED, 149 MediaStore.Files.FileColumns.MEDIA_TYPE, 150 MediaStore.Files.FileColumns._USER_ID, 151 MediaStore.Files.FileColumns.DATE_EXPIRES, 152 MediaStore.Files.FileColumns.OWNER_PACKAGE_NAME, 153 MediaStore.Files.FileColumns.GENERATION_MODIFIED, 154 MediaStore.Files.FileColumns.VOLUME_NAME 155 }; 156 157 /** 158 * Wait time of 20 seconds in millis. 159 */ 160 private static final long WAIT_TIME_20_SECONDS_IN_MILLIS = 20000; 161 162 /** 163 * Number of records to read from leveldb in a JNI call. 164 */ 165 protected static final int LEVEL_DB_READ_LIMIT = 100; 166 167 168 /** 169 * Re-entrant lock to ensure sequential update to leveldb by background threads. 170 */ 171 private final ReentrantLock mLevelDbUpdateLock = new ReentrantLock(); 172 173 /** 174 * Stores cached value of next owner id. This helps in improving performance by backing up next 175 * row id less frequently in the external storage. 176 */ 177 private AtomicInteger mNextOwnerId; 178 179 /** 180 * Stores value of next backup of owner id. 181 */ 182 private AtomicInteger mNextOwnerIdBackup; 183 private final ConfigStore mConfigStore; 184 private final VolumeCache mVolumeCache; 185 private Set<String> mSetupCompleteVolumes = ConcurrentHashMap.newKeySet(); 186 187 // Flag only used to enable/disable feature for testing 188 private boolean mIsStableUriEnabledForInternal = false; 189 190 // Flag only used to enable/disable feature for testing 191 private boolean mIsStableUriEnabledForExternal = false; 192 193 // Flag only used to enable/disable feature for testing 194 private boolean mIsStableUrisEnabledForPublic = false; 195 196 private static Map<String, String> sOwnerIdRelationMap; 197 198 public static final String STABLE_URI_INTERNAL_PROPERTY = 199 "persist.sys.fuse.backup.internal_db_backup"; 200 201 private static boolean STABLE_URI_INTERNAL_PROPERTY_VALUE = true; 202 203 public static final String STABLE_URI_EXTERNAL_PROPERTY = 204 "persist.sys.fuse.backup.external_volume_backup"; 205 206 private static boolean STABLE_URI_EXTERNAL_PROPERTY_VALUE = true; 207 208 public static final String STABLE_URI_PUBLIC_PROPERTY = 209 "persist.sys.fuse.backup.public_db_backup"; 210 211 private static boolean STABLE_URI_PUBLIC_PROPERTY_VALUE = true; 212 DatabaseBackupAndRecovery(ConfigStore configStore, VolumeCache volumeCache)213 DatabaseBackupAndRecovery(ConfigStore configStore, VolumeCache volumeCache) { 214 mConfigStore = configStore; 215 mVolumeCache = volumeCache; 216 } 217 218 /** 219 * Returns true if migration and recovery code flow for stable uris is enabled for given volume. 220 */ isStableUrisEnabled(String volumeName)221 boolean isStableUrisEnabled(String volumeName) { 222 // Check if flags are enabled for test for internal volume 223 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName) 224 && mIsStableUriEnabledForInternal) { 225 return true; 226 } 227 // Check if flags are enabled for test for external primary volume 228 if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 229 && mIsStableUriEnabledForExternal) { 230 return true; 231 } 232 233 // Check if flags are enabled for test for public volume 234 if (!MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName) 235 && !MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 236 && mIsStableUrisEnabledForPublic) { 237 return true; 238 } 239 240 // Feature is disabled for below S due to vold mount issues. 241 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { 242 return false; 243 } 244 245 switch (volumeName) { 246 case MediaStore.VOLUME_INTERNAL: 247 return mIsStableUriEnabledForInternal 248 || mConfigStore.isStableUrisForInternalVolumeEnabled() 249 || SystemProperties.getBoolean(STABLE_URI_INTERNAL_PROPERTY, 250 /* defaultValue */ STABLE_URI_INTERNAL_PROPERTY_VALUE); 251 case MediaStore.VOLUME_EXTERNAL_PRIMARY: 252 return mIsStableUriEnabledForExternal 253 || mConfigStore.isStableUrisForExternalVolumeEnabled() 254 || enableStableUrisForExternalPrimaryVolume() 255 || SystemProperties.getBoolean(STABLE_URI_EXTERNAL_PROPERTY, 256 /* defaultValue */ STABLE_URI_EXTERNAL_PROPERTY_VALUE); 257 default: 258 // public volume 259 return mIsStableUrisEnabledForPublic 260 || mConfigStore.isStableUrisForPublicVolumeEnabled() 261 || enableStableUrisForPublicVolume() 262 || SystemProperties.getBoolean(STABLE_URI_PUBLIC_PROPERTY, 263 /* defaultValue */ STABLE_URI_PUBLIC_PROPERTY_VALUE); 264 } 265 } 266 267 /** 268 * On device boot, leveldb setup is done as part of attachVolume call for primary external. 269 * Also, on device config flag change, we check if flag is enabled, if yes, we proceed to 270 * setup(no-op if connection already exists). So, we setup backup and recovery for internal 271 * volume on Media mount signal of EXTERNAL_PRIMARY. 272 */ setupVolumeDbBackupAndRecovery(String volumeName)273 synchronized void setupVolumeDbBackupAndRecovery(String volumeName) { 274 // Since internal volume does not have any fuse daemon thread, leveldb instance 275 // for internal volume is created by fuse daemon thread of EXTERNAL_PRIMARY. 276 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 277 // Set backup only for external primary for now. 278 return; 279 } 280 // Do not create leveldb instance if stable uris is not enabled for internal volume. 281 if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)) { 282 // Return if we are not supporting backup for internal volume 283 return; 284 } 285 286 if (mSetupCompleteVolumes.contains(volumeName)) { 287 // Return if setup is already done 288 return; 289 } 290 291 final long startTime = SystemClock.elapsedRealtime(); 292 int vol = MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 293 ? MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY 294 : MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC; 295 try { 296 if (!new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).exists()) { 297 new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).mkdirs(); 298 Log.v(TAG, "Created recovery directory:" + LOWER_FS_RECOVERY_DIRECTORY_PATH); 299 } 300 FuseDaemon fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait(new File( 301 DatabaseBackupAndRecovery.EXTERNAL_PRIMARY_ROOT_PATH)); 302 Log.d(TAG, "Received db backup Fuse Daemon for: " + volumeName); 303 if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) && ( 304 isStableUrisEnabled(MediaStore.VOLUME_INTERNAL) || isStableUrisEnabled( 305 MediaStore.VOLUME_EXTERNAL_PRIMARY))) { 306 // Setup internal and external volumes 307 MediaProviderStatsLog.write( 308 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 309 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__ATTEMPTED, vol); 310 fuseDaemonExternalPrimary.setupVolumeDbBackup(); 311 mSetupCompleteVolumes.add(volumeName); 312 MediaProviderStatsLog.write( 313 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 314 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__SUCCESS, vol); 315 } else if (isStableUrisEnabled(volumeName)) { 316 // Setup public volume 317 FuseDaemon fuseDaemonPublicVolume = getFuseDaemonForPath( 318 getFuseFilePathFromVolumeName(volumeName)); 319 MediaProviderStatsLog.write( 320 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 321 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__ATTEMPTED, vol); 322 fuseDaemonPublicVolume.setupPublicVolumeDbBackup(volumeName); 323 mSetupCompleteVolumes.add(volumeName); 324 MediaProviderStatsLog.write( 325 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 326 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__SUCCESS, vol); 327 } else { 328 return; 329 } 330 } catch (Exception e) { 331 MediaProviderStatsLog.write( 332 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 333 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__FAILURE, vol); 334 Log.e(TAG, "Failure in setting up backup and recovery for volume: " + volumeName, e); 335 return; 336 } finally { 337 Log.i(TAG, "Backup and recovery setup time taken in milliseconds:" + ( 338 SystemClock.elapsedRealtime() - startTime)); 339 } 340 Log.i(TAG, "Successfully set up backup and recovery for volume: " + volumeName); 341 } 342 343 /** 344 * Backs up databases to external storage to ensure stable URIs. 345 */ backupDatabases(DatabaseHelper internalDatabaseHelper, DatabaseHelper externalDatabaseHelper, CancellationSignal signal)346 void backupDatabases(DatabaseHelper internalDatabaseHelper, 347 DatabaseHelper externalDatabaseHelper, CancellationSignal signal) { 348 mLevelDbUpdateLock.lock(); 349 try { 350 setupVolumeDbBackupAndRecovery(MediaStore.VOLUME_EXTERNAL_PRIMARY); 351 Log.i(TAG, "Triggering database backup"); 352 backupInternalDatabase(internalDatabaseHelper, signal); 353 backupExternalDatabase(externalDatabaseHelper, 354 MediaStore.VOLUME_EXTERNAL_PRIMARY, signal); 355 356 for (MediaVolume mediaVolume : mVolumeCache.getExternalVolumes()) { 357 if (mediaVolume.isPublicVolume()) { 358 setupVolumeDbBackupAndRecovery(mediaVolume.getName()); 359 backupExternalDatabase(externalDatabaseHelper, mediaVolume.getName(), signal); 360 } 361 } 362 } catch (Exception e) { 363 Log.e(TAG, "Failure in backing up databases", e); 364 } finally { 365 mLevelDbUpdateLock.unlock(); 366 } 367 } 368 readDataFromBackup(String volumeName, String filePath)369 Optional<BackupIdRow> readDataFromBackup(String volumeName, String filePath) { 370 if (!isStableUrisEnabled(volumeName)) { 371 return Optional.empty(); 372 } 373 374 try { 375 final String data = getFuseDaemonForPath(getFuseFilePathFromVolumeName(volumeName)) 376 .readBackedUpData(filePath); 377 if (data == null || data.isEmpty()) { 378 Log.w(TAG, "No backup found for path: " + filePath); 379 return Optional.empty(); 380 } 381 382 return Optional.of(BackupIdRow.deserialize(data)); 383 } catch (Exception e) { 384 Log.e(TAG, "Failure in getting backed up data for filePath: " + filePath, e); 385 return Optional.empty(); 386 } 387 } 388 backupInternalDatabase(DatabaseHelper internalDbHelper, CancellationSignal signal)389 private void backupInternalDatabase(DatabaseHelper internalDbHelper, 390 CancellationSignal signal) { 391 if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL) 392 || internalDbHelper.isDatabaseRecovering()) { 393 return; 394 } 395 396 if (!mSetupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 397 Log.w(TAG, 398 "Setup is not present for backup of internal and external primary volume."); 399 return; 400 } 401 402 FuseDaemon fuseDaemon; 403 try { 404 fuseDaemon = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH); 405 } catch (FileNotFoundException e) { 406 Log.e(TAG, 407 "Fuse Daemon not found for primary external storage, skipping backing up of " 408 + "internal database.", 409 e); 410 return; 411 } 412 413 internalDbHelper.runWithTransaction((db) -> { 414 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, null, null, null, null, null, 415 null, signal)) { 416 while (c.moveToNext()) { 417 backupDataValues(fuseDaemon, c); 418 } 419 Log.d(TAG, String.format(Locale.ROOT, 420 "Backed up %d rows of internal database to external storage on idle " 421 + "maintenance.", 422 c.getCount())); 423 } catch (Exception e) { 424 Log.e(TAG, "Failure in backing up internal database to external storage.", e); 425 } 426 return null; 427 }); 428 } 429 backupExternalDatabase(DatabaseHelper externalDbHelper, String volumeName, CancellationSignal signal)430 private void backupExternalDatabase(DatabaseHelper externalDbHelper, 431 String volumeName, CancellationSignal signal) { 432 if (!isStableUrisEnabled(volumeName) 433 || externalDbHelper.isDatabaseRecovering()) { 434 return; 435 } 436 437 if (!mSetupCompleteVolumes.contains(volumeName)) { 438 return; 439 } 440 441 FuseDaemon fuseDaemonExternalPrimary; 442 try { 443 fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait( 444 new File(EXTERNAL_PRIMARY_ROOT_PATH)); 445 } catch (Exception e) { 446 Log.e(TAG, 447 "Error occurred while retrieving the Fuse Daemon for the external primary, " 448 + "skipping backing up of " 449 + volumeName, e); 450 return; 451 } 452 FuseDaemon fuseDaemonPublicVolume; 453 if (!isInternalOrExternalPrimary(volumeName)) { 454 try { 455 fuseDaemonPublicVolume = getFuseDaemonForFileWithWait(new File( 456 getFuseFilePathFromVolumeName(volumeName))); 457 } catch (Exception e) { 458 Log.e(TAG, 459 "Error occurred while retrieving the Fuse Daemon for " 460 + getFuseFilePathFromVolumeName(volumeName) 461 + ", skipping backing up of " + volumeName, 462 e); 463 return; 464 } 465 } else { 466 fuseDaemonPublicVolume = null; 467 } 468 469 final String backupPath = 470 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX + volumeName; 471 long lastBackedGenerationNumber = getLastBackedGenerationNumber(backupPath); 472 473 final String generationClause = MediaStore.Files.FileColumns.GENERATION_MODIFIED + " >= " 474 + lastBackedGenerationNumber; 475 final String volumeClause = MediaStore.Files.FileColumns.VOLUME_NAME + " = '" 476 + volumeName + "'"; 477 final String selectionClause = generationClause + " AND " + volumeClause; 478 479 externalDbHelper.runWithTransaction((db) -> { 480 long maxGeneration = lastBackedGenerationNumber; 481 Log.d(TAG, "Started to back up " + volumeName 482 + ", maxGeneration:" + maxGeneration); 483 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, selectionClause, null, null, 484 null, MediaStore.MediaColumns.GENERATION_MODIFIED + " ASC", null, signal)) { 485 while (c.moveToNext()) { 486 if (signal != null && signal.isCanceled()) { 487 Log.i(TAG, "Received a cancellation signal during the DB " 488 + "backup process"); 489 break; 490 } 491 if (isInternalOrExternalPrimary(volumeName)) { 492 backupDataValues(fuseDaemonExternalPrimary, c); 493 } else { 494 // public volume 495 backupDataValues(fuseDaemonExternalPrimary, fuseDaemonPublicVolume, c); 496 } 497 maxGeneration = Math.max(maxGeneration, c.getLong(9)); 498 } 499 setXattr(backupPath, LAST_BACKEDUP_GENERATION_XATTR_KEY, 500 String.valueOf(maxGeneration - 1)); 501 Log.d(TAG, String.format(Locale.ROOT, 502 "Backed up %d rows of " + volumeName + " to external storage on idle " 503 + "maintenance.", 504 c.getCount())); 505 } catch (Exception e) { 506 Log.e(TAG, "Failure in backing up " + volumeName + " to external storage.", e); 507 return null; 508 } 509 return null; 510 }); 511 } 512 backupDataValues(FuseDaemon fuseDaemon, Cursor c)513 private void backupDataValues(FuseDaemon fuseDaemon, Cursor c) throws IOException { 514 backupDataValues(fuseDaemon, null, c); 515 } 516 backupDataValues(FuseDaemon externalPrimaryFuseDaemon, FuseDaemon publicVolumeFuseDaemon, Cursor c)517 private void backupDataValues(FuseDaemon externalPrimaryFuseDaemon, 518 FuseDaemon publicVolumeFuseDaemon, Cursor c) throws IOException { 519 final long id = c.getLong(0); 520 final String data = c.getString(1); 521 final boolean isFavorite = c.getInt(2) != 0; 522 final boolean isPending = c.getInt(3) != 0; 523 final boolean isTrashed = c.getInt(4) != 0; 524 final int mediaType = c.getInt(5); 525 final int userId = c.getInt(6); 526 final String dateExpires = c.getString(7); 527 final String ownerPackageName = c.getString(8); 528 final String volumeName = c.getString(10); 529 BackupIdRow backupIdRow = createBackupIdRow(externalPrimaryFuseDaemon, id, mediaType, 530 isFavorite, isPending, isTrashed, userId, dateExpires, ownerPackageName); 531 if (isInternalOrExternalPrimary(volumeName)) { 532 externalPrimaryFuseDaemon.backupVolumeDbData(volumeName, data, 533 BackupIdRow.serialize(backupIdRow)); 534 } else { 535 // public volume 536 publicVolumeFuseDaemon.backupVolumeDbData(volumeName, data, 537 BackupIdRow.serialize(backupIdRow)); 538 } 539 } 540 deleteBackupForVolume(String volumeName)541 void deleteBackupForVolume(String volumeName) { 542 File dbFilePath = new File( 543 String.format(Locale.ROOT, "%s/%s.db", LOWER_FS_RECOVERY_DIRECTORY_PATH, 544 LEVEL_DB_PREFIX + volumeName)); 545 if (dbFilePath.exists()) { 546 dbFilePath.delete(); 547 } 548 } 549 readBackedUpFilePaths(String volumeName, String lastReadValue, int limit)550 String[] readBackedUpFilePaths(String volumeName, String lastReadValue, int limit) 551 throws IOException, UnsupportedOperationException { 552 if (!isStableUrisEnabled(volumeName)) { 553 throw new UnsupportedOperationException("Stable Uris are not enabled"); 554 } 555 556 return getFuseDaemonForPath(getFuseFilePathFromVolumeName(volumeName)) 557 .readBackedUpFilePaths(volumeName, lastReadValue, limit); 558 } 559 updateNextRowIdXattr(DatabaseHelper helper, long id)560 void updateNextRowIdXattr(DatabaseHelper helper, long id) { 561 if (helper.isInternal()) { 562 updateNextRowIdForInternal(helper, id); 563 return; 564 } 565 566 if (!helper.isNextRowIdBackupEnabled()) { 567 return; 568 } 569 570 Optional<Long> nextRowIdBackupOptional = helper.getNextRowId(); 571 if (!nextRowIdBackupOptional.isPresent()) { 572 throw new RuntimeException( 573 String.format(Locale.ROOT, "Cannot find next row id xattr for %s.", 574 helper.getDatabaseName())); 575 } 576 577 if (id >= nextRowIdBackupOptional.get()) { 578 helper.backupNextRowId(id); 579 } 580 } 581 getLastBackedGenerationNumber(String backupPath)582 private long getLastBackedGenerationNumber(String backupPath) { 583 // Read last backed up generation number 584 Optional<Long> lastBackedUpGenNum = getXattrOfLongValue( 585 backupPath, LAST_BACKEDUP_GENERATION_XATTR_KEY); 586 long lastBackedGenerationNumber = lastBackedUpGenNum.isPresent() 587 ? lastBackedUpGenNum.get() : 0; 588 if (lastBackedGenerationNumber > 0) { 589 Log.i(TAG, "Last backed up generation number for " + backupPath + " is " 590 + lastBackedGenerationNumber); 591 } 592 return lastBackedGenerationNumber; 593 } 594 595 @NonNull getFuseDaemonForPath(@onNull String path)596 private FuseDaemon getFuseDaemonForPath(@NonNull String path) 597 throws FileNotFoundException { 598 return MediaProvider.getFuseDaemonForFile(new File(path), mVolumeCache); 599 } 600 updateNextRowIdAndSetDirty(@onNull DatabaseHelper helper, @NonNull FileRow oldRow, @NonNull FileRow newRow)601 void updateNextRowIdAndSetDirty(@NonNull DatabaseHelper helper, 602 @NonNull FileRow oldRow, @NonNull FileRow newRow) { 603 updateNextRowIdXattr(helper, newRow.getId()); 604 markBackupAsDirty(helper, oldRow); 605 } 606 607 /** 608 * Backs up DB data in external storage to recover in case of DB rollback. 609 */ backupVolumeDbData(DatabaseHelper databaseHelper, FileRow insertedRow)610 void backupVolumeDbData(DatabaseHelper databaseHelper, FileRow insertedRow) { 611 if (!isBackupUpdateAllowed(databaseHelper, insertedRow.getVolumeName())) { 612 return; 613 } 614 615 mLevelDbUpdateLock.lock(); 616 try { 617 FuseDaemon fuseDaemonExternalPrimary = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH); 618 final BackupIdRow value = createBackupIdRow(fuseDaemonExternalPrimary, insertedRow); 619 if (isInternalOrExternalPrimary(insertedRow.getVolumeName())) { 620 fuseDaemonExternalPrimary.backupVolumeDbData(insertedRow.getVolumeName(), 621 insertedRow.getPath(), BackupIdRow.serialize(value)); 622 } else { 623 // public volume 624 final FuseDaemon fuseDaemonPublicVolume = getFuseDaemonForPath( 625 getFuseFilePathFromVolumeName(insertedRow.getVolumeName())); 626 fuseDaemonPublicVolume.backupVolumeDbData(insertedRow.getVolumeName(), 627 insertedRow.getPath(), BackupIdRow.serialize(value)); 628 } 629 } catch (Exception e) { 630 Log.e(TAG, "Failure in backing up data to external storage", e); 631 } finally { 632 mLevelDbUpdateLock.unlock(); 633 } 634 } 635 createBackupIdRow(FuseDaemon fuseDaemon, FileRow insertedRow)636 private BackupIdRow createBackupIdRow(FuseDaemon fuseDaemon, FileRow insertedRow) 637 throws IOException { 638 return createBackupIdRow(fuseDaemon, insertedRow.getId(), insertedRow.getMediaType(), 639 insertedRow.isFavorite(), insertedRow.isPending(), insertedRow.isTrashed(), 640 insertedRow.getUserId(), insertedRow.getDateExpires(), 641 insertedRow.getOwnerPackageName()); 642 } 643 createBackupIdRow(FuseDaemon fuseDaemon, long id, int mediaType, boolean isFavorite, boolean isPending, boolean isTrashed, int userId, String dateExpires, String ownerPackageName)644 private BackupIdRow createBackupIdRow(FuseDaemon fuseDaemon, long id, int mediaType, 645 boolean isFavorite, 646 boolean isPending, boolean isTrashed, int userId, String dateExpires, 647 String ownerPackageName) throws IOException { 648 BackupIdRow.Builder builder = BackupIdRow.newBuilder(id); 649 builder.setMediaType(mediaType); 650 builder.setIsFavorite(isFavorite ? 1 : 0); 651 builder.setIsPending(isPending ? 1 : 0); 652 builder.setIsTrashed(isTrashed ? 1 : 0); 653 builder.setUserId(userId); 654 builder.setDateExpires(dateExpires); 655 // We set owner package id instead of owner package name in the backup. When an 656 // application is uninstalled, all media rows corresponding to it will be orphaned and 657 // would have owner package name as null. This should not change if application is 658 // installed again. Therefore, we are storing owner id instead of owner package name. On 659 // package uninstallation, we delete the owner id relation from the backup. All rows 660 // recovered for orphaned owner ids will have package name as null. Since we also need to 661 // support cloned apps, we are storing a combination of owner package name and user id to 662 // uniquely identify a package. 663 builder.setOwnerPackagedId(getOwnerPackageId(fuseDaemon, ownerPackageName, userId)); 664 return builder.setIsDirty(false).build(); 665 } 666 667 getOwnerPackageId(FuseDaemon fuseDaemon, String ownerPackageName, int userId)668 private synchronized int getOwnerPackageId(FuseDaemon fuseDaemon, 669 String ownerPackageName, int userId) throws IOException { 670 // In synchronized block to avoid use of same owner id for multiple owner package relations 671 if (Strings.isNullOrEmpty(ownerPackageName) || ownerPackageName.equalsIgnoreCase("null")) { 672 // We store -1 in the backup if owner package name is null. 673 return -1; 674 } 675 676 // Create identifier of format "owner_pkg_name::user_id". Tightly coupling owner package 677 // name and user id helps in handling app cloning scenarios. 678 String ownerPackageIdentifier = createOwnerPackageIdentifier(ownerPackageName, userId); 679 // Read any existing entry for given owner package name and user id 680 String ownerId = fuseDaemon.readFromOwnershipBackup(ownerPackageIdentifier); 681 if (!ownerId.trim().isEmpty()) { 682 // Use existing owner id if found and is positive 683 int val = Integer.parseInt(ownerId); 684 if (val >= 0) { 685 return val; 686 } 687 } 688 689 int nextOwnerId = getAndIncrementNextOwnerId(); 690 fuseDaemon.createOwnerIdRelation(String.valueOf(nextOwnerId), ownerPackageIdentifier); 691 Log.v(TAG, "Created relation b/w " + nextOwnerId + " and " + ownerPackageIdentifier); 692 return nextOwnerId; 693 } 694 createOwnerPackageIdentifier(String ownerPackageName, int userId)695 private String createOwnerPackageIdentifier(String ownerPackageName, int userId) { 696 return ownerPackageName.trim().concat("::").concat(String.valueOf(userId)); 697 } 698 getPackageNameAndUserId(String ownerPackageIdentifier)699 private Pair<String, Integer> getPackageNameAndUserId(String ownerPackageIdentifier) { 700 if (ownerPackageIdentifier.trim().isEmpty()) { 701 return Pair.create(null, null); 702 } 703 704 String[] arr = ownerPackageIdentifier.trim().split("::"); 705 return Pair.create(arr[0], Integer.valueOf(arr[1])); 706 } 707 getAndIncrementNextOwnerId()708 private int getAndIncrementNextOwnerId() { 709 if (mNextOwnerId == null) { 710 Optional<Integer> nextOwnerIdOptional = getXattrOfIntegerValue( 711 OWNER_RELATION_LOWER_FS_BACKUP_PATH, 712 NEXT_OWNER_ID_XATTR_KEY); 713 mNextOwnerId = nextOwnerIdOptional.map(AtomicInteger::new).orElseGet( 714 () -> new AtomicInteger(NEXT_OWNER_ID_DEFAULT_VALUE)); 715 mNextOwnerIdBackup = new AtomicInteger(mNextOwnerId.get()); 716 } 717 if (mNextOwnerId.get() >= mNextOwnerIdBackup.get()) { 718 int nextBackup = mNextOwnerId.get() + NEXT_OWNER_ID_BACKUP_FREQUENCY; 719 updateNextOwnerId(nextBackup); 720 mNextOwnerIdBackup = new AtomicInteger(nextBackup); 721 } 722 int returnValue = mNextOwnerId.get(); 723 mNextOwnerId.set(returnValue + 1); 724 return returnValue; 725 } 726 updateNextOwnerId(int val)727 private void updateNextOwnerId(int val) { 728 setXattr(OWNER_RELATION_LOWER_FS_BACKUP_PATH, NEXT_OWNER_ID_XATTR_KEY, String.valueOf(val)); 729 Log.d(TAG, "Updated next owner id to: " + val); 730 } 731 removeOwnerIdToPackageRelation(String packageName, int userId)732 void removeOwnerIdToPackageRelation(String packageName, int userId) { 733 if (Strings.isNullOrEmpty(packageName) || packageName.equalsIgnoreCase("null") 734 || !isStableUrisEnabled(MediaStore.VOLUME_EXTERNAL_PRIMARY) 735 || !new File(OWNER_RELATION_LOWER_FS_BACKUP_PATH).exists() 736 || !mSetupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 737 return; 738 } 739 740 try { 741 FuseDaemon fuseDaemon = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH); 742 String ownerPackageIdentifier = createOwnerPackageIdentifier(packageName, userId); 743 String ownerId = fuseDaemon.readFromOwnershipBackup(ownerPackageIdentifier); 744 745 fuseDaemon.removeOwnerIdRelation(ownerId, ownerPackageIdentifier); 746 } catch (Exception e) { 747 Log.e(TAG, "Failure in removing owner id to package relation", e); 748 } 749 } 750 751 /** 752 * Deletes backed up data(needed for recovery) from external storage. 753 */ deleteFromDbBackup(DatabaseHelper databaseHelper, FileRow deletedRow)754 void deleteFromDbBackup(DatabaseHelper databaseHelper, FileRow deletedRow) { 755 if (!isBackupUpdateAllowed(databaseHelper, deletedRow.getVolumeName())) { 756 return; 757 } 758 759 String deletedFilePath = deletedRow.getPath(); 760 if (deletedFilePath == null) { 761 return; 762 } 763 764 mLevelDbUpdateLock.lock(); 765 try { 766 getFuseDaemonForPath(getFuseFilePathFromVolumeName(deletedRow.getVolumeName())) 767 .deleteDbBackup(deletedFilePath); 768 } catch (IOException e) { 769 Log.w(TAG, "Failure in deleting backup data for key: " + deletedFilePath, e); 770 } finally { 771 mLevelDbUpdateLock.unlock(); 772 } 773 } 774 isBackupUpdateAllowed(DatabaseHelper databaseHelper, String volumeName)775 private boolean isBackupUpdateAllowed(DatabaseHelper databaseHelper, String volumeName) { 776 // Backup only if stable uris is enabled, db is not recovering and backup setup is complete. 777 return isStableUrisEnabled(volumeName) && !databaseHelper.isDatabaseRecovering() 778 && mSetupCompleteVolumes.contains(volumeName); 779 } 780 isInternalOrExternalPrimary(String volumeName)781 private boolean isInternalOrExternalPrimary(String volumeName) { 782 if (Strings.isNullOrEmpty(volumeName)) { 783 // This should never happen 784 Log.e(TAG, "Volume name is " + volumeName + ", treating it as non public volume"); 785 return true; 786 } 787 return MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName) 788 || MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName); 789 } 790 updateNextRowIdForInternal(DatabaseHelper helper, long id)791 private void updateNextRowIdForInternal(DatabaseHelper helper, long id) { 792 if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)) { 793 return; 794 } 795 796 Optional<Long> nextRowIdBackupOptional = helper.getNextRowId(); 797 798 if (!nextRowIdBackupOptional.isPresent()) { 799 return; 800 } 801 802 if (id >= nextRowIdBackupOptional.get()) { 803 helper.backupNextRowId(id); 804 } 805 } 806 markBackupAsDirty(DatabaseHelper databaseHelper, FileRow updatedRow)807 private void markBackupAsDirty(DatabaseHelper databaseHelper, FileRow updatedRow) { 808 if (!isBackupUpdateAllowed(databaseHelper, updatedRow.getVolumeName())) { 809 return; 810 } 811 812 final String updatedFilePath = updatedRow.getPath(); 813 try { 814 getFuseDaemonForPath(getFuseFilePathFromVolumeName(updatedRow.getVolumeName())) 815 .backupVolumeDbData( 816 updatedRow.getVolumeName(), 817 updatedFilePath, 818 BackupIdRow.serialize(BackupIdRow.newBuilder(updatedRow.getId()) 819 .setIsDirty(true).build())); 820 } catch (IOException e) { 821 Log.e(TAG, "Failure in marking data as dirty to external storage for path:" 822 + updatedFilePath, e); 823 } 824 } 825 826 /** 827 * Reads value corresponding to given key from xattr on given path. 828 */ getXattr(String path, String key)829 static Optional<String> getXattr(String path, String key) { 830 try { 831 return Optional.of(Arrays.toString(Os.getxattr(path, key))); 832 } catch (Exception e) { 833 Log.w(TAG, String.format(Locale.ROOT, 834 "Exception encountered while reading xattr:%s from path:%s.", key, path)); 835 return Optional.empty(); 836 } 837 } 838 839 /** 840 * Reads long value corresponding to given key from xattr on given path. 841 */ getXattrOfLongValue(String path, String key)842 private static Optional<Long> getXattrOfLongValue(String path, String key) { 843 try { 844 return Optional.of(Long.parseLong(new String(Os.getxattr(path, key)))); 845 } catch (Exception e) { 846 Log.w(TAG, String.format(Locale.ROOT, 847 "Exception encountered while reading xattr:%s from path:%s.", key, path)); 848 return Optional.empty(); 849 } 850 } 851 852 /** 853 * Reads integer value corresponding to given key from xattr on given path. 854 */ getXattrOfIntegerValue(String path, String key)855 static Optional<Integer> getXattrOfIntegerValue(String path, String key) { 856 try { 857 return Optional.of(Integer.parseInt(new String(Os.getxattr(path, key)))); 858 } catch (Exception e) { 859 Log.w(TAG, String.format(Locale.ROOT, 860 "Exception encountered while reading xattr:%s from path:%s.", key, path)); 861 return Optional.empty(); 862 } 863 } 864 865 /** 866 * Sets key and value as xattr on given path. 867 */ setXattr(String path, String key, String value)868 static boolean setXattr(String path, String key, String value) { 869 try (ParcelFileDescriptor pfd = ParcelFileDescriptor.open(new File(path), 870 ParcelFileDescriptor.MODE_READ_ONLY)) { 871 // Map id value to xattr key 872 Os.setxattr(path, key, value.getBytes(), 0); 873 Os.fsync(pfd.getFileDescriptor()); 874 Log.d(TAG, String.format("xattr set to %s for key:%s on path: %s.", value, key, path)); 875 return true; 876 } catch (Exception e) { 877 Log.e(TAG, String.format(Locale.ROOT, "Failed to set xattr:%s to %s for path: %s.", key, 878 value, path), e); 879 return false; 880 } 881 } 882 883 /** 884 * Deletes xattr with given key on given path. Becomes a no-op when xattr is not present. 885 */ removeXattr(String path, String key)886 static boolean removeXattr(String path, String key) { 887 try (ParcelFileDescriptor pfd = ParcelFileDescriptor.open(new File(path), 888 ParcelFileDescriptor.MODE_READ_ONLY)) { 889 Os.removexattr(path, key); 890 Os.fsync(pfd.getFileDescriptor()); 891 Log.d(TAG, String.format("xattr key:%s removed on path: %s.", key, path)); 892 return true; 893 } catch (Exception e) { 894 if (e instanceof ErrnoException) { 895 ErrnoException exception = (ErrnoException) e; 896 if (exception.errno == OsConstants.ENODATA) { 897 Log.w(TAG, String.format(Locale.ROOT, 898 "xattr:%s is not removed as it is not found on path: %s.", key, path)); 899 return true; 900 } 901 } 902 903 Log.e(TAG, String.format(Locale.ROOT, "Failed to remove xattr:%s for path: %s.", key, 904 path), e); 905 return false; 906 } 907 } 908 909 /** 910 * Lists xattrs of given path. 911 */ listXattr(String path)912 static List<String> listXattr(String path) { 913 try { 914 return Arrays.asList(Os.listxattr(path)); 915 } catch (Exception e) { 916 Log.e(TAG, "Exception in reading xattrs on path: " + path, e); 917 return new ArrayList<>(); 918 } 919 } 920 insertDataInDatabase(SQLiteDatabase db, BackupIdRow row, String filePath, String volumeName)921 private boolean insertDataInDatabase(SQLiteDatabase db, BackupIdRow row, String filePath, 922 String volumeName) { 923 final ContentValues values = createValuesFromFileRow(row, filePath, volumeName); 924 return db.insertWithOnConflict("files", null, values, 925 SQLiteDatabase.CONFLICT_REPLACE) != -1; 926 } 927 createValuesFromFileRow(BackupIdRow row, String filePath, String volumeName)928 private ContentValues createValuesFromFileRow(BackupIdRow row, String filePath, 929 String volumeName) { 930 ContentValues values = new ContentValues(); 931 values.put(MediaStore.Files.FileColumns._ID, row.getId()); 932 values.put(MediaStore.Files.FileColumns.IS_FAVORITE, row.getIsFavorite()); 933 values.put(MediaStore.Files.FileColumns.IS_PENDING, row.getIsPending()); 934 values.put(MediaStore.Files.FileColumns.IS_TRASHED, row.getIsTrashed()); 935 values.put(MediaStore.Files.FileColumns.DATA, filePath); 936 values.put(MediaStore.Files.FileColumns.VOLUME_NAME, volumeName); 937 values.put(MediaStore.Files.FileColumns._USER_ID, row.getUserId()); 938 values.put(MediaStore.Files.FileColumns.MEDIA_TYPE, row.getMediaType()); 939 if (!StringUtils.isNullOrEmpty(row.getDateExpires())) { 940 values.put(MediaStore.Files.FileColumns.DATE_EXPIRES, 941 Long.valueOf(row.getDateExpires())); 942 } 943 if (row.getOwnerPackageId() >= 0) { 944 Pair<String, Integer> ownerPackageNameAndUidPair = getOwnerPackageNameAndUidPair( 945 row.getOwnerPackageId()); 946 if (ownerPackageNameAndUidPair.first != null) { 947 values.put(MediaStore.Files.FileColumns.OWNER_PACKAGE_NAME, 948 ownerPackageNameAndUidPair.first); 949 } 950 if (ownerPackageNameAndUidPair.second != null) { 951 values.put(MediaStore.Files.FileColumns._USER_ID, 952 ownerPackageNameAndUidPair.second); 953 } 954 } 955 956 return values; 957 } 958 getOwnerPackageNameAndUidPair(int ownerPackageId)959 private Pair<String, Integer> getOwnerPackageNameAndUidPair(int ownerPackageId) { 960 if (sOwnerIdRelationMap == null) { 961 try { 962 sOwnerIdRelationMap = readOwnerIdRelationsFromLevelDb(); 963 Log.v(TAG, "Cached owner id map"); 964 } catch (IOException e) { 965 Log.e(TAG, "Failure in reading owner details for owner id:" + ownerPackageId, e); 966 return Pair.create(null, null); 967 } 968 } 969 970 if (sOwnerIdRelationMap.containsKey(String.valueOf(ownerPackageId))) { 971 return getPackageNameAndUserId(sOwnerIdRelationMap.get(String.valueOf(ownerPackageId))); 972 } 973 974 return Pair.create(null, null); 975 } 976 readOwnerIdRelationsFromLevelDb()977 private Map<String, String> readOwnerIdRelationsFromLevelDb() throws IOException { 978 return getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH).readOwnerIdRelations(); 979 } 980 readOwnerPackageName(String ownerId)981 String readOwnerPackageName(String ownerId) throws IOException { 982 Map<String, String> ownerIdRelationMap = readOwnerIdRelationsFromLevelDb(); 983 if (ownerIdRelationMap.containsKey(String.valueOf(ownerId))) { 984 return getPackageNameAndUserId(ownerIdRelationMap.get(ownerId)).first; 985 } 986 987 return null; 988 } 989 markPublicVolumesRecovery()990 void markPublicVolumesRecovery() { 991 try { 992 File recoveryDir = new File(LOWER_FS_RECOVERY_DIRECTORY_PATH); 993 for (File levelDbFile : recoveryDir.listFiles()) { 994 if (!(LEVEL_DB_PREFIX + MediaStore.VOLUME_EXTERNAL_PRIMARY) 995 .equalsIgnoreCase(levelDbFile.getName()) 996 && !(LEVEL_DB_PREFIX + MediaStore.VOLUME_INTERNAL) 997 .equalsIgnoreCase(levelDbFile.getName()) 998 && !(OWNERSHIP_TABLE_NAME 999 .equalsIgnoreCase(levelDbFile.getName()))) { 1000 setXattr(levelDbFile.getAbsolutePath(), PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY, 1001 String.valueOf(true)); 1002 } 1003 } 1004 } catch (Exception e) { 1005 Log.e(TAG, "Exception while marking public volumes for recovery", e); 1006 } 1007 } 1008 isPublicVolumeMarkedForRecovery(String volumeName)1009 boolean isPublicVolumeMarkedForRecovery(String volumeName) { 1010 String filePath = LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX 1011 + volumeName.toLowerCase(Locale.ROOT); 1012 Optional<String> flag = getXattr(filePath, PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY); 1013 Log.d(TAG, "Public volume is " + (flag.isPresent() ? "" : "not ") 1014 + "marked for recovery, volume: " + volumeName); 1015 return flag.isPresent(); 1016 } 1017 removePublicVolumeRecoveryFlag(String volumeName)1018 void removePublicVolumeRecoveryFlag(String volumeName) { 1019 String filePath = LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX 1020 + volumeName.toLowerCase(Locale.ROOT); 1021 removeXattr(filePath, PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY); 1022 } 1023 recoverData(SQLiteDatabase db, String volumeName)1024 void recoverData(SQLiteDatabase db, String volumeName) throws Exception { 1025 long rowsRecovered = 0, dirtyRowsCount = 0, insertionFailuresCount = 0, 1026 totalLevelDbRows = 0; 1027 final long startTime = SystemClock.elapsedRealtime(); 1028 try { 1029 final String fuseFilePath = getFuseFilePathFromVolumeName(volumeName); 1030 // Wait for external primary to be attached as we use same thread for internal volume. 1031 // Maximum wait for 20s 1032 getFuseDaemonForFileWithWait(new File(fuseFilePath)); 1033 if (!isBackupPresent(volumeName)) { 1034 throw new FileNotFoundException("Backup file not found for " + volumeName); 1035 } 1036 1037 Log.d(TAG, "Backup is present for " + volumeName); 1038 try { 1039 waitForVolumeToBeAttached(MediaStore.VOLUME_EXTERNAL_PRIMARY); 1040 } catch (Exception e) { 1041 throw new IllegalStateException( 1042 "Volume not attached in given time. Cannot recover data.", e); 1043 } 1044 1045 String[] backedUpFilePaths; 1046 String lastReadValue = ""; 1047 while (true) { 1048 backedUpFilePaths = readBackedUpFilePaths(volumeName, lastReadValue, 1049 LEVEL_DB_READ_LIMIT); 1050 if (backedUpFilePaths == null || backedUpFilePaths.length == 0) { 1051 break; 1052 } 1053 totalLevelDbRows += backedUpFilePaths.length; 1054 1055 // Reset cached owner id relation map 1056 sOwnerIdRelationMap = null; 1057 for (String filePath : backedUpFilePaths) { 1058 Optional<BackupIdRow> fileRow = readDataFromBackup(volumeName, filePath); 1059 if (fileRow.isPresent()) { 1060 if (fileRow.get().getIsDirty()) { 1061 dirtyRowsCount++; 1062 continue; 1063 } 1064 1065 if (insertDataInDatabase(db, fileRow.get(), filePath, volumeName)) { 1066 rowsRecovered++; 1067 } else { 1068 insertionFailuresCount++; 1069 } 1070 } 1071 } 1072 1073 // Read less rows than expected 1074 if (backedUpFilePaths.length < LEVEL_DB_READ_LIMIT) { 1075 break; 1076 } 1077 lastReadValue = backedUpFilePaths[backedUpFilePaths.length - 1]; 1078 } 1079 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1080 publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount, 1081 totalLevelDbRows, insertionFailuresCount, 1082 MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__SUCCESS); 1083 Log.i(TAG, String.format(Locale.ROOT, "%d rows recovered for volume: %s." 1084 + " Total rows in levelDB: %d.", rowsRecovered, volumeName, 1085 totalLevelDbRows)); 1086 Log.i(TAG, String.format(Locale.ROOT, "Recovery time: %d ms", recoveryTime)); 1087 } catch (TimeoutException e) { 1088 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1089 publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount, 1090 totalLevelDbRows, insertionFailuresCount, 1091 MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__FUSE_DAEMON_TIMEOUT); 1092 throw e; 1093 } catch (FileNotFoundException e) { 1094 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1095 publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount, 1096 totalLevelDbRows, insertionFailuresCount, 1097 MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__BACKUP_MISSING); 1098 throw e; 1099 } catch (IOException e) { 1100 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1101 publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount, 1102 totalLevelDbRows, insertionFailuresCount, 1103 MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__GET_BACKUP_DATA_FAILURE); 1104 throw e; 1105 } catch (IllegalStateException e) { 1106 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1107 publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount, 1108 totalLevelDbRows, insertionFailuresCount, 1109 MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__VOLUME_NOT_ATTACHED); 1110 throw e; 1111 } catch (Exception e) { 1112 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1113 publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount, 1114 totalLevelDbRows, insertionFailuresCount, 1115 MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__OTHER_ERROR); 1116 throw e; 1117 } 1118 } 1119 resetLastBackedUpGenerationNumber(String volumeName)1120 void resetLastBackedUpGenerationNumber(String volumeName) { 1121 // Resetting generation number 1122 setXattr(LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX + volumeName, 1123 LAST_BACKEDUP_GENERATION_XATTR_KEY, String.valueOf(0)); 1124 Log.v(TAG, "Leveldb Last backed generation number reset done to 0 for " + volumeName); 1125 } 1126 isBackupPresent(String volumeName)1127 boolean isBackupPresent(String volumeName) { 1128 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 1129 return new File(INTERNAL_VOLUME_LOWER_FS_BACKUP_PATH).exists(); 1130 } else if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)) { 1131 return new File(EXTERNAL_PRIMARY_VOLUME_LOWER_FS_BACKUP_PATH).exists(); 1132 } else if (!Strings.isNullOrEmpty(volumeName)) { 1133 return new File(LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-" + volumeName) 1134 .exists(); 1135 } 1136 1137 return false; 1138 } 1139 waitForVolumeToBeAttached(String volumeName)1140 void waitForVolumeToBeAttached(String volumeName) throws TimeoutException { 1141 long time = 0; 1142 // Wait of 10 seconds 1143 long waitTimeInMilliseconds = 10000; 1144 // Poll every 100 milliseconds 1145 long pollTime = 100; 1146 while (time <= waitTimeInMilliseconds) { 1147 if (mSetupCompleteVolumes.contains(volumeName)) { 1148 Log.i(TAG, "Found " + volumeName + " volume attached."); 1149 return; 1150 } 1151 1152 SystemClock.sleep(pollTime); 1153 time += pollTime; 1154 } 1155 throw new TimeoutException("Timed out waiting for " + volumeName + " setup"); 1156 } 1157 getFuseDaemonForFileWithWait(File fuseFilePath)1158 FuseDaemon getFuseDaemonForFileWithWait(File fuseFilePath) 1159 throws FileNotFoundException, TimeoutException { 1160 pollForExternalStorageMountedState(); 1161 return MediaProvider.getFuseDaemonForFileWithWait(fuseFilePath, mVolumeCache, 1162 WAIT_TIME_20_SECONDS_IN_MILLIS); 1163 } 1164 setStableUrisGlobalFlag(String volumeName, boolean isEnabled)1165 void setStableUrisGlobalFlag(String volumeName, boolean isEnabled) { 1166 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 1167 mIsStableUriEnabledForInternal = isEnabled; 1168 } else if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)) { 1169 mIsStableUriEnabledForExternal = isEnabled; 1170 } else { 1171 mIsStableUrisEnabledForPublic = isEnabled; 1172 } 1173 } 1174 getVolumeNameForStatsLog(String volumeName)1175 private int getVolumeNameForStatsLog(String volumeName) { 1176 if (volumeName.equalsIgnoreCase(MediaStore.VOLUME_INTERNAL)) { 1177 return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__INTERNAL; 1178 } else if (volumeName.equalsIgnoreCase(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 1179 return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY; 1180 } 1181 1182 return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC; 1183 } 1184 getFuseFilePathFromVolumeName(String volumeName)1185 private static String getFuseFilePathFromVolumeName(String volumeName) { 1186 if (Strings.isNullOrEmpty(volumeName)) { 1187 // Returning EXTERNAL_PRIMARY_ROOT_PATH to avoid any regressions 1188 Log.e(TAG, "Trying to get a Fuse Daemon for volume name = " + volumeName); 1189 return EXTERNAL_PRIMARY_ROOT_PATH; 1190 } 1191 switch (volumeName) { 1192 case MediaStore.VOLUME_INTERNAL: 1193 case MediaStore.VOLUME_EXTERNAL_PRIMARY: 1194 return EXTERNAL_PRIMARY_ROOT_PATH; 1195 default: 1196 return "/storage/" + volumeName.toUpperCase(Locale.ROOT); 1197 } 1198 } 1199 1200 /** 1201 * Returns list of backed up files from external storage. 1202 */ getBackupFiles()1203 List<File> getBackupFiles() { 1204 return Arrays.asList(new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).listFiles()); 1205 } 1206 1207 /** 1208 * Updates backup in external storage to the latest values. Deletes backup of old file path if 1209 * file path has changed. 1210 */ updateBackup(DatabaseHelper helper, FileRow oldRow, FileRow newRow)1211 void updateBackup(DatabaseHelper helper, FileRow oldRow, FileRow newRow) { 1212 if (!isBackupUpdateAllowed(helper, newRow.getVolumeName())) { 1213 return; 1214 } 1215 1216 final FuseDaemon fuseDaemonExternalPrimary; 1217 try { 1218 fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait( 1219 new File(EXTERNAL_PRIMARY_ROOT_PATH)); 1220 } catch (Exception e) { 1221 Log.e(TAG, 1222 "Fuse Daemon not found for primary external storage, skipping update of " 1223 + newRow.getPath(), e); 1224 return; 1225 } 1226 1227 final FuseDaemon fuseDaemonPublicVolume; 1228 if (!isInternalOrExternalPrimary(newRow.getVolumeName())) { 1229 try { 1230 fuseDaemonPublicVolume = getFuseDaemonForFileWithWait(new File( 1231 getFuseFilePathFromVolumeName(newRow.getVolumeName()))); 1232 } catch (Exception e) { 1233 Log.e(TAG, 1234 "Error occurred while retrieving the Fuse Daemon for " 1235 + getFuseFilePathFromVolumeName(newRow.getVolumeName()) 1236 + ", skipping update of " + newRow.getPath(), 1237 e); 1238 return; 1239 } 1240 } else { 1241 fuseDaemonPublicVolume = null; 1242 } 1243 1244 mLevelDbUpdateLock.lock(); 1245 try { 1246 helper.runWithTransaction((db) -> { 1247 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, "_id=?", 1248 new String[]{String.valueOf(newRow.getId())}, null, null, null, 1249 null, null)) { 1250 if (c.moveToFirst()) { 1251 backupDataValues(fuseDaemonExternalPrimary, fuseDaemonPublicVolume, c); 1252 String newPath = c.getString(1); 1253 if (oldRow.getPath() != null && !oldRow.getPath().equalsIgnoreCase( 1254 newPath)) { 1255 // If file path has changed, update leveldb backup to delete old path. 1256 deleteFromDbBackup(helper, oldRow); 1257 Log.v(TAG, "Deleted backup of old file path: " 1258 + oldRow.getPath()); 1259 } 1260 } 1261 } catch (Exception e) { 1262 Log.e(TAG, "Failure in updating row in external storage backup.", e); 1263 } 1264 return null; 1265 }); 1266 } catch (Exception e) { 1267 Log.e(TAG, "Failure in updating row in external storage backup.", e); 1268 } finally { 1269 mLevelDbUpdateLock.unlock(); 1270 } 1271 } 1272 1273 /** 1274 * Removes database recovery data for given user id. This is done when a user is removed. 1275 */ removeRecoveryDataForUserId(int removedUserId)1276 void removeRecoveryDataForUserId(int removedUserId) { 1277 String removeduserIdString = String.valueOf(removedUserId); 1278 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1279 INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.concat( 1280 removeduserIdString)); 1281 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1282 EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.concat( 1283 removeduserIdString)); 1284 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1285 INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.concat(removeduserIdString)); 1286 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1287 EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.concat(removeduserIdString)); 1288 Log.v(TAG, "Removed recovery data for user id: " + removedUserId); 1289 } 1290 1291 /** 1292 * Removes database recovery data for obsolete user id. It accepts list of valid/active users 1293 * and removes the recovery data for ones not present in this list. 1294 * This is done during an idle maintenance. 1295 */ removeRecoveryDataExceptValidUsers(List<String> validUsers)1296 void removeRecoveryDataExceptValidUsers(List<String> validUsers) { 1297 List<String> xattrList = listXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH); 1298 Log.i(TAG, "Xattr list is " + xattrList); 1299 if (xattrList.isEmpty()) { 1300 return; 1301 } 1302 1303 Log.i(TAG, "Valid users list is " + validUsers); 1304 List<String> invalidUsers = getInvalidUsersList(xattrList, validUsers); 1305 Log.i(TAG, "Invalid users list is " + invalidUsers); 1306 for (String userIdToBeRemoved : invalidUsers) { 1307 if (userIdToBeRemoved != null && !userIdToBeRemoved.trim().isEmpty()) { 1308 removeRecoveryDataForUserId(Integer.parseInt(userIdToBeRemoved)); 1309 } 1310 } 1311 } 1312 publishRecoveryMetric(String volumeName, long recoveryTime, long rowsRecovered, long dirtyRowsCount, long totalLevelDbRows, long insertionFailureCount, int status)1313 private void publishRecoveryMetric(String volumeName, long recoveryTime, long rowsRecovered, 1314 long dirtyRowsCount, long totalLevelDbRows, long insertionFailureCount, int status) { 1315 MediaProviderStatsLog.write( 1316 MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED, 1317 getVolumeNameForStatsLog(volumeName), recoveryTime, rowsRecovered, 1318 dirtyRowsCount, totalLevelDbRows, insertionFailureCount, status); 1319 } 1320 getInvalidUsersList(List<String> recoveryData, List<String> validUsers)1321 static List<String> getInvalidUsersList(List<String> recoveryData, 1322 List<String> validUsers) { 1323 Set<String> presentUserIdsAsXattr = new HashSet<>(); 1324 for (String xattr : recoveryData) { 1325 if (xattr.startsWith(INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX)) { 1326 presentUserIdsAsXattr.add( 1327 xattr.substring(INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.length())); 1328 } else if (xattr.startsWith(EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX)) { 1329 presentUserIdsAsXattr.add( 1330 xattr.substring(EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.length())); 1331 } else if (xattr.startsWith(INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX)) { 1332 presentUserIdsAsXattr.add( 1333 xattr.substring(INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.length())); 1334 } else if (xattr.startsWith(EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX)) { 1335 presentUserIdsAsXattr.add( 1336 xattr.substring(EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.length())); 1337 } 1338 } 1339 // Remove valid users 1340 validUsers.forEach(presentUserIdsAsXattr::remove); 1341 return presentUserIdsAsXattr.stream().collect(Collectors.toList()); 1342 } 1343 pollForExternalStorageMountedState()1344 private static void pollForExternalStorageMountedState() throws TimeoutException { 1345 final File target = Environment.getExternalStorageDirectory(); 1346 for (int i = 0; i < WAIT_TIME_20_SECONDS_IN_MILLIS / 100; i++) { 1347 if (Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState(target))) { 1348 return; 1349 } 1350 Log.v(TAG, "Waiting for external storage..."); 1351 SystemClock.sleep(100); 1352 } 1353 throw new TimeoutException("Timed out while waiting for ExternalStorageState " 1354 + "to be MEDIA_MOUNTED"); 1355 } 1356 1357 /** 1358 * Performs actions to be taken on volume unmount. 1359 * @param volumeName name of volume which is detached 1360 */ onDetachVolume(String volumeName)1361 public void onDetachVolume(String volumeName) { 1362 if (mSetupCompleteVolumes.contains(volumeName)) { 1363 mSetupCompleteVolumes.remove(volumeName); 1364 Log.v(TAG, 1365 "Removed leveldb connections from in memory setup cache for volume:" 1366 + volumeName); 1367 } 1368 } 1369 } 1370