• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.providers.media;
18 
19 import static com.android.providers.media.DatabaseHelper.DATA_MEDIA_XATTR_DIRECTORY_PATH;
20 import static com.android.providers.media.DatabaseHelper.EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX;
21 import static com.android.providers.media.DatabaseHelper.EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX;
22 import static com.android.providers.media.DatabaseHelper.INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX;
23 import static com.android.providers.media.DatabaseHelper.INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX;
24 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__BACKUP_MISSING;
25 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__FUSE_DAEMON_TIMEOUT;
26 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__GET_BACKUP_DATA_FAILURE;
27 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__OTHER_ERROR;
28 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__SUCCESS;
29 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__VOLUME_NOT_ATTACHED;
30 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY;
31 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__INTERNAL;
32 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC;
33 import static com.android.providers.media.util.Logging.TAG;
34 import static com.android.providers.media.flags.Flags.enableStableUrisForExternalPrimaryVolume;
35 import static com.android.providers.media.flags.Flags.enableStableUrisForPublicVolume;
36 
37 import android.content.ContentValues;
38 import android.database.Cursor;
39 import android.database.sqlite.SQLiteDatabase;
40 import android.os.Build;
41 import android.os.CancellationSignal;
42 import android.os.Environment;
43 import android.os.ParcelFileDescriptor;
44 import android.os.SystemClock;
45 import android.os.SystemProperties;
46 import android.os.UserHandle;
47 import android.provider.MediaStore;
48 import android.system.ErrnoException;
49 import android.system.Os;
50 import android.system.OsConstants;
51 import android.util.Log;
52 import android.util.Pair;
53 
54 import androidx.annotation.NonNull;
55 
56 import com.android.providers.media.dao.FileRow;
57 import com.android.providers.media.fuse.FuseDaemon;
58 import com.android.providers.media.stableuris.dao.BackupIdRow;
59 import com.android.providers.media.util.StringUtils;
60 
61 import com.google.common.base.Strings;
62 
63 import java.io.File;
64 import java.io.FileNotFoundException;
65 import java.io.IOException;
66 import java.util.ArrayList;
67 import java.util.Arrays;
68 import java.util.HashSet;
69 import java.util.List;
70 import java.util.Locale;
71 import java.util.Map;
72 import java.util.Optional;
73 import java.util.Set;
74 import java.util.concurrent.ConcurrentHashMap;
75 import java.util.concurrent.TimeoutException;
76 import java.util.concurrent.atomic.AtomicInteger;
77 import java.util.concurrent.locks.ReentrantLock;
78 import java.util.stream.Collectors;
79 
80 /**
81  * To ensure that the ids of MediaStore database uris are stable and reliable.
82  */
83 public class DatabaseBackupAndRecovery {
84 
85     private static final String LOWER_FS_RECOVERY_DIRECTORY_PATH =
86             "/data/media/" + UserHandle.myUserId() + "/.transforms/recovery";
87 
88     /**
89      * Path for storing owner id to owner package identifier relation and vice versa.
90      * Lower file system path is used as upper file system does not support xattrs.
91      */
92     private static final String OWNER_RELATION_LOWER_FS_BACKUP_PATH =
93             "/data/media/" + UserHandle.myUserId() + "/.transforms/recovery/leveldb-ownership";
94 
95     private static final String INTERNAL_VOLUME_LOWER_FS_BACKUP_PATH =
96             LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-internal";
97 
98     private static final String EXTERNAL_PRIMARY_VOLUME_LOWER_FS_BACKUP_PATH =
99             LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-external_primary";
100 
101     /**
102      * Every LevelDB table name starts with this prefix.
103      */
104     private static final String LEVEL_DB_PREFIX = "leveldb-";
105 
106     private static final String OWNERSHIP_TABLE_NAME = LEVEL_DB_PREFIX + "ownership";
107 
108     /**
109      * Frequency at which next value of owner id is backed up in the external storage.
110      */
111     private static final int NEXT_OWNER_ID_BACKUP_FREQUENCY = 50;
112 
113     /**
114      * Start value used for next owner id.
115      */
116     private static final int NEXT_OWNER_ID_DEFAULT_VALUE = 0;
117 
118     /**
119      * Key name of xattr used to set next owner id on ownership DB.
120      */
121     private static final String NEXT_OWNER_ID_XATTR_KEY = "user.nextownerid";
122 
123     /**
124      * Key name of xattr used to store last modified generation number.
125      */
126     private static final String LAST_BACKEDUP_GENERATION_XATTR_KEY = "user.lastbackedgeneration";
127 
128     /**
129      * Key name of xattr used to store a public volume recovery flag.
130      */
131     private static final String PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY
132             = "user.publicvolumerecoveryflag";
133 
134     /**
135      * External primary storage root path for given user.
136      */
137     private static final String EXTERNAL_PRIMARY_ROOT_PATH =
138             "/storage/emulated/" + UserHandle.myUserId();
139 
140     /**
141      * Array of columns backed up in external storage.
142      */
143     private static final String[] QUERY_COLUMNS = new String[]{
144             MediaStore.Files.FileColumns._ID,
145             MediaStore.Files.FileColumns.DATA,
146             MediaStore.Files.FileColumns.IS_FAVORITE,
147             MediaStore.Files.FileColumns.IS_PENDING,
148             MediaStore.Files.FileColumns.IS_TRASHED,
149             MediaStore.Files.FileColumns.MEDIA_TYPE,
150             MediaStore.Files.FileColumns._USER_ID,
151             MediaStore.Files.FileColumns.DATE_EXPIRES,
152             MediaStore.Files.FileColumns.OWNER_PACKAGE_NAME,
153             MediaStore.Files.FileColumns.GENERATION_MODIFIED,
154             MediaStore.Files.FileColumns.VOLUME_NAME
155     };
156 
157     /**
158      * Wait time of 20 seconds in millis.
159      */
160     private static final long WAIT_TIME_20_SECONDS_IN_MILLIS = 20000;
161 
162     /**
163      * Number of records to read from leveldb in a JNI call.
164      */
165     protected static final int LEVEL_DB_READ_LIMIT = 100;
166 
167 
168     /**
169      * Re-entrant lock to ensure sequential update to leveldb by background threads.
170      */
171     private final ReentrantLock mLevelDbUpdateLock = new ReentrantLock();
172 
173     /**
174      * Stores cached value of next owner id. This helps in improving performance by backing up next
175      * row id less frequently in the external storage.
176      */
177     private AtomicInteger mNextOwnerId;
178 
179     /**
180      * Stores value of next backup of owner id.
181      */
182     private AtomicInteger mNextOwnerIdBackup;
183     private final ConfigStore mConfigStore;
184     private final VolumeCache mVolumeCache;
185     private Set<String> mSetupCompleteVolumes = ConcurrentHashMap.newKeySet();
186 
187     // Flag only used to enable/disable feature for testing
188     private boolean mIsStableUriEnabledForInternal = false;
189 
190     // Flag only used to enable/disable feature for testing
191     private boolean mIsStableUriEnabledForExternal = false;
192 
193     // Flag only used to enable/disable feature for testing
194     private boolean mIsStableUrisEnabledForPublic = false;
195 
196     private static Map<String, String> sOwnerIdRelationMap;
197 
198     public static final String STABLE_URI_INTERNAL_PROPERTY =
199             "persist.sys.fuse.backup.internal_db_backup";
200 
201     private static boolean STABLE_URI_INTERNAL_PROPERTY_VALUE = true;
202 
203     public static final String STABLE_URI_EXTERNAL_PROPERTY =
204             "persist.sys.fuse.backup.external_volume_backup";
205 
206     private static boolean STABLE_URI_EXTERNAL_PROPERTY_VALUE = true;
207 
208     public static final String STABLE_URI_PUBLIC_PROPERTY =
209             "persist.sys.fuse.backup.public_db_backup";
210 
211     private static boolean STABLE_URI_PUBLIC_PROPERTY_VALUE = true;
212 
DatabaseBackupAndRecovery(ConfigStore configStore, VolumeCache volumeCache)213     DatabaseBackupAndRecovery(ConfigStore configStore, VolumeCache volumeCache) {
214         mConfigStore = configStore;
215         mVolumeCache = volumeCache;
216     }
217 
218     /**
219      * Returns true if migration and recovery code flow for stable uris is enabled for given volume.
220      */
isStableUrisEnabled(String volumeName)221     boolean isStableUrisEnabled(String volumeName) {
222         // Check if flags are enabled for test for internal volume
223         if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)
224                 && mIsStableUriEnabledForInternal) {
225             return true;
226         }
227         // Check if flags are enabled for test for external primary volume
228         if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)
229                 && mIsStableUriEnabledForExternal) {
230             return true;
231         }
232 
233         // Check if flags are enabled for test for public volume
234         if (!MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)
235                 && !MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)
236                 && mIsStableUrisEnabledForPublic) {
237             return true;
238         }
239 
240         // Feature is disabled for below S due to vold mount issues.
241         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
242             return false;
243         }
244 
245         switch (volumeName) {
246             case MediaStore.VOLUME_INTERNAL:
247                 return mIsStableUriEnabledForInternal
248                         || mConfigStore.isStableUrisForInternalVolumeEnabled()
249                         || SystemProperties.getBoolean(STABLE_URI_INTERNAL_PROPERTY,
250                         /* defaultValue */ STABLE_URI_INTERNAL_PROPERTY_VALUE);
251             case MediaStore.VOLUME_EXTERNAL_PRIMARY:
252                 return mIsStableUriEnabledForExternal
253                         || mConfigStore.isStableUrisForExternalVolumeEnabled()
254                         || enableStableUrisForExternalPrimaryVolume()
255                         || SystemProperties.getBoolean(STABLE_URI_EXTERNAL_PROPERTY,
256                         /* defaultValue */ STABLE_URI_EXTERNAL_PROPERTY_VALUE);
257             default:
258                 // public volume
259                 return mIsStableUrisEnabledForPublic
260                         || mConfigStore.isStableUrisForPublicVolumeEnabled()
261                         || enableStableUrisForPublicVolume()
262                         || SystemProperties.getBoolean(STABLE_URI_PUBLIC_PROPERTY,
263                         /* defaultValue */ STABLE_URI_PUBLIC_PROPERTY_VALUE);
264         }
265     }
266 
267     /**
268      * On device boot, leveldb setup is done as part of attachVolume call for primary external.
269      * Also, on device config flag change, we check if flag is enabled, if yes, we proceed to
270      * setup(no-op if connection already exists). So, we setup backup and recovery for internal
271      * volume on Media mount signal of EXTERNAL_PRIMARY.
272      */
setupVolumeDbBackupAndRecovery(String volumeName)273     synchronized void setupVolumeDbBackupAndRecovery(String volumeName) {
274         // Since internal volume does not have any fuse daemon thread, leveldb instance
275         // for internal volume is created by fuse daemon thread of EXTERNAL_PRIMARY.
276         if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) {
277             // Set backup only for external primary for now.
278             return;
279         }
280         // Do not create leveldb instance if stable uris is not enabled for internal volume.
281         if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)) {
282             // Return if we are not supporting backup for internal volume
283             return;
284         }
285 
286         if (mSetupCompleteVolumes.contains(volumeName)) {
287             // Return if setup is already done
288             return;
289         }
290 
291         final long startTime = SystemClock.elapsedRealtime();
292         int vol = MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)
293                 ? MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY
294                 : MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC;
295         try {
296             if (!new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).exists()) {
297                 new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).mkdirs();
298                 Log.v(TAG, "Created recovery directory:" + LOWER_FS_RECOVERY_DIRECTORY_PATH);
299             }
300             FuseDaemon fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait(new File(
301                     DatabaseBackupAndRecovery.EXTERNAL_PRIMARY_ROOT_PATH));
302             Log.d(TAG, "Received db backup Fuse Daemon for: " + volumeName);
303             if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) && (
304                     isStableUrisEnabled(MediaStore.VOLUME_INTERNAL) || isStableUrisEnabled(
305                             MediaStore.VOLUME_EXTERNAL_PRIMARY))) {
306                 // Setup internal and external volumes
307                 MediaProviderStatsLog.write(
308                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED,
309                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__ATTEMPTED, vol);
310                 fuseDaemonExternalPrimary.setupVolumeDbBackup();
311                 mSetupCompleteVolumes.add(volumeName);
312                 MediaProviderStatsLog.write(
313                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED,
314                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__SUCCESS, vol);
315             } else if (isStableUrisEnabled(volumeName)) {
316                 // Setup public volume
317                 FuseDaemon fuseDaemonPublicVolume = getFuseDaemonForPath(
318                         getFuseFilePathFromVolumeName(volumeName));
319                 MediaProviderStatsLog.write(
320                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED,
321                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__ATTEMPTED, vol);
322                 fuseDaemonPublicVolume.setupPublicVolumeDbBackup(volumeName);
323                 mSetupCompleteVolumes.add(volumeName);
324                 MediaProviderStatsLog.write(
325                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED,
326                         MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__SUCCESS, vol);
327             } else {
328                 return;
329             }
330         } catch (Exception e) {
331             MediaProviderStatsLog.write(
332                     MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED,
333                     MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__FAILURE, vol);
334             Log.e(TAG, "Failure in setting up backup and recovery for volume: " + volumeName, e);
335             return;
336         } finally {
337             Log.i(TAG, "Backup and recovery setup time taken in milliseconds:" + (
338                     SystemClock.elapsedRealtime() - startTime));
339         }
340         Log.i(TAG, "Successfully set up backup and recovery for volume: " + volumeName);
341     }
342 
343     /**
344      * Backs up databases to external storage to ensure stable URIs.
345      */
backupDatabases(DatabaseHelper internalDatabaseHelper, DatabaseHelper externalDatabaseHelper, CancellationSignal signal)346     void backupDatabases(DatabaseHelper internalDatabaseHelper,
347             DatabaseHelper externalDatabaseHelper, CancellationSignal signal) {
348         mLevelDbUpdateLock.lock();
349         try {
350             setupVolumeDbBackupAndRecovery(MediaStore.VOLUME_EXTERNAL_PRIMARY);
351             Log.i(TAG, "Triggering database backup");
352             backupInternalDatabase(internalDatabaseHelper, signal);
353             backupExternalDatabase(externalDatabaseHelper,
354                     MediaStore.VOLUME_EXTERNAL_PRIMARY, signal);
355 
356             for (MediaVolume mediaVolume : mVolumeCache.getExternalVolumes()) {
357                 if (mediaVolume.isPublicVolume()) {
358                     setupVolumeDbBackupAndRecovery(mediaVolume.getName());
359                     backupExternalDatabase(externalDatabaseHelper, mediaVolume.getName(), signal);
360                 }
361             }
362         } catch (Exception e) {
363             Log.e(TAG, "Failure in backing up databases", e);
364         } finally {
365             mLevelDbUpdateLock.unlock();
366         }
367     }
368 
readDataFromBackup(String volumeName, String filePath)369     Optional<BackupIdRow> readDataFromBackup(String volumeName, String filePath) {
370         if (!isStableUrisEnabled(volumeName)) {
371             return Optional.empty();
372         }
373 
374         try {
375             final String data = getFuseDaemonForPath(getFuseFilePathFromVolumeName(volumeName))
376                     .readBackedUpData(filePath);
377             if (data == null || data.isEmpty()) {
378                 Log.w(TAG, "No backup found for path: " + filePath);
379                 return Optional.empty();
380             }
381 
382             return Optional.of(BackupIdRow.deserialize(data));
383         } catch (Exception e) {
384             Log.e(TAG, "Failure in getting backed up data for filePath: " + filePath, e);
385             return Optional.empty();
386         }
387     }
388 
backupInternalDatabase(DatabaseHelper internalDbHelper, CancellationSignal signal)389     private void backupInternalDatabase(DatabaseHelper internalDbHelper,
390             CancellationSignal signal) {
391         if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)
392                 || internalDbHelper.isDatabaseRecovering()) {
393             return;
394         }
395 
396         if (!mSetupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) {
397             Log.w(TAG,
398                 "Setup is not present for backup of internal and external primary volume.");
399             return;
400         }
401 
402         FuseDaemon fuseDaemon;
403         try {
404             fuseDaemon = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH);
405         } catch (FileNotFoundException e) {
406             Log.e(TAG,
407                     "Fuse Daemon not found for primary external storage, skipping backing up of "
408                             + "internal database.",
409                     e);
410             return;
411         }
412 
413         internalDbHelper.runWithTransaction((db) -> {
414             try (Cursor c = db.query(true, "files", QUERY_COLUMNS, null, null, null, null, null,
415                     null, signal)) {
416                 while (c.moveToNext()) {
417                     backupDataValues(fuseDaemon, c);
418                 }
419                 Log.d(TAG, String.format(Locale.ROOT,
420                         "Backed up %d rows of internal database to external storage on idle "
421                                 + "maintenance.",
422                         c.getCount()));
423             } catch (Exception e) {
424                 Log.e(TAG, "Failure in backing up internal database to external storage.", e);
425             }
426             return null;
427         });
428     }
429 
backupExternalDatabase(DatabaseHelper externalDbHelper, String volumeName, CancellationSignal signal)430     private void backupExternalDatabase(DatabaseHelper externalDbHelper,
431             String volumeName, CancellationSignal signal) {
432         if (!isStableUrisEnabled(volumeName)
433                 || externalDbHelper.isDatabaseRecovering()) {
434             return;
435         }
436 
437         if (!mSetupCompleteVolumes.contains(volumeName)) {
438             return;
439         }
440 
441         FuseDaemon fuseDaemonExternalPrimary;
442         try {
443             fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait(
444                     new File(EXTERNAL_PRIMARY_ROOT_PATH));
445         } catch (Exception e) {
446             Log.e(TAG,
447                     "Error occurred while retrieving the Fuse Daemon for the external primary, "
448                             + "skipping backing up of "
449                             + volumeName, e);
450             return;
451         }
452         FuseDaemon fuseDaemonPublicVolume;
453         if (!isInternalOrExternalPrimary(volumeName)) {
454             try {
455                 fuseDaemonPublicVolume = getFuseDaemonForFileWithWait(new File(
456                         getFuseFilePathFromVolumeName(volumeName)));
457             } catch (Exception e) {
458                 Log.e(TAG,
459                         "Error occurred while retrieving the Fuse Daemon for "
460                                 + getFuseFilePathFromVolumeName(volumeName)
461                                 + ", skipping backing up of " + volumeName,
462                         e);
463                 return;
464             }
465         } else {
466             fuseDaemonPublicVolume = null;
467         }
468 
469         final String backupPath =
470                 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX + volumeName;
471         long lastBackedGenerationNumber = getLastBackedGenerationNumber(backupPath);
472 
473         final String generationClause = MediaStore.Files.FileColumns.GENERATION_MODIFIED + " >= "
474                 + lastBackedGenerationNumber;
475         final String volumeClause = MediaStore.Files.FileColumns.VOLUME_NAME + " = '"
476                 + volumeName + "'";
477         final String selectionClause = generationClause + " AND " + volumeClause;
478 
479         externalDbHelper.runWithTransaction((db) -> {
480             long maxGeneration = lastBackedGenerationNumber;
481             Log.d(TAG, "Started to back up " + volumeName
482                     + ", maxGeneration:" + maxGeneration);
483             try (Cursor c = db.query(true, "files", QUERY_COLUMNS, selectionClause, null, null,
484                     null, MediaStore.MediaColumns.GENERATION_MODIFIED + " ASC", null, signal)) {
485                 while (c.moveToNext()) {
486                     if (signal != null && signal.isCanceled()) {
487                         Log.i(TAG, "Received a cancellation signal during the DB "
488                                 + "backup process");
489                         break;
490                     }
491                     if (isInternalOrExternalPrimary(volumeName)) {
492                         backupDataValues(fuseDaemonExternalPrimary, c);
493                     } else {
494                         // public volume
495                         backupDataValues(fuseDaemonExternalPrimary, fuseDaemonPublicVolume, c);
496                     }
497                     maxGeneration = Math.max(maxGeneration, c.getLong(9));
498                 }
499                 setXattr(backupPath, LAST_BACKEDUP_GENERATION_XATTR_KEY,
500                         String.valueOf(maxGeneration - 1));
501                 Log.d(TAG, String.format(Locale.ROOT,
502                         "Backed up %d rows of " + volumeName + " to external storage on idle "
503                                 + "maintenance.",
504                         c.getCount()));
505             } catch (Exception e) {
506                 Log.e(TAG, "Failure in backing up " + volumeName + " to external storage.", e);
507                 return null;
508             }
509             return null;
510         });
511     }
512 
backupDataValues(FuseDaemon fuseDaemon, Cursor c)513     private void backupDataValues(FuseDaemon fuseDaemon, Cursor c) throws IOException {
514         backupDataValues(fuseDaemon, null, c);
515     }
516 
backupDataValues(FuseDaemon externalPrimaryFuseDaemon, FuseDaemon publicVolumeFuseDaemon, Cursor c)517     private void backupDataValues(FuseDaemon externalPrimaryFuseDaemon,
518             FuseDaemon publicVolumeFuseDaemon, Cursor c) throws IOException {
519         final long id = c.getLong(0);
520         final String data = c.getString(1);
521         final boolean isFavorite = c.getInt(2) != 0;
522         final boolean isPending = c.getInt(3) != 0;
523         final boolean isTrashed = c.getInt(4) != 0;
524         final int mediaType = c.getInt(5);
525         final int userId = c.getInt(6);
526         final String dateExpires = c.getString(7);
527         final String ownerPackageName = c.getString(8);
528         final String volumeName = c.getString(10);
529         BackupIdRow backupIdRow = createBackupIdRow(externalPrimaryFuseDaemon, id, mediaType,
530                 isFavorite, isPending, isTrashed, userId, dateExpires, ownerPackageName);
531         if (isInternalOrExternalPrimary(volumeName)) {
532             externalPrimaryFuseDaemon.backupVolumeDbData(volumeName, data,
533                     BackupIdRow.serialize(backupIdRow));
534         } else {
535             // public volume
536             publicVolumeFuseDaemon.backupVolumeDbData(volumeName, data,
537                     BackupIdRow.serialize(backupIdRow));
538         }
539     }
540 
deleteBackupForVolume(String volumeName)541     void deleteBackupForVolume(String volumeName) {
542         File dbFilePath = new File(
543                 String.format(Locale.ROOT, "%s/%s.db", LOWER_FS_RECOVERY_DIRECTORY_PATH,
544                         LEVEL_DB_PREFIX + volumeName));
545         if (dbFilePath.exists()) {
546             dbFilePath.delete();
547         }
548     }
549 
readBackedUpFilePaths(String volumeName, String lastReadValue, int limit)550     String[] readBackedUpFilePaths(String volumeName, String lastReadValue, int limit)
551             throws IOException, UnsupportedOperationException {
552         if (!isStableUrisEnabled(volumeName)) {
553             throw new UnsupportedOperationException("Stable Uris are not enabled");
554         }
555 
556         return getFuseDaemonForPath(getFuseFilePathFromVolumeName(volumeName))
557                 .readBackedUpFilePaths(volumeName, lastReadValue, limit);
558     }
559 
updateNextRowIdXattr(DatabaseHelper helper, long id)560     void updateNextRowIdXattr(DatabaseHelper helper, long id) {
561         if (helper.isInternal()) {
562             updateNextRowIdForInternal(helper, id);
563             return;
564         }
565 
566         if (!helper.isNextRowIdBackupEnabled()) {
567             return;
568         }
569 
570         Optional<Long> nextRowIdBackupOptional = helper.getNextRowId();
571         if (!nextRowIdBackupOptional.isPresent()) {
572             throw new RuntimeException(
573                     String.format(Locale.ROOT, "Cannot find next row id xattr for %s.",
574                             helper.getDatabaseName()));
575         }
576 
577         if (id >= nextRowIdBackupOptional.get()) {
578             helper.backupNextRowId(id);
579         }
580     }
581 
getLastBackedGenerationNumber(String backupPath)582     private long getLastBackedGenerationNumber(String backupPath) {
583         // Read last backed up generation number
584         Optional<Long> lastBackedUpGenNum = getXattrOfLongValue(
585                 backupPath, LAST_BACKEDUP_GENERATION_XATTR_KEY);
586         long lastBackedGenerationNumber = lastBackedUpGenNum.isPresent()
587                 ? lastBackedUpGenNum.get() : 0;
588         if (lastBackedGenerationNumber > 0) {
589             Log.i(TAG, "Last backed up generation number for " + backupPath + " is "
590                     + lastBackedGenerationNumber);
591         }
592         return lastBackedGenerationNumber;
593     }
594 
595     @NonNull
getFuseDaemonForPath(@onNull String path)596     private FuseDaemon getFuseDaemonForPath(@NonNull String path)
597             throws FileNotFoundException {
598         return MediaProvider.getFuseDaemonForFile(new File(path), mVolumeCache);
599     }
600 
updateNextRowIdAndSetDirty(@onNull DatabaseHelper helper, @NonNull FileRow oldRow, @NonNull FileRow newRow)601     void updateNextRowIdAndSetDirty(@NonNull DatabaseHelper helper,
602             @NonNull FileRow oldRow, @NonNull FileRow newRow) {
603         updateNextRowIdXattr(helper, newRow.getId());
604         markBackupAsDirty(helper, oldRow);
605     }
606 
607     /**
608      * Backs up DB data in external storage to recover in case of DB rollback.
609      */
backupVolumeDbData(DatabaseHelper databaseHelper, FileRow insertedRow)610     void backupVolumeDbData(DatabaseHelper databaseHelper, FileRow insertedRow) {
611         if (!isBackupUpdateAllowed(databaseHelper, insertedRow.getVolumeName())) {
612             return;
613         }
614 
615         mLevelDbUpdateLock.lock();
616         try {
617             FuseDaemon fuseDaemonExternalPrimary = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH);
618             final BackupIdRow value = createBackupIdRow(fuseDaemonExternalPrimary, insertedRow);
619             if (isInternalOrExternalPrimary(insertedRow.getVolumeName())) {
620                 fuseDaemonExternalPrimary.backupVolumeDbData(insertedRow.getVolumeName(),
621                         insertedRow.getPath(), BackupIdRow.serialize(value));
622             } else {
623                 // public volume
624                 final FuseDaemon fuseDaemonPublicVolume = getFuseDaemonForPath(
625                         getFuseFilePathFromVolumeName(insertedRow.getVolumeName()));
626                 fuseDaemonPublicVolume.backupVolumeDbData(insertedRow.getVolumeName(),
627                         insertedRow.getPath(), BackupIdRow.serialize(value));
628             }
629         } catch (Exception e) {
630             Log.e(TAG, "Failure in backing up data to external storage", e);
631         } finally {
632             mLevelDbUpdateLock.unlock();
633         }
634     }
635 
createBackupIdRow(FuseDaemon fuseDaemon, FileRow insertedRow)636     private BackupIdRow createBackupIdRow(FuseDaemon fuseDaemon, FileRow insertedRow)
637             throws IOException {
638         return createBackupIdRow(fuseDaemon, insertedRow.getId(), insertedRow.getMediaType(),
639                 insertedRow.isFavorite(), insertedRow.isPending(), insertedRow.isTrashed(),
640                 insertedRow.getUserId(), insertedRow.getDateExpires(),
641                 insertedRow.getOwnerPackageName());
642     }
643 
createBackupIdRow(FuseDaemon fuseDaemon, long id, int mediaType, boolean isFavorite, boolean isPending, boolean isTrashed, int userId, String dateExpires, String ownerPackageName)644     private BackupIdRow createBackupIdRow(FuseDaemon fuseDaemon, long id, int mediaType,
645             boolean isFavorite,
646             boolean isPending, boolean isTrashed, int userId, String dateExpires,
647             String ownerPackageName) throws IOException {
648         BackupIdRow.Builder builder = BackupIdRow.newBuilder(id);
649         builder.setMediaType(mediaType);
650         builder.setIsFavorite(isFavorite ? 1 : 0);
651         builder.setIsPending(isPending ? 1 : 0);
652         builder.setIsTrashed(isTrashed ? 1 : 0);
653         builder.setUserId(userId);
654         builder.setDateExpires(dateExpires);
655         // We set owner package id instead of owner package name in the backup. When an
656         // application is uninstalled, all media rows corresponding to it will be orphaned and
657         // would have owner package name as null. This should not change if application is
658         // installed again. Therefore, we are storing owner id instead of owner package name. On
659         // package uninstallation, we delete the owner id relation from the backup. All rows
660         // recovered for orphaned owner ids will have package name as null. Since we also need to
661         // support cloned apps, we are storing a combination of owner package name and user id to
662         // uniquely identify a package.
663         builder.setOwnerPackagedId(getOwnerPackageId(fuseDaemon, ownerPackageName, userId));
664         return builder.setIsDirty(false).build();
665     }
666 
667 
getOwnerPackageId(FuseDaemon fuseDaemon, String ownerPackageName, int userId)668     private synchronized int getOwnerPackageId(FuseDaemon fuseDaemon,
669             String ownerPackageName, int userId) throws IOException {
670         // In synchronized block to avoid use of same owner id for multiple owner package relations
671         if (Strings.isNullOrEmpty(ownerPackageName) || ownerPackageName.equalsIgnoreCase("null")) {
672             // We store -1 in the backup if owner package name is null.
673             return -1;
674         }
675 
676         // Create identifier of format "owner_pkg_name::user_id". Tightly coupling owner package
677         // name and user id helps in handling app cloning scenarios.
678         String ownerPackageIdentifier = createOwnerPackageIdentifier(ownerPackageName, userId);
679         // Read any existing entry for given owner package name and user id
680         String ownerId = fuseDaemon.readFromOwnershipBackup(ownerPackageIdentifier);
681         if (!ownerId.trim().isEmpty()) {
682             // Use existing owner id if found and is positive
683             int val = Integer.parseInt(ownerId);
684             if (val >= 0) {
685                 return val;
686             }
687         }
688 
689         int nextOwnerId = getAndIncrementNextOwnerId();
690         fuseDaemon.createOwnerIdRelation(String.valueOf(nextOwnerId), ownerPackageIdentifier);
691         Log.v(TAG, "Created relation b/w " + nextOwnerId + " and " + ownerPackageIdentifier);
692         return nextOwnerId;
693     }
694 
createOwnerPackageIdentifier(String ownerPackageName, int userId)695     private String createOwnerPackageIdentifier(String ownerPackageName, int userId) {
696         return ownerPackageName.trim().concat("::").concat(String.valueOf(userId));
697     }
698 
getPackageNameAndUserId(String ownerPackageIdentifier)699     private Pair<String, Integer> getPackageNameAndUserId(String ownerPackageIdentifier) {
700         if (ownerPackageIdentifier.trim().isEmpty()) {
701             return Pair.create(null, null);
702         }
703 
704         String[] arr = ownerPackageIdentifier.trim().split("::");
705         return Pair.create(arr[0], Integer.valueOf(arr[1]));
706     }
707 
getAndIncrementNextOwnerId()708     private int getAndIncrementNextOwnerId() {
709         if (mNextOwnerId == null) {
710             Optional<Integer> nextOwnerIdOptional = getXattrOfIntegerValue(
711                     OWNER_RELATION_LOWER_FS_BACKUP_PATH,
712                     NEXT_OWNER_ID_XATTR_KEY);
713             mNextOwnerId = nextOwnerIdOptional.map(AtomicInteger::new).orElseGet(
714                     () -> new AtomicInteger(NEXT_OWNER_ID_DEFAULT_VALUE));
715             mNextOwnerIdBackup = new AtomicInteger(mNextOwnerId.get());
716         }
717         if (mNextOwnerId.get() >= mNextOwnerIdBackup.get()) {
718             int nextBackup = mNextOwnerId.get() + NEXT_OWNER_ID_BACKUP_FREQUENCY;
719             updateNextOwnerId(nextBackup);
720             mNextOwnerIdBackup = new AtomicInteger(nextBackup);
721         }
722         int returnValue = mNextOwnerId.get();
723         mNextOwnerId.set(returnValue + 1);
724         return returnValue;
725     }
726 
updateNextOwnerId(int val)727     private void updateNextOwnerId(int val) {
728         setXattr(OWNER_RELATION_LOWER_FS_BACKUP_PATH, NEXT_OWNER_ID_XATTR_KEY, String.valueOf(val));
729         Log.d(TAG, "Updated next owner id to: " + val);
730     }
731 
removeOwnerIdToPackageRelation(String packageName, int userId)732     void removeOwnerIdToPackageRelation(String packageName, int userId) {
733         if (Strings.isNullOrEmpty(packageName) || packageName.equalsIgnoreCase("null")
734                 || !isStableUrisEnabled(MediaStore.VOLUME_EXTERNAL_PRIMARY)
735                 || !new File(OWNER_RELATION_LOWER_FS_BACKUP_PATH).exists()
736                 || !mSetupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) {
737             return;
738         }
739 
740         try {
741             FuseDaemon fuseDaemon = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH);
742             String ownerPackageIdentifier = createOwnerPackageIdentifier(packageName, userId);
743             String ownerId = fuseDaemon.readFromOwnershipBackup(ownerPackageIdentifier);
744 
745             fuseDaemon.removeOwnerIdRelation(ownerId, ownerPackageIdentifier);
746         } catch (Exception e) {
747             Log.e(TAG, "Failure in removing owner id to package relation", e);
748         }
749     }
750 
751     /**
752      * Deletes backed up data(needed for recovery) from external storage.
753      */
deleteFromDbBackup(DatabaseHelper databaseHelper, FileRow deletedRow)754     void deleteFromDbBackup(DatabaseHelper databaseHelper, FileRow deletedRow) {
755         if (!isBackupUpdateAllowed(databaseHelper, deletedRow.getVolumeName())) {
756             return;
757         }
758 
759         String deletedFilePath = deletedRow.getPath();
760         if (deletedFilePath == null) {
761             return;
762         }
763 
764         mLevelDbUpdateLock.lock();
765         try {
766             getFuseDaemonForPath(getFuseFilePathFromVolumeName(deletedRow.getVolumeName()))
767                     .deleteDbBackup(deletedFilePath);
768         } catch (IOException e) {
769             Log.w(TAG, "Failure in deleting backup data for key: " + deletedFilePath, e);
770         } finally {
771             mLevelDbUpdateLock.unlock();
772         }
773     }
774 
isBackupUpdateAllowed(DatabaseHelper databaseHelper, String volumeName)775     private boolean isBackupUpdateAllowed(DatabaseHelper databaseHelper, String volumeName) {
776         // Backup only if stable uris is enabled, db is not recovering and backup setup is complete.
777         return isStableUrisEnabled(volumeName) && !databaseHelper.isDatabaseRecovering()
778                 && mSetupCompleteVolumes.contains(volumeName);
779     }
780 
isInternalOrExternalPrimary(String volumeName)781     private boolean isInternalOrExternalPrimary(String volumeName) {
782         if (Strings.isNullOrEmpty(volumeName)) {
783             // This should never happen
784             Log.e(TAG, "Volume name is " + volumeName + ", treating it as non public volume");
785             return true;
786         }
787         return MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)
788                 || MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName);
789     }
790 
updateNextRowIdForInternal(DatabaseHelper helper, long id)791     private void updateNextRowIdForInternal(DatabaseHelper helper, long id) {
792         if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)) {
793             return;
794         }
795 
796         Optional<Long> nextRowIdBackupOptional = helper.getNextRowId();
797 
798         if (!nextRowIdBackupOptional.isPresent()) {
799             return;
800         }
801 
802         if (id >= nextRowIdBackupOptional.get()) {
803             helper.backupNextRowId(id);
804         }
805     }
806 
markBackupAsDirty(DatabaseHelper databaseHelper, FileRow updatedRow)807     private void markBackupAsDirty(DatabaseHelper databaseHelper, FileRow updatedRow) {
808         if (!isBackupUpdateAllowed(databaseHelper, updatedRow.getVolumeName())) {
809             return;
810         }
811 
812         final String updatedFilePath = updatedRow.getPath();
813         try {
814             getFuseDaemonForPath(getFuseFilePathFromVolumeName(updatedRow.getVolumeName()))
815                     .backupVolumeDbData(
816                             updatedRow.getVolumeName(),
817                             updatedFilePath,
818                             BackupIdRow.serialize(BackupIdRow.newBuilder(updatedRow.getId())
819                                     .setIsDirty(true).build()));
820         } catch (IOException e) {
821             Log.e(TAG, "Failure in marking data as dirty to external storage for path:"
822                     + updatedFilePath, e);
823         }
824     }
825 
826     /**
827      * Reads value corresponding to given key from xattr on given path.
828      */
getXattr(String path, String key)829     static Optional<String> getXattr(String path, String key) {
830         try {
831             return Optional.of(Arrays.toString(Os.getxattr(path, key)));
832         } catch (Exception e) {
833             Log.w(TAG, String.format(Locale.ROOT,
834                     "Exception encountered while reading xattr:%s from path:%s.", key, path));
835             return Optional.empty();
836         }
837     }
838 
839     /**
840      * Reads long value corresponding to given key from xattr on given path.
841      */
getXattrOfLongValue(String path, String key)842     private static Optional<Long> getXattrOfLongValue(String path, String key) {
843         try {
844             return Optional.of(Long.parseLong(new String(Os.getxattr(path, key))));
845         } catch (Exception e) {
846             Log.w(TAG, String.format(Locale.ROOT,
847                     "Exception encountered while reading xattr:%s from path:%s.", key, path));
848             return Optional.empty();
849         }
850     }
851 
852     /**
853      * Reads integer value corresponding to given key from xattr on given path.
854      */
getXattrOfIntegerValue(String path, String key)855     static Optional<Integer> getXattrOfIntegerValue(String path, String key) {
856         try {
857             return Optional.of(Integer.parseInt(new String(Os.getxattr(path, key))));
858         } catch (Exception e) {
859             Log.w(TAG, String.format(Locale.ROOT,
860                     "Exception encountered while reading xattr:%s from path:%s.", key, path));
861             return Optional.empty();
862         }
863     }
864 
865     /**
866      * Sets key and value as xattr on given path.
867      */
setXattr(String path, String key, String value)868     static boolean setXattr(String path, String key, String value) {
869         try (ParcelFileDescriptor pfd = ParcelFileDescriptor.open(new File(path),
870                 ParcelFileDescriptor.MODE_READ_ONLY)) {
871             // Map id value to xattr key
872             Os.setxattr(path, key, value.getBytes(), 0);
873             Os.fsync(pfd.getFileDescriptor());
874             Log.d(TAG, String.format("xattr set to %s for key:%s on path: %s.", value, key, path));
875             return true;
876         } catch (Exception e) {
877             Log.e(TAG, String.format(Locale.ROOT, "Failed to set xattr:%s to %s for path: %s.", key,
878                     value, path), e);
879             return false;
880         }
881     }
882 
883     /**
884      * Deletes xattr with given key on given path. Becomes a no-op when xattr is not present.
885      */
removeXattr(String path, String key)886     static boolean removeXattr(String path, String key) {
887         try (ParcelFileDescriptor pfd = ParcelFileDescriptor.open(new File(path),
888                 ParcelFileDescriptor.MODE_READ_ONLY)) {
889             Os.removexattr(path, key);
890             Os.fsync(pfd.getFileDescriptor());
891             Log.d(TAG, String.format("xattr key:%s removed on path: %s.", key, path));
892             return true;
893         } catch (Exception e) {
894             if (e instanceof ErrnoException) {
895                 ErrnoException exception = (ErrnoException) e;
896                 if (exception.errno == OsConstants.ENODATA) {
897                     Log.w(TAG, String.format(Locale.ROOT,
898                             "xattr:%s is not removed as it is not found on path: %s.", key, path));
899                     return true;
900                 }
901             }
902 
903             Log.e(TAG, String.format(Locale.ROOT, "Failed to remove xattr:%s for path: %s.", key,
904                     path), e);
905             return false;
906         }
907     }
908 
909     /**
910      * Lists xattrs of given path.
911      */
listXattr(String path)912     static List<String> listXattr(String path) {
913         try {
914             return Arrays.asList(Os.listxattr(path));
915         } catch (Exception e) {
916             Log.e(TAG, "Exception in reading xattrs on path: " + path, e);
917             return new ArrayList<>();
918         }
919     }
920 
insertDataInDatabase(SQLiteDatabase db, BackupIdRow row, String filePath, String volumeName)921     private boolean insertDataInDatabase(SQLiteDatabase db, BackupIdRow row, String filePath,
922             String volumeName) {
923         final ContentValues values = createValuesFromFileRow(row, filePath, volumeName);
924         return db.insertWithOnConflict("files", null, values,
925                 SQLiteDatabase.CONFLICT_REPLACE) != -1;
926     }
927 
createValuesFromFileRow(BackupIdRow row, String filePath, String volumeName)928     private ContentValues createValuesFromFileRow(BackupIdRow row, String filePath,
929             String volumeName) {
930         ContentValues values = new ContentValues();
931         values.put(MediaStore.Files.FileColumns._ID, row.getId());
932         values.put(MediaStore.Files.FileColumns.IS_FAVORITE, row.getIsFavorite());
933         values.put(MediaStore.Files.FileColumns.IS_PENDING, row.getIsPending());
934         values.put(MediaStore.Files.FileColumns.IS_TRASHED, row.getIsTrashed());
935         values.put(MediaStore.Files.FileColumns.DATA, filePath);
936         values.put(MediaStore.Files.FileColumns.VOLUME_NAME, volumeName);
937         values.put(MediaStore.Files.FileColumns._USER_ID, row.getUserId());
938         values.put(MediaStore.Files.FileColumns.MEDIA_TYPE, row.getMediaType());
939         if (!StringUtils.isNullOrEmpty(row.getDateExpires())) {
940             values.put(MediaStore.Files.FileColumns.DATE_EXPIRES,
941                     Long.valueOf(row.getDateExpires()));
942         }
943         if (row.getOwnerPackageId() >= 0) {
944             Pair<String, Integer> ownerPackageNameAndUidPair = getOwnerPackageNameAndUidPair(
945                     row.getOwnerPackageId());
946             if (ownerPackageNameAndUidPair.first != null) {
947                 values.put(MediaStore.Files.FileColumns.OWNER_PACKAGE_NAME,
948                         ownerPackageNameAndUidPair.first);
949             }
950             if (ownerPackageNameAndUidPair.second != null) {
951                 values.put(MediaStore.Files.FileColumns._USER_ID,
952                         ownerPackageNameAndUidPair.second);
953             }
954         }
955 
956         return values;
957     }
958 
getOwnerPackageNameAndUidPair(int ownerPackageId)959     private Pair<String, Integer> getOwnerPackageNameAndUidPair(int ownerPackageId) {
960         if (sOwnerIdRelationMap == null) {
961             try {
962                 sOwnerIdRelationMap = readOwnerIdRelationsFromLevelDb();
963                 Log.v(TAG, "Cached owner id map");
964             } catch (IOException e) {
965                 Log.e(TAG, "Failure in reading owner details for owner id:" + ownerPackageId, e);
966                 return Pair.create(null, null);
967             }
968         }
969 
970         if (sOwnerIdRelationMap.containsKey(String.valueOf(ownerPackageId))) {
971             return getPackageNameAndUserId(sOwnerIdRelationMap.get(String.valueOf(ownerPackageId)));
972         }
973 
974         return Pair.create(null, null);
975     }
976 
readOwnerIdRelationsFromLevelDb()977     private Map<String, String> readOwnerIdRelationsFromLevelDb() throws IOException {
978         return getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH).readOwnerIdRelations();
979     }
980 
readOwnerPackageName(String ownerId)981     String readOwnerPackageName(String ownerId) throws IOException {
982         Map<String, String> ownerIdRelationMap = readOwnerIdRelationsFromLevelDb();
983         if (ownerIdRelationMap.containsKey(String.valueOf(ownerId))) {
984             return getPackageNameAndUserId(ownerIdRelationMap.get(ownerId)).first;
985         }
986 
987         return null;
988     }
989 
markPublicVolumesRecovery()990     void markPublicVolumesRecovery() {
991         try {
992             File recoveryDir = new File(LOWER_FS_RECOVERY_DIRECTORY_PATH);
993             for (File levelDbFile : recoveryDir.listFiles()) {
994                 if (!(LEVEL_DB_PREFIX + MediaStore.VOLUME_EXTERNAL_PRIMARY)
995                         .equalsIgnoreCase(levelDbFile.getName())
996                         && !(LEVEL_DB_PREFIX + MediaStore.VOLUME_INTERNAL)
997                         .equalsIgnoreCase(levelDbFile.getName())
998                         && !(OWNERSHIP_TABLE_NAME
999                         .equalsIgnoreCase(levelDbFile.getName()))) {
1000                     setXattr(levelDbFile.getAbsolutePath(), PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY,
1001                             String.valueOf(true));
1002                 }
1003             }
1004         } catch (Exception e) {
1005             Log.e(TAG, "Exception while marking public volumes for recovery", e);
1006         }
1007     }
1008 
isPublicVolumeMarkedForRecovery(String volumeName)1009     boolean isPublicVolumeMarkedForRecovery(String volumeName) {
1010         String filePath = LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX
1011                 + volumeName.toLowerCase(Locale.ROOT);
1012         Optional<String> flag = getXattr(filePath, PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY);
1013         Log.d(TAG, "Public volume is " + (flag.isPresent() ? "" : "not ")
1014                 + "marked for recovery, volume: " + volumeName);
1015         return flag.isPresent();
1016     }
1017 
removePublicVolumeRecoveryFlag(String volumeName)1018     void removePublicVolumeRecoveryFlag(String volumeName) {
1019         String filePath = LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX
1020                 + volumeName.toLowerCase(Locale.ROOT);
1021         removeXattr(filePath, PUBLIC_VOLUME_RECOVERY_FLAG_XATTR_KEY);
1022     }
1023 
recoverData(SQLiteDatabase db, String volumeName)1024     void recoverData(SQLiteDatabase db, String volumeName) throws Exception {
1025         long rowsRecovered = 0, dirtyRowsCount = 0, insertionFailuresCount = 0,
1026                 totalLevelDbRows = 0;
1027         final long startTime = SystemClock.elapsedRealtime();
1028         try {
1029             final String fuseFilePath = getFuseFilePathFromVolumeName(volumeName);
1030             // Wait for external primary to be attached as we use same thread for internal volume.
1031             // Maximum wait for 20s
1032             getFuseDaemonForFileWithWait(new File(fuseFilePath));
1033             if (!isBackupPresent(volumeName)) {
1034                 throw new FileNotFoundException("Backup file not found for " + volumeName);
1035             }
1036 
1037             Log.d(TAG, "Backup is present for " + volumeName);
1038             try {
1039                 waitForVolumeToBeAttached(MediaStore.VOLUME_EXTERNAL_PRIMARY);
1040             } catch (Exception e) {
1041                 throw new IllegalStateException(
1042                         "Volume not attached in given time. Cannot recover data.", e);
1043             }
1044 
1045             String[] backedUpFilePaths;
1046             String lastReadValue = "";
1047             while (true) {
1048                 backedUpFilePaths = readBackedUpFilePaths(volumeName, lastReadValue,
1049                         LEVEL_DB_READ_LIMIT);
1050                 if (backedUpFilePaths == null || backedUpFilePaths.length == 0) {
1051                     break;
1052                 }
1053                 totalLevelDbRows += backedUpFilePaths.length;
1054 
1055                 // Reset cached owner id relation map
1056                 sOwnerIdRelationMap = null;
1057                 for (String filePath : backedUpFilePaths) {
1058                     Optional<BackupIdRow> fileRow = readDataFromBackup(volumeName, filePath);
1059                     if (fileRow.isPresent()) {
1060                         if (fileRow.get().getIsDirty()) {
1061                             dirtyRowsCount++;
1062                             continue;
1063                         }
1064 
1065                         if (insertDataInDatabase(db, fileRow.get(), filePath, volumeName)) {
1066                             rowsRecovered++;
1067                         } else {
1068                             insertionFailuresCount++;
1069                         }
1070                     }
1071                 }
1072 
1073                 // Read less rows than expected
1074                 if (backedUpFilePaths.length < LEVEL_DB_READ_LIMIT) {
1075                     break;
1076                 }
1077                 lastReadValue = backedUpFilePaths[backedUpFilePaths.length - 1];
1078             }
1079             long recoveryTime = SystemClock.elapsedRealtime() - startTime;
1080             publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount,
1081                     totalLevelDbRows, insertionFailuresCount,
1082                     MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__SUCCESS);
1083             Log.i(TAG, String.format(Locale.ROOT, "%d rows recovered for volume: %s."
1084                             + " Total rows in levelDB: %d.", rowsRecovered, volumeName,
1085                     totalLevelDbRows));
1086             Log.i(TAG, String.format(Locale.ROOT, "Recovery time: %d ms", recoveryTime));
1087         } catch (TimeoutException e) {
1088             long recoveryTime = SystemClock.elapsedRealtime() - startTime;
1089             publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount,
1090                     totalLevelDbRows, insertionFailuresCount,
1091                     MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__FUSE_DAEMON_TIMEOUT);
1092             throw e;
1093         } catch (FileNotFoundException e) {
1094             long recoveryTime = SystemClock.elapsedRealtime() - startTime;
1095             publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount,
1096                     totalLevelDbRows, insertionFailuresCount,
1097                     MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__BACKUP_MISSING);
1098             throw e;
1099         } catch (IOException e) {
1100             long recoveryTime = SystemClock.elapsedRealtime() - startTime;
1101             publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount,
1102                     totalLevelDbRows, insertionFailuresCount,
1103                     MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__GET_BACKUP_DATA_FAILURE);
1104             throw e;
1105         } catch (IllegalStateException e) {
1106             long recoveryTime = SystemClock.elapsedRealtime() - startTime;
1107             publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount,
1108                     totalLevelDbRows, insertionFailuresCount,
1109                     MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__VOLUME_NOT_ATTACHED);
1110             throw e;
1111         } catch (Exception e) {
1112             long recoveryTime = SystemClock.elapsedRealtime() - startTime;
1113             publishRecoveryMetric(volumeName, recoveryTime, rowsRecovered, dirtyRowsCount,
1114                     totalLevelDbRows, insertionFailuresCount,
1115                     MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__STATUS__OTHER_ERROR);
1116             throw e;
1117         }
1118     }
1119 
resetLastBackedUpGenerationNumber(String volumeName)1120     void resetLastBackedUpGenerationNumber(String volumeName) {
1121         // Resetting generation number
1122         setXattr(LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX + volumeName,
1123                 LAST_BACKEDUP_GENERATION_XATTR_KEY, String.valueOf(0));
1124         Log.v(TAG, "Leveldb Last backed generation number reset done to 0 for " + volumeName);
1125     }
1126 
isBackupPresent(String volumeName)1127     boolean isBackupPresent(String volumeName) {
1128         if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) {
1129             return new File(INTERNAL_VOLUME_LOWER_FS_BACKUP_PATH).exists();
1130         } else if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)) {
1131             return new File(EXTERNAL_PRIMARY_VOLUME_LOWER_FS_BACKUP_PATH).exists();
1132         } else if (!Strings.isNullOrEmpty(volumeName)) {
1133             return new File(LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-" + volumeName)
1134                     .exists();
1135         }
1136 
1137         return false;
1138     }
1139 
waitForVolumeToBeAttached(String volumeName)1140     void waitForVolumeToBeAttached(String volumeName) throws TimeoutException {
1141         long time = 0;
1142         // Wait of 10 seconds
1143         long waitTimeInMilliseconds = 10000;
1144         // Poll every 100 milliseconds
1145         long pollTime = 100;
1146         while (time <= waitTimeInMilliseconds) {
1147             if (mSetupCompleteVolumes.contains(volumeName)) {
1148                 Log.i(TAG, "Found " + volumeName + " volume attached.");
1149                 return;
1150             }
1151 
1152             SystemClock.sleep(pollTime);
1153             time += pollTime;
1154         }
1155         throw new TimeoutException("Timed out waiting for " + volumeName + " setup");
1156     }
1157 
getFuseDaemonForFileWithWait(File fuseFilePath)1158     FuseDaemon getFuseDaemonForFileWithWait(File fuseFilePath)
1159             throws FileNotFoundException, TimeoutException {
1160         pollForExternalStorageMountedState();
1161         return MediaProvider.getFuseDaemonForFileWithWait(fuseFilePath, mVolumeCache,
1162                 WAIT_TIME_20_SECONDS_IN_MILLIS);
1163     }
1164 
setStableUrisGlobalFlag(String volumeName, boolean isEnabled)1165     void setStableUrisGlobalFlag(String volumeName, boolean isEnabled) {
1166         if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) {
1167             mIsStableUriEnabledForInternal = isEnabled;
1168         } else if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)) {
1169             mIsStableUriEnabledForExternal = isEnabled;
1170         } else {
1171             mIsStableUrisEnabledForPublic = isEnabled;
1172         }
1173     }
1174 
getVolumeNameForStatsLog(String volumeName)1175     private int getVolumeNameForStatsLog(String volumeName) {
1176         if (volumeName.equalsIgnoreCase(MediaStore.VOLUME_INTERNAL)) {
1177             return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__INTERNAL;
1178         } else if (volumeName.equalsIgnoreCase(MediaStore.VOLUME_EXTERNAL_PRIMARY)) {
1179             return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY;
1180         }
1181 
1182         return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC;
1183     }
1184 
getFuseFilePathFromVolumeName(String volumeName)1185     private static String getFuseFilePathFromVolumeName(String volumeName) {
1186         if (Strings.isNullOrEmpty(volumeName)) {
1187             // Returning EXTERNAL_PRIMARY_ROOT_PATH to avoid any regressions
1188             Log.e(TAG, "Trying to get a Fuse Daemon for volume name = " + volumeName);
1189             return EXTERNAL_PRIMARY_ROOT_PATH;
1190         }
1191         switch (volumeName) {
1192             case MediaStore.VOLUME_INTERNAL:
1193             case MediaStore.VOLUME_EXTERNAL_PRIMARY:
1194                 return EXTERNAL_PRIMARY_ROOT_PATH;
1195             default:
1196                 return "/storage/" + volumeName.toUpperCase(Locale.ROOT);
1197         }
1198     }
1199 
1200     /**
1201      * Returns list of backed up files from external storage.
1202      */
getBackupFiles()1203     List<File> getBackupFiles() {
1204         return Arrays.asList(new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).listFiles());
1205     }
1206 
1207     /**
1208      * Updates backup in external storage to the latest values. Deletes backup of old file path if
1209      * file path has changed.
1210      */
updateBackup(DatabaseHelper helper, FileRow oldRow, FileRow newRow)1211     void updateBackup(DatabaseHelper helper, FileRow oldRow, FileRow newRow) {
1212         if (!isBackupUpdateAllowed(helper, newRow.getVolumeName())) {
1213             return;
1214         }
1215 
1216         final FuseDaemon fuseDaemonExternalPrimary;
1217         try {
1218             fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait(
1219                     new File(EXTERNAL_PRIMARY_ROOT_PATH));
1220         } catch (Exception e) {
1221             Log.e(TAG,
1222                     "Fuse Daemon not found for primary external storage, skipping update of "
1223                             + newRow.getPath(), e);
1224             return;
1225         }
1226 
1227         final FuseDaemon fuseDaemonPublicVolume;
1228         if (!isInternalOrExternalPrimary(newRow.getVolumeName())) {
1229             try {
1230                 fuseDaemonPublicVolume = getFuseDaemonForFileWithWait(new File(
1231                         getFuseFilePathFromVolumeName(newRow.getVolumeName())));
1232             } catch (Exception e) {
1233                 Log.e(TAG,
1234                         "Error occurred while retrieving the Fuse Daemon for "
1235                                 + getFuseFilePathFromVolumeName(newRow.getVolumeName())
1236                                 + ", skipping update of " + newRow.getPath(),
1237                         e);
1238                 return;
1239             }
1240         } else {
1241             fuseDaemonPublicVolume = null;
1242         }
1243 
1244         mLevelDbUpdateLock.lock();
1245         try {
1246             helper.runWithTransaction((db) -> {
1247                 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, "_id=?",
1248                         new String[]{String.valueOf(newRow.getId())}, null, null, null,
1249                         null, null)) {
1250                     if (c.moveToFirst()) {
1251                         backupDataValues(fuseDaemonExternalPrimary, fuseDaemonPublicVolume, c);
1252                         String newPath = c.getString(1);
1253                         if (oldRow.getPath() != null && !oldRow.getPath().equalsIgnoreCase(
1254                                 newPath)) {
1255                             // If file path has changed, update leveldb backup to delete old path.
1256                             deleteFromDbBackup(helper, oldRow);
1257                             Log.v(TAG, "Deleted backup of old file path: "
1258                                     + oldRow.getPath());
1259                         }
1260                     }
1261                 } catch (Exception e) {
1262                     Log.e(TAG, "Failure in updating row in external storage backup.", e);
1263                 }
1264                 return null;
1265             });
1266         } catch (Exception e) {
1267             Log.e(TAG, "Failure in updating row in external storage backup.", e);
1268         } finally {
1269             mLevelDbUpdateLock.unlock();
1270         }
1271     }
1272 
1273     /**
1274      * Removes database recovery data for given user id. This is done when a user is removed.
1275      */
removeRecoveryDataForUserId(int removedUserId)1276     void removeRecoveryDataForUserId(int removedUserId) {
1277         String removeduserIdString = String.valueOf(removedUserId);
1278         removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH,
1279                 INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.concat(
1280                         removeduserIdString));
1281         removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH,
1282                 EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.concat(
1283                         removeduserIdString));
1284         removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH,
1285                 INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.concat(removeduserIdString));
1286         removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH,
1287                 EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.concat(removeduserIdString));
1288         Log.v(TAG, "Removed recovery data for user id: " + removedUserId);
1289     }
1290 
1291     /**
1292      * Removes database recovery data for obsolete user id. It accepts list of valid/active users
1293      * and removes the recovery data for ones not present in this list.
1294      * This is done during an idle maintenance.
1295      */
removeRecoveryDataExceptValidUsers(List<String> validUsers)1296     void removeRecoveryDataExceptValidUsers(List<String> validUsers) {
1297         List<String> xattrList = listXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH);
1298         Log.i(TAG, "Xattr list is " + xattrList);
1299         if (xattrList.isEmpty()) {
1300             return;
1301         }
1302 
1303         Log.i(TAG, "Valid users list is " + validUsers);
1304         List<String> invalidUsers = getInvalidUsersList(xattrList, validUsers);
1305         Log.i(TAG, "Invalid users list is " + invalidUsers);
1306         for (String userIdToBeRemoved : invalidUsers) {
1307             if (userIdToBeRemoved != null && !userIdToBeRemoved.trim().isEmpty()) {
1308                 removeRecoveryDataForUserId(Integer.parseInt(userIdToBeRemoved));
1309             }
1310         }
1311     }
1312 
publishRecoveryMetric(String volumeName, long recoveryTime, long rowsRecovered, long dirtyRowsCount, long totalLevelDbRows, long insertionFailureCount, int status)1313     private void publishRecoveryMetric(String volumeName, long recoveryTime, long rowsRecovered,
1314             long dirtyRowsCount, long totalLevelDbRows, long insertionFailureCount, int status) {
1315         MediaProviderStatsLog.write(
1316                 MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED,
1317                 getVolumeNameForStatsLog(volumeName), recoveryTime, rowsRecovered,
1318                 dirtyRowsCount, totalLevelDbRows, insertionFailureCount, status);
1319     }
1320 
getInvalidUsersList(List<String> recoveryData, List<String> validUsers)1321     static List<String> getInvalidUsersList(List<String> recoveryData,
1322             List<String> validUsers) {
1323         Set<String> presentUserIdsAsXattr = new HashSet<>();
1324         for (String xattr : recoveryData) {
1325             if (xattr.startsWith(INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX)) {
1326                 presentUserIdsAsXattr.add(
1327                         xattr.substring(INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.length()));
1328             } else if (xattr.startsWith(EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX)) {
1329                 presentUserIdsAsXattr.add(
1330                         xattr.substring(EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.length()));
1331             } else if (xattr.startsWith(INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX)) {
1332                 presentUserIdsAsXattr.add(
1333                         xattr.substring(INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.length()));
1334             } else if (xattr.startsWith(EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX)) {
1335                 presentUserIdsAsXattr.add(
1336                         xattr.substring(EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.length()));
1337             }
1338         }
1339         // Remove valid users
1340         validUsers.forEach(presentUserIdsAsXattr::remove);
1341         return presentUserIdsAsXattr.stream().collect(Collectors.toList());
1342     }
1343 
pollForExternalStorageMountedState()1344     private static void pollForExternalStorageMountedState() throws TimeoutException {
1345         final File target = Environment.getExternalStorageDirectory();
1346         for (int i = 0; i < WAIT_TIME_20_SECONDS_IN_MILLIS / 100; i++) {
1347             if (Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState(target))) {
1348                 return;
1349             }
1350             Log.v(TAG, "Waiting for external storage...");
1351             SystemClock.sleep(100);
1352         }
1353         throw new TimeoutException("Timed out while waiting for ExternalStorageState "
1354                 + "to be MEDIA_MOUNTED");
1355     }
1356 
1357     /**
1358      * Performs actions to be taken on volume unmount.
1359      * @param volumeName name of volume which is detached
1360      */
onDetachVolume(String volumeName)1361     public void onDetachVolume(String volumeName) {
1362         if (mSetupCompleteVolumes.contains(volumeName)) {
1363             mSetupCompleteVolumes.remove(volumeName);
1364             Log.v(TAG,
1365                     "Removed leveldb connections from in memory setup cache for volume:"
1366                             + volumeName);
1367         }
1368     }
1369 }
1370