1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server; 18 19 import static android.provider.DeviceConfig.Properties; 20 21 import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo; 22 23 import android.annotation.IntDef; 24 import android.annotation.NonNull; 25 import android.annotation.Nullable; 26 import android.content.ContentResolver; 27 import android.content.Context; 28 import android.content.pm.ApplicationInfo; 29 import android.content.pm.PackageManager; 30 import android.content.pm.VersionedPackage; 31 import android.crashrecovery.flags.Flags; 32 import android.os.Build; 33 import android.os.Environment; 34 import android.os.FileUtils; 35 import android.os.PowerManager; 36 import android.os.RecoverySystem; 37 import android.os.SystemClock; 38 import android.os.SystemProperties; 39 import android.os.UserHandle; 40 import android.provider.DeviceConfig; 41 import android.provider.Settings; 42 import android.sysprop.CrashRecoveryProperties; 43 import android.text.TextUtils; 44 import android.util.ArraySet; 45 import android.util.Log; 46 import android.util.Slog; 47 48 import com.android.internal.annotations.GuardedBy; 49 import com.android.internal.annotations.VisibleForTesting; 50 import com.android.internal.util.ArrayUtils; 51 import com.android.server.PackageWatchdog.FailureReasons; 52 import com.android.server.PackageWatchdog.PackageHealthObserver; 53 import com.android.server.PackageWatchdog.PackageHealthObserverImpact; 54 import com.android.server.am.SettingsToPropertiesMapper; 55 import com.android.server.crashrecovery.proto.CrashRecoveryStatsLog; 56 57 import java.io.File; 58 import java.lang.annotation.Retention; 59 import java.lang.annotation.RetentionPolicy; 60 import java.util.ArrayList; 61 import java.util.Arrays; 62 import java.util.HashMap; 63 import java.util.HashSet; 64 import java.util.Iterator; 65 import java.util.List; 66 import java.util.Map; 67 import java.util.Set; 68 import java.util.concurrent.Executors; 69 import java.util.concurrent.TimeUnit; 70 71 /** 72 * Utilities to help rescue the system from crash loops. Callers are expected to 73 * report boot events and persistent app crashes, and if they happen frequently 74 * enough this class will slowly escalate through several rescue operations 75 * before finally rebooting and prompting the user if they want to wipe data as 76 * a last resort. 77 * 78 * @hide 79 */ 80 public class RescueParty { 81 @VisibleForTesting 82 static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue"; 83 @VisibleForTesting 84 static final int LEVEL_NONE = 0; 85 @VisibleForTesting 86 static final int LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 1; 87 @VisibleForTesting 88 static final int LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 2; 89 @VisibleForTesting 90 static final int LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 3; 91 @VisibleForTesting 92 static final int LEVEL_WARM_REBOOT = 4; 93 @VisibleForTesting 94 static final int LEVEL_FACTORY_RESET = 5; 95 @VisibleForTesting 96 static final int RESCUE_LEVEL_NONE = 0; 97 @VisibleForTesting 98 static final int RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET = 1; 99 @VisibleForTesting 100 static final int RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET = 2; 101 @VisibleForTesting 102 static final int RESCUE_LEVEL_WARM_REBOOT = 3; 103 @VisibleForTesting 104 static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 4; 105 @VisibleForTesting 106 static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 5; 107 @VisibleForTesting 108 static final int RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 6; 109 @VisibleForTesting 110 static final int RESCUE_LEVEL_FACTORY_RESET = 7; 111 112 @IntDef(prefix = { "RESCUE_LEVEL_" }, value = { 113 RESCUE_LEVEL_NONE, 114 RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET, 115 RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET, 116 RESCUE_LEVEL_WARM_REBOOT, 117 RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS, 118 RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES, 119 RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS, 120 RESCUE_LEVEL_FACTORY_RESET 121 }) 122 @Retention(RetentionPolicy.SOURCE) 123 @interface RescueLevels {} 124 125 @VisibleForTesting 126 static final String RESCUE_NON_REBOOT_LEVEL_LIMIT = "persist.sys.rescue_non_reboot_level_limit"; 127 @VisibleForTesting 128 static final int DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT = RESCUE_LEVEL_WARM_REBOOT - 1; 129 @VisibleForTesting 130 static final String TAG = "RescueParty"; 131 @VisibleForTesting 132 static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2); 133 @VisibleForTesting 134 static final int DEVICE_CONFIG_RESET_MODE = Settings.RESET_MODE_TRUSTED_DEFAULTS; 135 // The DeviceConfig namespace containing all RescueParty switches. 136 @VisibleForTesting 137 static final String NAMESPACE_CONFIGURATION = "configuration"; 138 @VisibleForTesting 139 static final String NAMESPACE_TO_PACKAGE_MAPPING_FLAG = 140 "namespace_to_package_mapping"; 141 @VisibleForTesting 142 static final long DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN = 1440; 143 144 private static final String NAME = "rescue-party-observer"; 145 146 private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue"; 147 private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device"; 148 private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG = 149 "persist.device_config.configuration.disable_rescue_party"; 150 private static final String PROP_DISABLE_FACTORY_RESET_FLAG = 151 "persist.device_config.configuration.disable_rescue_party_factory_reset"; 152 private static final String PROP_THROTTLE_DURATION_MIN_FLAG = 153 "persist.device_config.configuration.rescue_party_throttle_duration_min"; 154 155 private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT 156 | ApplicationInfo.FLAG_SYSTEM; 157 158 /** Register the Rescue Party observer as a Package Watchdog health observer */ registerHealthObserver(Context context)159 public static void registerHealthObserver(Context context) { 160 PackageWatchdog.getInstance(context).registerHealthObserver( 161 RescuePartyObserver.getInstance(context)); 162 } 163 isDisabled()164 private static boolean isDisabled() { 165 // Check if we're explicitly enabled for testing 166 if (SystemProperties.getBoolean(PROP_ENABLE_RESCUE, false)) { 167 return false; 168 } 169 170 // We're disabled if the DeviceConfig disable flag is set to true. 171 // This is in case that an emergency rollback of the feature is needed. 172 if (SystemProperties.getBoolean(PROP_DEVICE_CONFIG_DISABLE_FLAG, false)) { 173 Slog.v(TAG, "Disabled because of DeviceConfig flag"); 174 return true; 175 } 176 177 // We're disabled on all engineering devices 178 if (Build.TYPE.equals("eng")) { 179 Slog.v(TAG, "Disabled because of eng build"); 180 return true; 181 } 182 183 // We're disabled on userdebug devices connected over USB, since that's 184 // a decent signal that someone is actively trying to debug the device, 185 // or that it's in a lab environment. 186 if (Build.TYPE.equals("userdebug") && isUsbActive()) { 187 Slog.v(TAG, "Disabled because of active USB connection"); 188 return true; 189 } 190 191 // One last-ditch check 192 if (SystemProperties.getBoolean(PROP_DISABLE_RESCUE, false)) { 193 Slog.v(TAG, "Disabled because of manual property"); 194 return true; 195 } 196 197 return false; 198 } 199 200 /** 201 * Check if we're currently attempting to reboot for a factory reset. This method must 202 * return true if RescueParty tries to reboot early during a boot loop, since the device 203 * will not be fully booted at this time. 204 */ isRecoveryTriggeredReboot()205 public static boolean isRecoveryTriggeredReboot() { 206 return isFactoryResetPropertySet() || isRebootPropertySet(); 207 } 208 isFactoryResetPropertySet()209 static boolean isFactoryResetPropertySet() { 210 return CrashRecoveryProperties.attemptingFactoryReset().orElse(false); 211 } 212 isRebootPropertySet()213 static boolean isRebootPropertySet() { 214 return CrashRecoveryProperties.attemptingReboot().orElse(false); 215 } 216 getLastFactoryResetTimeMs()217 protected static long getLastFactoryResetTimeMs() { 218 return CrashRecoveryProperties.lastFactoryResetTimeMs().orElse(0L); 219 } 220 getMaxRescueLevelAttempted()221 protected static int getMaxRescueLevelAttempted() { 222 return CrashRecoveryProperties.maxRescueLevelAttempted().orElse(LEVEL_NONE); 223 } 224 setFactoryResetProperty(boolean value)225 protected static void setFactoryResetProperty(boolean value) { 226 CrashRecoveryProperties.attemptingFactoryReset(value); 227 } setRebootProperty(boolean value)228 protected static void setRebootProperty(boolean value) { 229 CrashRecoveryProperties.attemptingReboot(value); 230 } 231 setLastFactoryResetTimeMs(long value)232 protected static void setLastFactoryResetTimeMs(long value) { 233 CrashRecoveryProperties.lastFactoryResetTimeMs(value); 234 } 235 setMaxRescueLevelAttempted(int level)236 protected static void setMaxRescueLevelAttempted(int level) { 237 CrashRecoveryProperties.maxRescueLevelAttempted(level); 238 } 239 240 /** 241 * Called when {@code SettingsProvider} has been published, which is a good 242 * opportunity to reset any settings depending on our rescue level. 243 */ onSettingsProviderPublished(Context context)244 public static void onSettingsProviderPublished(Context context) { 245 handleNativeRescuePartyResets(); 246 ContentResolver contentResolver = context.getContentResolver(); 247 DeviceConfig.setMonitorCallback( 248 contentResolver, 249 Executors.newSingleThreadExecutor(), 250 new RescuePartyMonitorCallback(context)); 251 } 252 253 254 /** 255 * Called when {@code RollbackManager} performs Mainline module rollbacks, 256 * to avoid rolled back modules consuming flag values only expected to work 257 * on modules of newer versions. 258 */ resetDeviceConfigForPackages(List<String> packageNames)259 public static void resetDeviceConfigForPackages(List<String> packageNames) { 260 if (packageNames == null) { 261 return; 262 } 263 Set<String> namespacesToReset = new ArraySet<String>(); 264 Iterator<String> it = packageNames.iterator(); 265 RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstanceIfCreated(); 266 // Get runtime package to namespace mapping if created. 267 if (rescuePartyObserver != null) { 268 while (it.hasNext()) { 269 String packageName = it.next(); 270 Set<String> runtimeAffectedNamespaces = 271 rescuePartyObserver.getAffectedNamespaceSet(packageName); 272 if (runtimeAffectedNamespaces != null) { 273 namespacesToReset.addAll(runtimeAffectedNamespaces); 274 } 275 } 276 } 277 // Get preset package to namespace mapping if created. 278 Set<String> presetAffectedNamespaces = getPresetNamespacesForPackages( 279 packageNames); 280 if (presetAffectedNamespaces != null) { 281 namespacesToReset.addAll(presetAffectedNamespaces); 282 } 283 284 // Clear flags under the namespaces mapped to these packages. 285 // Using setProperties since DeviceConfig.resetToDefaults bans the current flag set. 286 Iterator<String> namespaceIt = namespacesToReset.iterator(); 287 while (namespaceIt.hasNext()) { 288 String namespaceToReset = namespaceIt.next(); 289 Properties properties = new Properties.Builder(namespaceToReset).build(); 290 try { 291 if (!DeviceConfig.setProperties(properties)) { 292 logCriticalInfo(Log.ERROR, "Failed to clear properties under " 293 + namespaceToReset 294 + ". Running `device_config get_sync_disabled_for_tests` will confirm" 295 + " if config-bulk-update is enabled."); 296 } 297 } catch (DeviceConfig.BadConfigException exception) { 298 logCriticalInfo(Log.WARN, "namespace " + namespaceToReset 299 + " is already banned, skip reset."); 300 } 301 } 302 } 303 getPresetNamespacesForPackages(List<String> packageNames)304 private static Set<String> getPresetNamespacesForPackages(List<String> packageNames) { 305 Set<String> resultSet = new ArraySet<String>(); 306 try { 307 String flagVal = DeviceConfig.getString(NAMESPACE_CONFIGURATION, 308 NAMESPACE_TO_PACKAGE_MAPPING_FLAG, ""); 309 String[] mappingEntries = flagVal.split(","); 310 for (int i = 0; i < mappingEntries.length; i++) { 311 if (TextUtils.isEmpty(mappingEntries[i])) { 312 continue; 313 } 314 String[] splittedEntry = mappingEntries[i].split(":"); 315 if (splittedEntry.length != 2) { 316 throw new RuntimeException("Invalid mapping entry: " + mappingEntries[i]); 317 } 318 String namespace = splittedEntry[0]; 319 String packageName = splittedEntry[1]; 320 321 if (packageNames.contains(packageName)) { 322 resultSet.add(namespace); 323 } 324 } 325 } catch (Exception e) { 326 resultSet.clear(); 327 Slog.e(TAG, "Failed to read preset package to namespaces mapping.", e); 328 } finally { 329 return resultSet; 330 } 331 } 332 333 @VisibleForTesting getElapsedRealtime()334 static long getElapsedRealtime() { 335 return SystemClock.elapsedRealtime(); 336 } 337 338 private static class RescuePartyMonitorCallback implements DeviceConfig.MonitorCallback { 339 Context mContext; 340 RescuePartyMonitorCallback(Context context)341 RescuePartyMonitorCallback(Context context) { 342 this.mContext = context; 343 } 344 onNamespaceUpdate(@onNull String updatedNamespace)345 public void onNamespaceUpdate(@NonNull String updatedNamespace) { 346 startObservingPackages(mContext, updatedNamespace); 347 } 348 onDeviceConfigAccess(@onNull String callingPackage, @NonNull String namespace)349 public void onDeviceConfigAccess(@NonNull String callingPackage, 350 @NonNull String namespace) { 351 RescuePartyObserver.getInstance(mContext).recordDeviceConfigAccess( 352 callingPackage, 353 namespace); 354 } 355 } 356 startObservingPackages(Context context, @NonNull String updatedNamespace)357 private static void startObservingPackages(Context context, @NonNull String updatedNamespace) { 358 RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context); 359 Set<String> callingPackages = rescuePartyObserver.getCallingPackagesSet(updatedNamespace); 360 if (callingPackages == null) { 361 return; 362 } 363 List<String> callingPackageList = new ArrayList<>(); 364 callingPackageList.addAll(callingPackages); 365 Slog.i(TAG, "Starting to observe: " + callingPackageList + ", updated namespace: " 366 + updatedNamespace); 367 PackageWatchdog.getInstance(context).startObservingHealth( 368 rescuePartyObserver, 369 callingPackageList, 370 DEFAULT_OBSERVING_DURATION_MS); 371 } 372 handleNativeRescuePartyResets()373 private static void handleNativeRescuePartyResets() { 374 if (SettingsToPropertiesMapper.isNativeFlagsResetPerformed()) { 375 String[] resetNativeCategories = SettingsToPropertiesMapper.getResetNativeCategories(); 376 for (int i = 0; i < resetNativeCategories.length; i++) { 377 // Don't let RescueParty reset the namespace for RescueParty switches. 378 if (NAMESPACE_CONFIGURATION.equals(resetNativeCategories[i])) { 379 continue; 380 } 381 DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, 382 resetNativeCategories[i]); 383 } 384 } 385 } 386 getMaxRescueLevel(boolean mayPerformReboot)387 private static int getMaxRescueLevel(boolean mayPerformReboot) { 388 if (Flags.recoverabilityDetection()) { 389 if (!mayPerformReboot 390 || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) { 391 return SystemProperties.getInt(RESCUE_NON_REBOOT_LEVEL_LIMIT, 392 DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT); 393 } 394 return RESCUE_LEVEL_FACTORY_RESET; 395 } else { 396 if (!mayPerformReboot 397 || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) { 398 return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS; 399 } 400 return LEVEL_FACTORY_RESET; 401 } 402 } 403 404 /** 405 * Get the rescue level to perform if this is the n-th attempt at mitigating failure. 406 * 407 * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.) 408 * @param mayPerformReboot: whether or not a reboot and factory reset may be performed 409 * for the given failure. 410 * @return the rescue level for the n-th mitigation attempt. 411 */ getRescueLevel(int mitigationCount, boolean mayPerformReboot)412 private static int getRescueLevel(int mitigationCount, boolean mayPerformReboot) { 413 if (mitigationCount == 1) { 414 return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS; 415 } else if (mitigationCount == 2) { 416 return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES; 417 } else if (mitigationCount == 3) { 418 return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS; 419 } else if (mitigationCount == 4) { 420 return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_WARM_REBOOT); 421 } else if (mitigationCount >= 5) { 422 return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_FACTORY_RESET); 423 } else { 424 Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount); 425 return LEVEL_NONE; 426 } 427 } 428 429 /** 430 * Get the rescue level to perform if this is the n-th attempt at mitigating failure. 431 * When failedPackage is null then 1st and 2nd mitigation counts are redundant (scoped and 432 * all device config reset). Behaves as if one mitigation attempt was already done. 433 * 434 * @param mitigationCount the mitigation attempt number (1 = first attempt etc.). 435 * @param mayPerformReboot whether or not a reboot and factory reset may be performed 436 * for the given failure. 437 * @param failedPackage in case of bootloop this is null. 438 * @return the rescue level for the n-th mitigation attempt. 439 */ getRescueLevel(int mitigationCount, boolean mayPerformReboot, @Nullable VersionedPackage failedPackage)440 private static @RescueLevels int getRescueLevel(int mitigationCount, boolean mayPerformReboot, 441 @Nullable VersionedPackage failedPackage) { 442 // Skipping RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET since it's not defined without a failed 443 // package. 444 if (failedPackage == null && mitigationCount > 0) { 445 mitigationCount += 1; 446 } 447 if (mitigationCount == 1) { 448 return RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET; 449 } else if (mitigationCount == 2) { 450 return RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET; 451 } else if (mitigationCount == 3) { 452 return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_WARM_REBOOT); 453 } else if (mitigationCount == 4) { 454 return Math.min(getMaxRescueLevel(mayPerformReboot), 455 RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS); 456 } else if (mitigationCount == 5) { 457 return Math.min(getMaxRescueLevel(mayPerformReboot), 458 RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES); 459 } else if (mitigationCount == 6) { 460 return Math.min(getMaxRescueLevel(mayPerformReboot), 461 RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS); 462 } else if (mitigationCount >= 7) { 463 return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_FACTORY_RESET); 464 } else { 465 return RESCUE_LEVEL_NONE; 466 } 467 } 468 executeRescueLevel(Context context, @Nullable String failedPackage, int level)469 private static void executeRescueLevel(Context context, @Nullable String failedPackage, 470 int level) { 471 Slog.w(TAG, "Attempting rescue level " + levelToString(level)); 472 try { 473 executeRescueLevelInternal(context, level, failedPackage); 474 EventLogTags.writeRescueSuccess(level); 475 String successMsg = "Finished rescue level " + levelToString(level); 476 if (!TextUtils.isEmpty(failedPackage)) { 477 successMsg += " for package " + failedPackage; 478 } 479 logCriticalInfo(Log.DEBUG, successMsg); 480 } catch (Throwable t) { 481 logRescueException(level, failedPackage, t); 482 } 483 } 484 executeRescueLevelInternal(Context context, int level, @Nullable String failedPackage)485 private static void executeRescueLevelInternal(Context context, int level, @Nullable 486 String failedPackage) throws Exception { 487 if (Flags.recoverabilityDetection()) { 488 executeRescueLevelInternalNew(context, level, failedPackage); 489 } else { 490 executeRescueLevelInternalOld(context, level, failedPackage); 491 } 492 } 493 executeRescueLevelInternalOld(Context context, int level, @Nullable String failedPackage)494 private static void executeRescueLevelInternalOld(Context context, int level, @Nullable 495 String failedPackage) throws Exception { 496 497 // Note: DeviceConfig reset is disabled currently and would be enabled using the flag, 498 // after we have figured out a way to reset flags without interfering with trunk 499 // development. TODO: b/287618292 For enabling flag resets. 500 if (!Flags.allowRescuePartyFlagResets() && level <= LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS) { 501 return; 502 } 503 504 CrashRecoveryStatsLog.write(CrashRecoveryStatsLog.RESCUE_PARTY_RESET_REPORTED, 505 level, levelToString(level)); 506 // Try our best to reset all settings possible, and once finished 507 // rethrow any exception that we encountered 508 Exception res = null; 509 switch (level) { 510 case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: 511 try { 512 resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS, 513 level); 514 } catch (Exception e) { 515 res = e; 516 } 517 try { 518 resetDeviceConfig(context, /*isScoped=*/true, failedPackage); 519 } catch (Exception e) { 520 res = e; 521 } 522 break; 523 case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: 524 try { 525 resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES, 526 level); 527 } catch (Exception e) { 528 res = e; 529 } 530 try { 531 resetDeviceConfig(context, /*isScoped=*/true, failedPackage); 532 } catch (Exception e) { 533 res = e; 534 } 535 break; 536 case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: 537 try { 538 resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS, 539 level); 540 } catch (Exception e) { 541 res = e; 542 } 543 try { 544 resetDeviceConfig(context, /*isScoped=*/false, failedPackage); 545 } catch (Exception e) { 546 res = e; 547 } 548 break; 549 case LEVEL_WARM_REBOOT: 550 executeWarmReboot(context, level, failedPackage); 551 break; 552 case LEVEL_FACTORY_RESET: 553 // Before the completion of Reboot, if any crash happens then PackageWatchdog 554 // escalates to next level i.e. factory reset, as they happen in separate threads. 555 // Adding a check to prevent factory reset to execute before above reboot completes. 556 // Note: this reboot property is not persistent resets after reboot is completed. 557 if (isRebootPropertySet()) { 558 return; 559 } 560 executeFactoryReset(context, level, failedPackage); 561 break; 562 } 563 564 if (res != null) { 565 throw res; 566 } 567 } 568 executeRescueLevelInternalNew(Context context, @RescueLevels int level, @Nullable String failedPackage)569 private static void executeRescueLevelInternalNew(Context context, @RescueLevels int level, 570 @Nullable String failedPackage) throws Exception { 571 CrashRecoveryStatsLog.write(CrashRecoveryStatsLog.RESCUE_PARTY_RESET_REPORTED, 572 level, levelToString(level)); 573 switch (level) { 574 case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET: 575 // Enable deviceConfig reset behind flag 576 if (Flags.allowRescuePartyFlagResets()) { 577 resetDeviceConfig(context, /*isScoped=*/true, failedPackage); 578 } 579 break; 580 case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET: 581 // Enable deviceConfig reset behind flag 582 if (Flags.allowRescuePartyFlagResets()) { 583 resetDeviceConfig(context, /*isScoped=*/false, failedPackage); 584 } 585 break; 586 case RESCUE_LEVEL_WARM_REBOOT: 587 executeWarmReboot(context, level, failedPackage); 588 break; 589 case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: 590 resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS, level); 591 break; 592 case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: 593 resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES, level); 594 break; 595 case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: 596 resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS, level); 597 break; 598 case RESCUE_LEVEL_FACTORY_RESET: 599 // Before the completion of Reboot, if any crash happens then PackageWatchdog 600 // escalates to next level i.e. factory reset, as they happen in separate threads. 601 // Adding a check to prevent factory reset to execute before above reboot completes. 602 // Note: this reboot property is not persistent resets after reboot is completed. 603 if (isRebootPropertySet()) { 604 return; 605 } 606 executeFactoryReset(context, level, failedPackage); 607 break; 608 } 609 } 610 executeWarmReboot(Context context, int level, @Nullable String failedPackage)611 private static void executeWarmReboot(Context context, int level, 612 @Nullable String failedPackage) { 613 // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog 614 // when device shutting down. 615 setRebootProperty(true); 616 Runnable runnable = () -> { 617 try { 618 PowerManager pm = context.getSystemService(PowerManager.class); 619 if (pm != null) { 620 pm.reboot(TAG); 621 } 622 } catch (Throwable t) { 623 logRescueException(level, failedPackage, t); 624 } 625 }; 626 Thread thread = new Thread(runnable); 627 thread.start(); 628 } 629 executeFactoryReset(Context context, int level, @Nullable String failedPackage)630 private static void executeFactoryReset(Context context, int level, 631 @Nullable String failedPackage) { 632 setFactoryResetProperty(true); 633 long now = System.currentTimeMillis(); 634 setLastFactoryResetTimeMs(now); 635 Runnable runnable = new Runnable() { 636 @Override 637 public void run() { 638 try { 639 RecoverySystem.rebootPromptAndWipeUserData(context, TAG); 640 } catch (Throwable t) { 641 logRescueException(level, failedPackage, t); 642 } 643 } 644 }; 645 Thread thread = new Thread(runnable); 646 thread.start(); 647 } 648 649 getCompleteMessage(Throwable t)650 private static String getCompleteMessage(Throwable t) { 651 final StringBuilder builder = new StringBuilder(); 652 builder.append(t.getMessage()); 653 while ((t = t.getCause()) != null) { 654 builder.append(": ").append(t.getMessage()); 655 } 656 return builder.toString(); 657 } 658 logRescueException(int level, @Nullable String failedPackageName, Throwable t)659 private static void logRescueException(int level, @Nullable String failedPackageName, 660 Throwable t) { 661 final String msg = getCompleteMessage(t); 662 EventLogTags.writeRescueFailure(level, msg); 663 String failureMsg = "Failed rescue level " + levelToString(level); 664 if (!TextUtils.isEmpty(failedPackageName)) { 665 failureMsg += " for package " + failedPackageName; 666 } 667 logCriticalInfo(Log.ERROR, failureMsg + ": " + msg); 668 } 669 mapRescueLevelToUserImpact(int rescueLevel)670 private static int mapRescueLevelToUserImpact(int rescueLevel) { 671 if (Flags.recoverabilityDetection()) { 672 switch (rescueLevel) { 673 case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET: 674 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10; 675 case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET: 676 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_40; 677 case RESCUE_LEVEL_WARM_REBOOT: 678 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50; 679 case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: 680 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_71; 681 case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: 682 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_75; 683 case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: 684 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_80; 685 case RESCUE_LEVEL_FACTORY_RESET: 686 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100; 687 default: 688 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0; 689 } 690 } else { 691 switch (rescueLevel) { 692 case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: 693 case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: 694 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10; 695 case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: 696 case LEVEL_WARM_REBOOT: 697 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50; 698 case LEVEL_FACTORY_RESET: 699 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100; 700 default: 701 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0; 702 } 703 } 704 } 705 resetAllSettingsIfNecessary(Context context, int mode, int level)706 private static void resetAllSettingsIfNecessary(Context context, int mode, 707 int level) throws Exception { 708 // No need to reset Settings again if they are already reset in the current level once. 709 if (getMaxRescueLevelAttempted() >= level) { 710 return; 711 } 712 setMaxRescueLevelAttempted(level); 713 // Try our best to reset all settings possible, and once finished 714 // rethrow any exception that we encountered 715 Exception res = null; 716 final ContentResolver resolver = context.getContentResolver(); 717 try { 718 Settings.Global.resetToDefaultsAsUser(resolver, null, mode, 719 UserHandle.SYSTEM.getIdentifier()); 720 } catch (Exception e) { 721 res = new RuntimeException("Failed to reset global settings", e); 722 } 723 for (int userId : getAllUserIds()) { 724 try { 725 Settings.Secure.resetToDefaultsAsUser(resolver, null, mode, userId); 726 } catch (Exception e) { 727 res = new RuntimeException("Failed to reset secure settings for " + userId, e); 728 } 729 } 730 if (res != null) { 731 throw res; 732 } 733 } 734 resetDeviceConfig(Context context, boolean isScoped, @Nullable String failedPackage)735 private static void resetDeviceConfig(Context context, boolean isScoped, 736 @Nullable String failedPackage) throws Exception { 737 final ContentResolver resolver = context.getContentResolver(); 738 try { 739 if (!isScoped || failedPackage == null) { 740 resetAllAffectedNamespaces(context); 741 } else { 742 performScopedReset(context, failedPackage); 743 } 744 } catch (Exception e) { 745 throw new RuntimeException("Failed to reset config settings", e); 746 } 747 } 748 resetAllAffectedNamespaces(Context context)749 private static void resetAllAffectedNamespaces(Context context) { 750 RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context); 751 Set<String> allAffectedNamespaces = rescuePartyObserver.getAllAffectedNamespaceSet(); 752 753 Slog.w(TAG, 754 "Performing reset for all affected namespaces: " 755 + Arrays.toString(allAffectedNamespaces.toArray())); 756 Iterator<String> it = allAffectedNamespaces.iterator(); 757 while (it.hasNext()) { 758 String namespace = it.next(); 759 // Don't let RescueParty reset the namespace for RescueParty switches. 760 if (NAMESPACE_CONFIGURATION.equals(namespace)) { 761 continue; 762 } 763 DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, namespace); 764 } 765 } 766 performScopedReset(Context context, @NonNull String failedPackage)767 private static void performScopedReset(Context context, @NonNull String failedPackage) { 768 RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context); 769 Set<String> affectedNamespaces = rescuePartyObserver.getAffectedNamespaceSet( 770 failedPackage); 771 // If we can't find namespaces affected for current package, 772 // skip this round of reset. 773 if (affectedNamespaces != null) { 774 Slog.w(TAG, 775 "Performing scoped reset for package: " + failedPackage 776 + ", affected namespaces: " 777 + Arrays.toString(affectedNamespaces.toArray())); 778 Iterator<String> it = affectedNamespaces.iterator(); 779 while (it.hasNext()) { 780 String namespace = it.next(); 781 // Don't let RescueParty reset the namespace for RescueParty switches. 782 if (NAMESPACE_CONFIGURATION.equals(namespace)) { 783 continue; 784 } 785 DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, namespace); 786 } 787 } 788 } 789 790 /** 791 * Handle mitigation action for package failures. This observer will be register to Package 792 * Watchdog and will receive calls about package failures. This observer is persistent so it 793 * may choose to mitigate failures for packages it has not explicitly asked to observe. 794 */ 795 public static class RescuePartyObserver implements PackageHealthObserver { 796 797 private final Context mContext; 798 private final Map<String, Set<String>> mCallingPackageNamespaceSetMap = new HashMap<>(); 799 private final Map<String, Set<String>> mNamespaceCallingPackageSetMap = new HashMap<>(); 800 801 @GuardedBy("RescuePartyObserver.class") 802 static RescuePartyObserver sRescuePartyObserver; 803 RescuePartyObserver(Context context)804 private RescuePartyObserver(Context context) { 805 mContext = context; 806 } 807 808 /** Creates or gets singleton instance of RescueParty. */ getInstance(Context context)809 public static RescuePartyObserver getInstance(Context context) { 810 synchronized (RescuePartyObserver.class) { 811 if (sRescuePartyObserver == null) { 812 sRescuePartyObserver = new RescuePartyObserver(context); 813 } 814 return sRescuePartyObserver; 815 } 816 } 817 818 /** Gets singleton instance. It returns null if the instance is not created yet.*/ 819 @Nullable getInstanceIfCreated()820 public static RescuePartyObserver getInstanceIfCreated() { 821 synchronized (RescuePartyObserver.class) { 822 return sRescuePartyObserver; 823 } 824 } 825 826 @VisibleForTesting reset()827 static void reset() { 828 synchronized (RescuePartyObserver.class) { 829 sRescuePartyObserver = null; 830 } 831 } 832 833 @Override onHealthCheckFailed(@ullable VersionedPackage failedPackage, @FailureReasons int failureReason, int mitigationCount)834 public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage, 835 @FailureReasons int failureReason, int mitigationCount) { 836 if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH 837 || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) { 838 if (Flags.recoverabilityDetection()) { 839 return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, 840 mayPerformReboot(failedPackage), failedPackage)); 841 } else { 842 return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, 843 mayPerformReboot(failedPackage))); 844 } 845 } else { 846 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0; 847 } 848 } 849 850 @Override execute(@ullable VersionedPackage failedPackage, @FailureReasons int failureReason, int mitigationCount)851 public boolean execute(@Nullable VersionedPackage failedPackage, 852 @FailureReasons int failureReason, int mitigationCount) { 853 if (isDisabled()) { 854 return false; 855 } 856 if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH 857 || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) { 858 final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount, 859 mayPerformReboot(failedPackage), failedPackage) 860 : getRescueLevel(mitigationCount, 861 mayPerformReboot(failedPackage)); 862 executeRescueLevel(mContext, 863 failedPackage == null ? null : failedPackage.getPackageName(), level); 864 return true; 865 } else { 866 return false; 867 } 868 } 869 870 @Override isPersistent()871 public boolean isPersistent() { 872 return true; 873 } 874 875 @Override mayObservePackage(String packageName)876 public boolean mayObservePackage(String packageName) { 877 PackageManager pm = mContext.getPackageManager(); 878 try { 879 // A package is a module if this is non-null 880 if (pm.getModuleInfo(packageName, 0) != null) { 881 return true; 882 } 883 } catch (PackageManager.NameNotFoundException | IllegalStateException ignore) { 884 } 885 886 return isPersistentSystemApp(packageName); 887 } 888 889 @Override onBootLoop(int mitigationCount)890 public int onBootLoop(int mitigationCount) { 891 if (isDisabled()) { 892 return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0; 893 } 894 if (Flags.recoverabilityDetection()) { 895 return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, 896 true, /*failedPackage=*/ null)); 897 } else { 898 return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true)); 899 } 900 } 901 902 @Override executeBootLoopMitigation(int mitigationCount)903 public boolean executeBootLoopMitigation(int mitigationCount) { 904 if (isDisabled()) { 905 return false; 906 } 907 boolean mayPerformReboot = !shouldThrottleReboot(); 908 final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount, 909 mayPerformReboot, /*failedPackage=*/ null) 910 : getRescueLevel(mitigationCount, mayPerformReboot); 911 executeRescueLevel(mContext, /*failedPackage=*/ null, level); 912 return true; 913 } 914 915 @Override getName()916 public String getName() { 917 return NAME; 918 } 919 920 /** 921 * Returns {@code true} if the failing package is non-null and performing a reboot or 922 * prompting a factory reset is an acceptable mitigation strategy for the package's 923 * failure, {@code false} otherwise. 924 */ mayPerformReboot(@ullable VersionedPackage failingPackage)925 private boolean mayPerformReboot(@Nullable VersionedPackage failingPackage) { 926 if (failingPackage == null) { 927 return false; 928 } 929 if (shouldThrottleReboot()) { 930 return false; 931 } 932 933 return isPersistentSystemApp(failingPackage.getPackageName()); 934 } 935 936 /** 937 * Returns {@code true} if Rescue Party is allowed to attempt a reboot or factory reset. 938 * Will return {@code false} if a factory reset was already offered recently. 939 */ shouldThrottleReboot()940 private boolean shouldThrottleReboot() { 941 Long lastResetTime = getLastFactoryResetTimeMs(); 942 long now = System.currentTimeMillis(); 943 long throttleDurationMin = SystemProperties.getLong(PROP_THROTTLE_DURATION_MIN_FLAG, 944 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN); 945 return now < lastResetTime + TimeUnit.MINUTES.toMillis(throttleDurationMin); 946 } 947 isPersistentSystemApp(@onNull String packageName)948 private boolean isPersistentSystemApp(@NonNull String packageName) { 949 PackageManager pm = mContext.getPackageManager(); 950 try { 951 ApplicationInfo info = pm.getApplicationInfo(packageName, 0); 952 return (info.flags & PERSISTENT_MASK) == PERSISTENT_MASK; 953 } catch (PackageManager.NameNotFoundException e) { 954 return false; 955 } 956 } 957 recordDeviceConfigAccess(@onNull String callingPackage, @NonNull String namespace)958 private synchronized void recordDeviceConfigAccess(@NonNull String callingPackage, 959 @NonNull String namespace) { 960 // Record it in calling packages to namespace map 961 Set<String> namespaceSet = mCallingPackageNamespaceSetMap.get(callingPackage); 962 if (namespaceSet == null) { 963 namespaceSet = new ArraySet<>(); 964 mCallingPackageNamespaceSetMap.put(callingPackage, namespaceSet); 965 } 966 namespaceSet.add(namespace); 967 // Record it in namespace to calling packages map 968 Set<String> callingPackageSet = mNamespaceCallingPackageSetMap.get(namespace); 969 if (callingPackageSet == null) { 970 callingPackageSet = new ArraySet<>(); 971 } 972 callingPackageSet.add(callingPackage); 973 mNamespaceCallingPackageSetMap.put(namespace, callingPackageSet); 974 } 975 getAffectedNamespaceSet(String failedPackage)976 private synchronized Set<String> getAffectedNamespaceSet(String failedPackage) { 977 return mCallingPackageNamespaceSetMap.get(failedPackage); 978 } 979 getAllAffectedNamespaceSet()980 private synchronized Set<String> getAllAffectedNamespaceSet() { 981 return new HashSet<String>(mNamespaceCallingPackageSetMap.keySet()); 982 } 983 getCallingPackagesSet(String namespace)984 private synchronized Set<String> getCallingPackagesSet(String namespace) { 985 return mNamespaceCallingPackageSetMap.get(namespace); 986 } 987 } 988 getAllUserIds()989 private static int[] getAllUserIds() { 990 int systemUserId = UserHandle.SYSTEM.getIdentifier(); 991 int[] userIds = { systemUserId }; 992 try { 993 for (File file : FileUtils.listFilesOrEmpty(Environment.getDataSystemDeDirectory())) { 994 try { 995 final int userId = Integer.parseInt(file.getName()); 996 if (userId != systemUserId) { 997 userIds = ArrayUtils.appendInt(userIds, userId); 998 } 999 } catch (NumberFormatException ignored) { 1000 } 1001 } 1002 } catch (Throwable t) { 1003 Slog.w(TAG, "Trouble discovering users", t); 1004 } 1005 return userIds; 1006 } 1007 1008 /** 1009 * Hacky test to check if the device has an active USB connection, which is 1010 * a good proxy for someone doing local development work. 1011 */ isUsbActive()1012 private static boolean isUsbActive() { 1013 if (SystemProperties.getBoolean(PROP_VIRTUAL_DEVICE, false)) { 1014 Slog.v(TAG, "Assuming virtual device is connected over USB"); 1015 return true; 1016 } 1017 try { 1018 final String state = FileUtils 1019 .readTextFile(new File("/sys/class/android_usb/android0/state"), 128, ""); 1020 return "CONFIGURED".equals(state.trim()); 1021 } catch (Throwable t) { 1022 Slog.w(TAG, "Failed to determine if device was on USB", t); 1023 return false; 1024 } 1025 } 1026 levelToString(int level)1027 private static String levelToString(int level) { 1028 if (Flags.recoverabilityDetection()) { 1029 switch (level) { 1030 case RESCUE_LEVEL_NONE: 1031 return "NONE"; 1032 case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET: 1033 return "SCOPED_DEVICE_CONFIG_RESET"; 1034 case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET: 1035 return "ALL_DEVICE_CONFIG_RESET"; 1036 case RESCUE_LEVEL_WARM_REBOOT: 1037 return "WARM_REBOOT"; 1038 case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: 1039 return "RESET_SETTINGS_UNTRUSTED_DEFAULTS"; 1040 case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: 1041 return "RESET_SETTINGS_UNTRUSTED_CHANGES"; 1042 case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: 1043 return "RESET_SETTINGS_TRUSTED_DEFAULTS"; 1044 case RESCUE_LEVEL_FACTORY_RESET: 1045 return "FACTORY_RESET"; 1046 default: 1047 return Integer.toString(level); 1048 } 1049 } else { 1050 switch (level) { 1051 case LEVEL_NONE: 1052 return "NONE"; 1053 case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: 1054 return "RESET_SETTINGS_UNTRUSTED_DEFAULTS"; 1055 case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: 1056 return "RESET_SETTINGS_UNTRUSTED_CHANGES"; 1057 case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: 1058 return "RESET_SETTINGS_TRUSTED_DEFAULTS"; 1059 case LEVEL_WARM_REBOOT: 1060 return "WARM_REBOOT"; 1061 case LEVEL_FACTORY_RESET: 1062 return "FACTORY_RESET"; 1063 default: 1064 return Integer.toString(level); 1065 } 1066 } 1067 } 1068 } 1069