1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server; 18 19 import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig; 20 21 import static java.lang.annotation.RetentionPolicy.SOURCE; 22 23 import android.annotation.IntDef; 24 import android.annotation.Nullable; 25 import android.content.Context; 26 import android.content.pm.PackageManager; 27 import android.content.pm.VersionedPackage; 28 import android.net.NetworkStackClient; 29 import android.os.Environment; 30 import android.os.Handler; 31 import android.os.Looper; 32 import android.os.SystemClock; 33 import android.provider.DeviceConfig; 34 import android.text.TextUtils; 35 import android.util.ArrayMap; 36 import android.util.ArraySet; 37 import android.util.AtomicFile; 38 import android.util.Slog; 39 import android.util.Xml; 40 41 import com.android.internal.annotations.GuardedBy; 42 import com.android.internal.annotations.VisibleForTesting; 43 import com.android.internal.os.BackgroundThread; 44 import com.android.internal.util.FastXmlSerializer; 45 import com.android.internal.util.XmlUtils; 46 47 import libcore.io.IoUtils; 48 49 import org.xmlpull.v1.XmlPullParser; 50 import org.xmlpull.v1.XmlPullParserException; 51 import org.xmlpull.v1.XmlSerializer; 52 53 import java.io.File; 54 import java.io.FileNotFoundException; 55 import java.io.FileOutputStream; 56 import java.io.IOException; 57 import java.io.InputStream; 58 import java.lang.annotation.Retention; 59 import java.nio.charset.StandardCharsets; 60 import java.util.ArrayList; 61 import java.util.Collections; 62 import java.util.Iterator; 63 import java.util.List; 64 import java.util.Map; 65 import java.util.Set; 66 import java.util.concurrent.TimeUnit; 67 68 /** 69 * Monitors the health of packages on the system and notifies interested observers when packages 70 * fail. On failure, the registered observer with the least user impacting mitigation will 71 * be notified. 72 */ 73 public class PackageWatchdog { 74 private static final String TAG = "PackageWatchdog"; 75 76 static final String PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS = 77 "watchdog_trigger_failure_duration_millis"; 78 static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT = 79 "watchdog_trigger_failure_count"; 80 static final String PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED = 81 "watchdog_explicit_health_check_enabled"; 82 83 // Duration to count package failures before it resets to 0 84 private static final int DEFAULT_TRIGGER_FAILURE_DURATION_MS = 85 (int) TimeUnit.MINUTES.toMillis(1); 86 // Number of package failures within the duration above before we notify observers 87 private static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5; 88 // Whether explicit health checks are enabled or not 89 private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true; 90 91 private static final int DB_VERSION = 1; 92 private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog"; 93 private static final String TAG_PACKAGE = "package"; 94 private static final String TAG_OBSERVER = "observer"; 95 private static final String ATTR_VERSION = "version"; 96 private static final String ATTR_NAME = "name"; 97 private static final String ATTR_DURATION = "duration"; 98 private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration"; 99 private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check"; 100 101 @GuardedBy("PackageWatchdog.class") 102 private static PackageWatchdog sPackageWatchdog; 103 104 private final Object mLock = new Object(); 105 // System server context 106 private final Context mContext; 107 // Handler to run short running tasks 108 private final Handler mShortTaskHandler; 109 // Handler for processing IO and long running tasks 110 private final Handler mLongTaskHandler; 111 // Contains (observer-name -> observer-handle) that have ever been registered from 112 // previous boots. Observers with all packages expired are periodically pruned. 113 // It is saved to disk on system shutdown and repouplated on startup so it survives reboots. 114 @GuardedBy("mLock") 115 private final ArrayMap<String, ObserverInternal> mAllObservers = new ArrayMap<>(); 116 // File containing the XML data of monitored packages /data/system/package-watchdog.xml 117 private final AtomicFile mPolicyFile; 118 private final ExplicitHealthCheckController mHealthCheckController; 119 private final NetworkStackClient mNetworkStackClient; 120 @GuardedBy("mLock") 121 private boolean mIsPackagesReady; 122 // Flag to control whether explicit health checks are supported or not 123 @GuardedBy("mLock") 124 private boolean mIsHealthCheckEnabled = DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED; 125 @GuardedBy("mLock") 126 private int mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS; 127 @GuardedBy("mLock") 128 private int mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT; 129 // SystemClock#uptimeMillis when we last executed #syncState 130 // 0 if no prune is scheduled. 131 @GuardedBy("mLock") 132 private long mUptimeAtLastStateSync; 133 PackageWatchdog(Context context)134 private PackageWatchdog(Context context) { 135 // Needs to be constructed inline 136 this(context, new AtomicFile( 137 new File(new File(Environment.getDataDirectory(), "system"), 138 "package-watchdog.xml")), 139 new Handler(Looper.myLooper()), BackgroundThread.getHandler(), 140 new ExplicitHealthCheckController(context), 141 NetworkStackClient.getInstance()); 142 } 143 144 /** 145 * Creates a PackageWatchdog that allows injecting dependencies. 146 */ 147 @VisibleForTesting PackageWatchdog(Context context, AtomicFile policyFile, Handler shortTaskHandler, Handler longTaskHandler, ExplicitHealthCheckController controller, NetworkStackClient networkStackClient)148 PackageWatchdog(Context context, AtomicFile policyFile, Handler shortTaskHandler, 149 Handler longTaskHandler, ExplicitHealthCheckController controller, 150 NetworkStackClient networkStackClient) { 151 mContext = context; 152 mPolicyFile = policyFile; 153 mShortTaskHandler = shortTaskHandler; 154 mLongTaskHandler = longTaskHandler; 155 mHealthCheckController = controller; 156 mNetworkStackClient = networkStackClient; 157 loadFromFile(); 158 } 159 160 /** Creates or gets singleton instance of PackageWatchdog. */ getInstance(Context context)161 public static PackageWatchdog getInstance(Context context) { 162 synchronized (PackageWatchdog.class) { 163 if (sPackageWatchdog == null) { 164 sPackageWatchdog = new PackageWatchdog(context); 165 } 166 return sPackageWatchdog; 167 } 168 } 169 170 /** 171 * Called during boot to notify when packages are ready on the device so we can start 172 * binding. 173 */ onPackagesReady()174 public void onPackagesReady() { 175 synchronized (mLock) { 176 mIsPackagesReady = true; 177 mHealthCheckController.setCallbacks(packageName -> onHealthCheckPassed(packageName), 178 packages -> onSupportedPackages(packages), 179 () -> syncRequestsAsync()); 180 setPropertyChangedListenerLocked(); 181 updateConfigs(); 182 registerNetworkStackHealthListener(); 183 } 184 } 185 186 /** 187 * Registers {@code observer} to listen for package failures 188 * 189 * <p>Observers are expected to call this on boot. It does not specify any packages but 190 * it will resume observing any packages requested from a previous boot. 191 */ registerHealthObserver(PackageHealthObserver observer)192 public void registerHealthObserver(PackageHealthObserver observer) { 193 synchronized (mLock) { 194 ObserverInternal internalObserver = mAllObservers.get(observer.getName()); 195 if (internalObserver != null) { 196 internalObserver.mRegisteredObserver = observer; 197 } 198 } 199 } 200 201 /** 202 * Starts observing the health of the {@code packages} for {@code observer} and notifies 203 * {@code observer} of any package failures within the monitoring duration. 204 * 205 * <p>If monitoring a package supporting explicit health check, at the end of the monitoring 206 * duration if {@link #onHealthCheckPassed} was never called, 207 * {@link PackageHealthObserver#execute} will be called as if the package failed. 208 * 209 * <p>If {@code observer} is already monitoring a package in {@code packageNames}, 210 * the monitoring window of that package will be reset to {@code durationMs} and the health 211 * check state will be reset to a default depending on if the package is contained in 212 * {@link mPackagesWithExplicitHealthCheckEnabled}. 213 * 214 * @throws IllegalArgumentException if {@code packageNames} is empty 215 * or {@code durationMs} is less than 1 216 */ startObservingHealth(PackageHealthObserver observer, List<String> packageNames, long durationMs)217 public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames, 218 long durationMs) { 219 if (packageNames.isEmpty()) { 220 Slog.wtf(TAG, "No packages to observe, " + observer.getName()); 221 return; 222 } 223 if (durationMs < 1) { 224 // TODO: Instead of failing, monitor for default? 48hrs? 225 throw new IllegalArgumentException("Invalid duration " + durationMs + "ms for observer " 226 + observer.getName() + ". Not observing packages " + packageNames); 227 } 228 229 List<MonitoredPackage> packages = new ArrayList<>(); 230 for (int i = 0; i < packageNames.size(); i++) { 231 // Health checks not available yet so health check state will start INACTIVE 232 packages.add(new MonitoredPackage(packageNames.get(i), durationMs, false)); 233 } 234 235 // Sync before we add the new packages to the observers. This will #pruneObservers, 236 // causing any elapsed time to be deducted from all existing packages before we add new 237 // packages. This maintains the invariant that the elapsed time for ALL (new and existing) 238 // packages is the same. 239 syncState("observing new packages"); 240 241 synchronized (mLock) { 242 ObserverInternal oldObserver = mAllObservers.get(observer.getName()); 243 if (oldObserver == null) { 244 Slog.d(TAG, observer.getName() + " started monitoring health " 245 + "of packages " + packageNames); 246 mAllObservers.put(observer.getName(), 247 new ObserverInternal(observer.getName(), packages)); 248 } else { 249 Slog.d(TAG, observer.getName() + " added the following " 250 + "packages to monitor " + packageNames); 251 oldObserver.updatePackagesLocked(packages); 252 } 253 } 254 255 // Register observer in case not already registered 256 registerHealthObserver(observer); 257 258 // Sync after we add the new packages to the observers. We may have received packges 259 // requiring an earlier schedule than we are currently scheduled for. 260 syncState("updated observers"); 261 } 262 263 /** 264 * Unregisters {@code observer} from listening to package failure. 265 * Additionally, this stops observing any packages that may have previously been observed 266 * even from a previous boot. 267 */ unregisterHealthObserver(PackageHealthObserver observer)268 public void unregisterHealthObserver(PackageHealthObserver observer) { 269 synchronized (mLock) { 270 mAllObservers.remove(observer.getName()); 271 } 272 syncState("unregistering observer: " + observer.getName()); 273 } 274 275 /** 276 * Returns packages observed by {@code observer} 277 * 278 * @return an empty set if {@code observer} has some packages observerd from a previous boot 279 * but has not registered itself in the current boot to receive notifications. Returns null 280 * if there are no active packages monitored from any boot. 281 */ 282 @Nullable getPackages(PackageHealthObserver observer)283 public Set<String> getPackages(PackageHealthObserver observer) { 284 synchronized (mLock) { 285 for (int i = 0; i < mAllObservers.size(); i++) { 286 if (observer.getName().equals(mAllObservers.keyAt(i))) { 287 if (observer.equals(mAllObservers.valueAt(i).mRegisteredObserver)) { 288 return mAllObservers.valueAt(i).mPackages.keySet(); 289 } 290 return Collections.emptySet(); 291 } 292 } 293 } 294 return null; 295 } 296 297 /** 298 * Called when a process fails either due to a crash or ANR. 299 * 300 * <p>For each package contained in the process, one registered observer with the least user 301 * impact will be notified for mitigation. 302 * 303 * <p>This method could be called frequently if there is a severe problem on the device. 304 */ onPackageFailure(List<VersionedPackage> packages)305 public void onPackageFailure(List<VersionedPackage> packages) { 306 mLongTaskHandler.post(() -> { 307 synchronized (mLock) { 308 if (mAllObservers.isEmpty()) { 309 return; 310 } 311 312 for (int pIndex = 0; pIndex < packages.size(); pIndex++) { 313 VersionedPackage versionedPackage = packages.get(pIndex); 314 // Observer that will receive failure for versionedPackage 315 PackageHealthObserver currentObserverToNotify = null; 316 int currentObserverImpact = Integer.MAX_VALUE; 317 318 // Find observer with least user impact 319 for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { 320 ObserverInternal observer = mAllObservers.valueAt(oIndex); 321 PackageHealthObserver registeredObserver = observer.mRegisteredObserver; 322 if (registeredObserver != null 323 && observer.onPackageFailureLocked( 324 versionedPackage.getPackageName())) { 325 int impact = registeredObserver.onHealthCheckFailed(versionedPackage); 326 if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE 327 && impact < currentObserverImpact) { 328 currentObserverToNotify = registeredObserver; 329 currentObserverImpact = impact; 330 } 331 } 332 } 333 334 // Execute action with least user impact 335 if (currentObserverToNotify != null) { 336 currentObserverToNotify.execute(versionedPackage); 337 } 338 } 339 } 340 }); 341 } 342 343 // TODO(b/120598832): Optimize write? Maybe only write a separate smaller file? Also 344 // avoid holding lock? 345 // This currently adds about 7ms extra to shutdown thread 346 /** Writes the package information to file during shutdown. */ writeNow()347 public void writeNow() { 348 synchronized (mLock) { 349 // Must only run synchronous tasks as this runs on the ShutdownThread and no other 350 // thread is guaranteed to run during shutdown. 351 if (!mAllObservers.isEmpty()) { 352 mLongTaskHandler.removeCallbacks(this::saveToFileAsync); 353 pruneObserversLocked(); 354 saveToFile(); 355 Slog.i(TAG, "Last write to update package durations"); 356 } 357 } 358 } 359 360 /** 361 * Enables or disables explicit health checks. 362 * <p> If explicit health checks are enabled, the health check service is started. 363 * <p> If explicit health checks are disabled, pending explicit health check requests are 364 * passed and the health check service is stopped. 365 */ setExplicitHealthCheckEnabled(boolean enabled)366 private void setExplicitHealthCheckEnabled(boolean enabled) { 367 synchronized (mLock) { 368 mIsHealthCheckEnabled = enabled; 369 mHealthCheckController.setEnabled(enabled); 370 // Prune to update internal state whenever health check is enabled/disabled 371 syncState("health check state " + (enabled ? "enabled" : "disabled")); 372 } 373 } 374 375 /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */ 376 @Retention(SOURCE) 377 @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_NONE, 378 PackageHealthObserverImpact.USER_IMPACT_LOW, 379 PackageHealthObserverImpact.USER_IMPACT_MEDIUM, 380 PackageHealthObserverImpact.USER_IMPACT_HIGH}) 381 public @interface PackageHealthObserverImpact { 382 /** No action to take. */ 383 int USER_IMPACT_NONE = 0; 384 /* Action has low user impact, user of a device will barely notice. */ 385 int USER_IMPACT_LOW = 1; 386 /* Action has medium user impact, user of a device will likely notice. */ 387 int USER_IMPACT_MEDIUM = 3; 388 /* Action has high user impact, a last resort, user of a device will be very frustrated. */ 389 int USER_IMPACT_HIGH = 5; 390 } 391 392 /** Register instances of this interface to receive notifications on package failure. */ 393 public interface PackageHealthObserver { 394 /** 395 * Called when health check fails for the {@code versionedPackage}. 396 * 397 * @return any one of {@link PackageHealthObserverImpact} to express the impact 398 * to the user on {@link #execute} 399 */ onHealthCheckFailed(VersionedPackage versionedPackage)400 @PackageHealthObserverImpact int onHealthCheckFailed(VersionedPackage versionedPackage); 401 402 /** 403 * Executes mitigation for {@link #onHealthCheckFailed}. 404 * 405 * @return {@code true} if action was executed successfully, {@code false} otherwise 406 */ execute(VersionedPackage versionedPackage)407 boolean execute(VersionedPackage versionedPackage); 408 409 // TODO(b/120598832): Ensure uniqueness? 410 /** 411 * Identifier for the observer, should not change across device updates otherwise the 412 * watchdog may drop observing packages with the old name. 413 */ getName()414 String getName(); 415 } 416 getTriggerFailureCount()417 long getTriggerFailureCount() { 418 synchronized (mLock) { 419 return mTriggerFailureCount; 420 } 421 } 422 423 /** 424 * Serializes and syncs health check requests with the {@link ExplicitHealthCheckController}. 425 */ syncRequestsAsync()426 private void syncRequestsAsync() { 427 mShortTaskHandler.removeCallbacks(this::syncRequests); 428 mShortTaskHandler.post(this::syncRequests); 429 } 430 431 /** 432 * Syncs health check requests with the {@link ExplicitHealthCheckController}. 433 * Calls to this must be serialized. 434 * 435 * @see #syncRequestsAsync 436 */ syncRequests()437 private void syncRequests() { 438 Set<String> packages = null; 439 synchronized (mLock) { 440 if (mIsPackagesReady) { 441 packages = getPackagesPendingHealthChecksLocked(); 442 } // else, we will sync requests when packages become ready 443 } 444 445 // Call outside lock to avoid holding lock when calling into the controller. 446 if (packages != null) { 447 Slog.i(TAG, "Syncing health check requests for packages: " + packages); 448 mHealthCheckController.syncRequests(packages); 449 } 450 } 451 452 /** 453 * Updates the observers monitoring {@code packageName} that explicit health check has passed. 454 * 455 * <p> This update is strictly for registered observers at the time of the call 456 * Observers that register after this signal will have no knowledge of prior signals and will 457 * effectively behave as if the explicit health check hasn't passed for {@code packageName}. 458 * 459 * <p> {@code packageName} can still be considered failed if reported by 460 * {@link #onPackageFailureLocked} before the package expires. 461 * 462 * <p> Triggered by components outside the system server when they are fully functional after an 463 * update. 464 */ onHealthCheckPassed(String packageName)465 private void onHealthCheckPassed(String packageName) { 466 Slog.i(TAG, "Health check passed for package: " + packageName); 467 boolean isStateChanged = false; 468 469 synchronized (mLock) { 470 for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) { 471 ObserverInternal observer = mAllObservers.valueAt(observerIdx); 472 MonitoredPackage monitoredPackage = observer.mPackages.get(packageName); 473 474 if (monitoredPackage != null) { 475 int oldState = monitoredPackage.getHealthCheckStateLocked(); 476 int newState = monitoredPackage.tryPassHealthCheckLocked(); 477 isStateChanged |= oldState != newState; 478 } 479 } 480 } 481 482 if (isStateChanged) { 483 syncState("health check passed for " + packageName); 484 } 485 } 486 onSupportedPackages(List<PackageConfig> supportedPackages)487 private void onSupportedPackages(List<PackageConfig> supportedPackages) { 488 boolean isStateChanged = false; 489 490 Map<String, Long> supportedPackageTimeouts = new ArrayMap<>(); 491 Iterator<PackageConfig> it = supportedPackages.iterator(); 492 while (it.hasNext()) { 493 PackageConfig info = it.next(); 494 supportedPackageTimeouts.put(info.getPackageName(), info.getHealthCheckTimeoutMillis()); 495 } 496 497 synchronized (mLock) { 498 Slog.d(TAG, "Received supported packages " + supportedPackages); 499 Iterator<ObserverInternal> oit = mAllObservers.values().iterator(); 500 while (oit.hasNext()) { 501 Iterator<MonitoredPackage> pit = oit.next().mPackages.values().iterator(); 502 while (pit.hasNext()) { 503 MonitoredPackage monitoredPackage = pit.next(); 504 String packageName = monitoredPackage.getName(); 505 int oldState = monitoredPackage.getHealthCheckStateLocked(); 506 int newState; 507 508 if (supportedPackageTimeouts.containsKey(packageName)) { 509 // Supported packages become ACTIVE if currently INACTIVE 510 newState = monitoredPackage.setHealthCheckActiveLocked( 511 supportedPackageTimeouts.get(packageName)); 512 } else { 513 // Unsupported packages are marked as PASSED unless already FAILED 514 newState = monitoredPackage.tryPassHealthCheckLocked(); 515 } 516 isStateChanged |= oldState != newState; 517 } 518 } 519 } 520 521 if (isStateChanged) { 522 syncState("updated health check supported packages " + supportedPackages); 523 } 524 } 525 526 @GuardedBy("mLock") getPackagesPendingHealthChecksLocked()527 private Set<String> getPackagesPendingHealthChecksLocked() { 528 Slog.d(TAG, "Getting all observed packages pending health checks"); 529 Set<String> packages = new ArraySet<>(); 530 Iterator<ObserverInternal> oit = mAllObservers.values().iterator(); 531 while (oit.hasNext()) { 532 ObserverInternal observer = oit.next(); 533 Iterator<MonitoredPackage> pit = 534 observer.mPackages.values().iterator(); 535 while (pit.hasNext()) { 536 MonitoredPackage monitoredPackage = pit.next(); 537 String packageName = monitoredPackage.getName(); 538 if (monitoredPackage.isPendingHealthChecksLocked()) { 539 packages.add(packageName); 540 } 541 } 542 } 543 return packages; 544 } 545 546 /** 547 * Syncs the state of the observers. 548 * 549 * <p> Prunes all observers, saves new state to disk, syncs health check requests with the 550 * health check service and schedules the next state sync. 551 */ syncState(String reason)552 private void syncState(String reason) { 553 synchronized (mLock) { 554 Slog.i(TAG, "Syncing state, reason: " + reason); 555 pruneObserversLocked(); 556 557 saveToFileAsync(); 558 syncRequestsAsync(); 559 560 // Done syncing state, schedule the next state sync 561 scheduleNextSyncStateLocked(); 562 } 563 } 564 syncStateWithScheduledReason()565 private void syncStateWithScheduledReason() { 566 syncState("scheduled"); 567 } 568 569 @GuardedBy("mLock") scheduleNextSyncStateLocked()570 private void scheduleNextSyncStateLocked() { 571 long durationMs = getNextStateSyncMillisLocked(); 572 mShortTaskHandler.removeCallbacks(this::syncStateWithScheduledReason); 573 if (durationMs == Long.MAX_VALUE) { 574 Slog.i(TAG, "Cancelling state sync, nothing to sync"); 575 mUptimeAtLastStateSync = 0; 576 } else { 577 Slog.i(TAG, "Scheduling next state sync in " + durationMs + "ms"); 578 mUptimeAtLastStateSync = SystemClock.uptimeMillis(); 579 mShortTaskHandler.postDelayed(this::syncStateWithScheduledReason, durationMs); 580 } 581 } 582 583 /** 584 * Returns the next duration in millis to sync the watchdog state. 585 * 586 * @returns Long#MAX_VALUE if there are no observed packages. 587 */ 588 @GuardedBy("mLock") getNextStateSyncMillisLocked()589 private long getNextStateSyncMillisLocked() { 590 long shortestDurationMs = Long.MAX_VALUE; 591 for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { 592 ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).mPackages; 593 for (int pIndex = 0; pIndex < packages.size(); pIndex++) { 594 MonitoredPackage mp = packages.valueAt(pIndex); 595 long duration = mp.getShortestScheduleDurationMsLocked(); 596 if (duration < shortestDurationMs) { 597 shortestDurationMs = duration; 598 } 599 } 600 } 601 return shortestDurationMs; 602 } 603 604 /** 605 * Removes {@code elapsedMs} milliseconds from all durations on monitored packages 606 * and updates other internal state. 607 */ 608 @GuardedBy("mLock") pruneObserversLocked()609 private void pruneObserversLocked() { 610 long elapsedMs = mUptimeAtLastStateSync == 0 611 ? 0 : SystemClock.uptimeMillis() - mUptimeAtLastStateSync; 612 if (elapsedMs <= 0) { 613 Slog.i(TAG, "Not pruning observers, elapsed time: " + elapsedMs + "ms"); 614 return; 615 } 616 617 Slog.i(TAG, "Removing " + elapsedMs + "ms from all packages on all observers"); 618 Iterator<ObserverInternal> it = mAllObservers.values().iterator(); 619 while (it.hasNext()) { 620 ObserverInternal observer = it.next(); 621 Set<MonitoredPackage> failedPackages = 622 observer.prunePackagesLocked(elapsedMs); 623 if (!failedPackages.isEmpty()) { 624 onHealthCheckFailed(observer, failedPackages); 625 } 626 if (observer.mPackages.isEmpty()) { 627 Slog.i(TAG, "Discarding observer " + observer.mName + ". All packages expired"); 628 it.remove(); 629 } 630 } 631 } 632 onHealthCheckFailed(ObserverInternal observer, Set<MonitoredPackage> failedPackages)633 private void onHealthCheckFailed(ObserverInternal observer, 634 Set<MonitoredPackage> failedPackages) { 635 mLongTaskHandler.post(() -> { 636 synchronized (mLock) { 637 PackageHealthObserver registeredObserver = observer.mRegisteredObserver; 638 if (registeredObserver != null) { 639 Iterator<MonitoredPackage> it = failedPackages.iterator(); 640 while (it.hasNext()) { 641 String failedPackage = it.next().getName(); 642 Slog.i(TAG, "Explicit health check failed for package " + failedPackage); 643 VersionedPackage versionedPkg = getVersionedPackage(failedPackage); 644 if (versionedPkg == null) { 645 Slog.w(TAG, "Explicit health check failed but could not find package " 646 + failedPackage); 647 // TODO(b/120598832): Skip. We only continue to pass tests for now since 648 // the tests don't install any packages 649 versionedPkg = new VersionedPackage(failedPackage, 0L); 650 } 651 registeredObserver.execute(versionedPkg); 652 } 653 } 654 } 655 }); 656 } 657 658 @Nullable getVersionedPackage(String packageName)659 private VersionedPackage getVersionedPackage(String packageName) { 660 final PackageManager pm = mContext.getPackageManager(); 661 if (pm == null) { 662 return null; 663 } 664 try { 665 final long versionCode = pm.getPackageInfo( 666 packageName, 0 /* flags */).getLongVersionCode(); 667 return new VersionedPackage(packageName, versionCode); 668 } catch (PackageManager.NameNotFoundException e) { 669 return null; 670 } 671 } 672 673 /** 674 * Loads mAllObservers from file. 675 * 676 * <p>Note that this is <b>not</b> thread safe and should only called be called 677 * from the constructor. 678 */ loadFromFile()679 private void loadFromFile() { 680 InputStream infile = null; 681 mAllObservers.clear(); 682 try { 683 infile = mPolicyFile.openRead(); 684 final XmlPullParser parser = Xml.newPullParser(); 685 parser.setInput(infile, StandardCharsets.UTF_8.name()); 686 XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG); 687 int outerDepth = parser.getDepth(); 688 while (XmlUtils.nextElementWithin(parser, outerDepth)) { 689 ObserverInternal observer = ObserverInternal.read(parser, this); 690 if (observer != null) { 691 mAllObservers.put(observer.mName, observer); 692 } 693 } 694 } catch (FileNotFoundException e) { 695 // Nothing to monitor 696 } catch (IOException | NumberFormatException | XmlPullParserException e) { 697 Slog.wtf(TAG, "Unable to read monitored packages, deleting file", e); 698 mPolicyFile.delete(); 699 } finally { 700 IoUtils.closeQuietly(infile); 701 } 702 } 703 704 /** Adds a {@link DeviceConfig#OnPropertiesChangedListener}. */ setPropertyChangedListenerLocked()705 private void setPropertyChangedListenerLocked() { 706 DeviceConfig.addOnPropertiesChangedListener( 707 DeviceConfig.NAMESPACE_ROLLBACK, 708 mContext.getMainExecutor(), 709 (properties) -> { 710 if (!DeviceConfig.NAMESPACE_ROLLBACK.equals(properties.getNamespace())) { 711 return; 712 } 713 updateConfigs(); 714 }); 715 } 716 717 /** 718 * Health check is enabled or disabled after reading the flags 719 * from DeviceConfig. 720 */ updateConfigs()721 private void updateConfigs() { 722 synchronized (mLock) { 723 mTriggerFailureCount = DeviceConfig.getInt( 724 DeviceConfig.NAMESPACE_ROLLBACK, 725 PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT, 726 DEFAULT_TRIGGER_FAILURE_COUNT); 727 if (mTriggerFailureCount <= 0) { 728 mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT; 729 } 730 731 mTriggerFailureDurationMs = DeviceConfig.getInt( 732 DeviceConfig.NAMESPACE_ROLLBACK, 733 PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS, 734 DEFAULT_TRIGGER_FAILURE_DURATION_MS); 735 if (mTriggerFailureDurationMs <= 0) { 736 mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_COUNT; 737 } 738 739 setExplicitHealthCheckEnabled(DeviceConfig.getBoolean( 740 DeviceConfig.NAMESPACE_ROLLBACK, 741 PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED, 742 DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED)); 743 } 744 } 745 registerNetworkStackHealthListener()746 private void registerNetworkStackHealthListener() { 747 // TODO: have an internal method to trigger a rollback by reporting high severity errors, 748 // and rely on ActivityManager to inform the watchdog of severe network stack crashes 749 // instead of having this listener in parallel. 750 mNetworkStackClient.registerHealthListener( 751 packageName -> { 752 final VersionedPackage pkg = getVersionedPackage(packageName); 753 if (pkg == null) { 754 Slog.wtf(TAG, "NetworkStack failed but could not find its package"); 755 return; 756 } 757 // This is a severe failure and recovery should be attempted immediately. 758 // TODO: have a better way to handle such failures. 759 final List<VersionedPackage> pkgList = Collections.singletonList(pkg); 760 final long failureCount = getTriggerFailureCount(); 761 for (int i = 0; i < failureCount; i++) { 762 onPackageFailure(pkgList); 763 } 764 }); 765 } 766 767 /** 768 * Persists mAllObservers to file. Threshold information is ignored. 769 */ saveToFile()770 private boolean saveToFile() { 771 Slog.i(TAG, "Saving observer state to file"); 772 synchronized (mLock) { 773 FileOutputStream stream; 774 try { 775 stream = mPolicyFile.startWrite(); 776 } catch (IOException e) { 777 Slog.w(TAG, "Cannot update monitored packages", e); 778 return false; 779 } 780 781 try { 782 XmlSerializer out = new FastXmlSerializer(); 783 out.setOutput(stream, StandardCharsets.UTF_8.name()); 784 out.startDocument(null, true); 785 out.startTag(null, TAG_PACKAGE_WATCHDOG); 786 out.attribute(null, ATTR_VERSION, Integer.toString(DB_VERSION)); 787 for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { 788 mAllObservers.valueAt(oIndex).writeLocked(out); 789 } 790 out.endTag(null, TAG_PACKAGE_WATCHDOG); 791 out.endDocument(); 792 mPolicyFile.finishWrite(stream); 793 return true; 794 } catch (IOException e) { 795 Slog.w(TAG, "Failed to save monitored packages, restoring backup", e); 796 mPolicyFile.failWrite(stream); 797 return false; 798 } finally { 799 IoUtils.closeQuietly(stream); 800 } 801 } 802 } 803 saveToFileAsync()804 private void saveToFileAsync() { 805 if (!mLongTaskHandler.hasCallbacks(this::saveToFile)) { 806 mLongTaskHandler.post(this::saveToFile); 807 } 808 } 809 810 /** 811 * Represents an observer monitoring a set of packages along with the failure thresholds for 812 * each package. 813 * 814 * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing 815 * instances of this class. 816 */ 817 //TODO(b/120598832): Remove 'm' from non-private fields 818 private static class ObserverInternal { 819 public final String mName; 820 //TODO(b/120598832): Add getter for mPackages 821 @GuardedBy("mLock") 822 public final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>(); 823 @Nullable 824 @GuardedBy("mLock") 825 public PackageHealthObserver mRegisteredObserver; 826 ObserverInternal(String name, List<MonitoredPackage> packages)827 ObserverInternal(String name, List<MonitoredPackage> packages) { 828 mName = name; 829 updatePackagesLocked(packages); 830 } 831 832 /** 833 * Writes important {@link MonitoredPackage} details for this observer to file. 834 * Does not persist any package failure thresholds. 835 */ 836 @GuardedBy("mLock") writeLocked(XmlSerializer out)837 public boolean writeLocked(XmlSerializer out) { 838 try { 839 out.startTag(null, TAG_OBSERVER); 840 out.attribute(null, ATTR_NAME, mName); 841 for (int i = 0; i < mPackages.size(); i++) { 842 MonitoredPackage p = mPackages.valueAt(i); 843 p.writeLocked(out); 844 } 845 out.endTag(null, TAG_OBSERVER); 846 return true; 847 } catch (IOException e) { 848 Slog.w(TAG, "Cannot save observer", e); 849 return false; 850 } 851 } 852 853 @GuardedBy("mLock") updatePackagesLocked(List<MonitoredPackage> packages)854 public void updatePackagesLocked(List<MonitoredPackage> packages) { 855 for (int pIndex = 0; pIndex < packages.size(); pIndex++) { 856 MonitoredPackage p = packages.get(pIndex); 857 mPackages.put(p.mName, p); 858 } 859 } 860 861 /** 862 * Reduces the monitoring durations of all packages observed by this observer by 863 * {@code elapsedMs}. If any duration is less than 0, the package is removed from 864 * observation. If any health check duration is less than 0, the health check result 865 * is evaluated. 866 * 867 * @return a {@link Set} of packages that were removed from the observer without explicit 868 * health check passing, or an empty list if no package expired for which an explicit health 869 * check was still pending 870 */ 871 @GuardedBy("mLock") prunePackagesLocked(long elapsedMs)872 private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) { 873 Set<MonitoredPackage> failedPackages = new ArraySet<>(); 874 Iterator<MonitoredPackage> it = mPackages.values().iterator(); 875 while (it.hasNext()) { 876 MonitoredPackage p = it.next(); 877 int oldState = p.getHealthCheckStateLocked(); 878 int newState = p.handleElapsedTimeLocked(elapsedMs); 879 if (oldState != MonitoredPackage.STATE_FAILED 880 && newState == MonitoredPackage.STATE_FAILED) { 881 Slog.i(TAG, "Package " + p.mName + " failed health check"); 882 failedPackages.add(p); 883 } 884 if (p.isExpiredLocked()) { 885 it.remove(); 886 } 887 } 888 return failedPackages; 889 } 890 891 /** 892 * Increments failure counts of {@code packageName}. 893 * @returns {@code true} if failure threshold is exceeded, {@code false} otherwise 894 */ 895 @GuardedBy("mLock") onPackageFailureLocked(String packageName)896 public boolean onPackageFailureLocked(String packageName) { 897 MonitoredPackage p = mPackages.get(packageName); 898 if (p != null) { 899 return p.onFailureLocked(); 900 } 901 return false; 902 } 903 904 /** 905 * Returns one ObserverInternal from the {@code parser} and advances its state. 906 * 907 * <p>Note that this method is <b>not</b> thread safe. It should only be called from 908 * #loadFromFile which in turn is only called on construction of the 909 * singleton PackageWatchdog. 910 **/ read(XmlPullParser parser, PackageWatchdog watchdog)911 public static ObserverInternal read(XmlPullParser parser, PackageWatchdog watchdog) { 912 String observerName = null; 913 if (TAG_OBSERVER.equals(parser.getName())) { 914 observerName = parser.getAttributeValue(null, ATTR_NAME); 915 if (TextUtils.isEmpty(observerName)) { 916 Slog.wtf(TAG, "Unable to read observer name"); 917 return null; 918 } 919 } 920 List<MonitoredPackage> packages = new ArrayList<>(); 921 int innerDepth = parser.getDepth(); 922 try { 923 while (XmlUtils.nextElementWithin(parser, innerDepth)) { 924 if (TAG_PACKAGE.equals(parser.getName())) { 925 try { 926 String packageName = parser.getAttributeValue(null, ATTR_NAME); 927 long duration = Long.parseLong( 928 parser.getAttributeValue(null, ATTR_DURATION)); 929 long healthCheckDuration = Long.parseLong( 930 parser.getAttributeValue(null, 931 ATTR_EXPLICIT_HEALTH_CHECK_DURATION)); 932 boolean hasPassedHealthCheck = Boolean.parseBoolean( 933 parser.getAttributeValue(null, ATTR_PASSED_HEALTH_CHECK)); 934 if (!TextUtils.isEmpty(packageName)) { 935 packages.add(watchdog.new MonitoredPackage(packageName, duration, 936 healthCheckDuration, hasPassedHealthCheck)); 937 } 938 } catch (NumberFormatException e) { 939 Slog.wtf(TAG, "Skipping package for observer " + observerName, e); 940 continue; 941 } 942 } 943 } 944 } catch (XmlPullParserException | IOException e) { 945 Slog.wtf(TAG, "Unable to read observer " + observerName, e); 946 return null; 947 } 948 if (packages.isEmpty()) { 949 return null; 950 } 951 return new ObserverInternal(observerName, packages); 952 } 953 } 954 955 /** 956 * Represents a package and its health check state along with the time 957 * it should be monitored for. 958 * 959 * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing 960 * instances of this class. 961 */ 962 class MonitoredPackage { 963 // Health check states 964 // TODO(b/120598832): Prefix with HEALTH_CHECK 965 // mName has not passed health check but has requested a health check 966 public static final int STATE_ACTIVE = 0; 967 // mName has not passed health check and has not requested a health check 968 public static final int STATE_INACTIVE = 1; 969 // mName has passed health check 970 public static final int STATE_PASSED = 2; 971 // mName has failed health check 972 public static final int STATE_FAILED = 3; 973 974 //TODO(b/120598832): VersionedPackage? 975 private final String mName; 976 // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after 977 // methods that could change the health check state: handleElapsedTimeLocked and 978 // tryPassHealthCheckLocked 979 private int mHealthCheckState = STATE_INACTIVE; 980 // Whether an explicit health check has passed. 981 // This value in addition with mHealthCheckDurationMs determines the health check state 982 // of the package, see #getHealthCheckStateLocked 983 @GuardedBy("mLock") 984 private boolean mHasPassedHealthCheck; 985 // System uptime duration to monitor package. 986 @GuardedBy("mLock") 987 private long mDurationMs; 988 // System uptime duration to check the result of an explicit health check 989 // Initially, MAX_VALUE until we get a value from the health check service 990 // and request health checks. 991 // This value in addition with mHasPassedHealthCheck determines the health check state 992 // of the package, see #getHealthCheckStateLocked 993 @GuardedBy("mLock") 994 private long mHealthCheckDurationMs = Long.MAX_VALUE; 995 // System uptime of first package failure 996 @GuardedBy("mLock") 997 private long mUptimeStartMs; 998 // Number of failures since mUptimeStartMs 999 @GuardedBy("mLock") 1000 private int mFailures; 1001 MonitoredPackage(String name, long durationMs, boolean hasPassedHealthCheck)1002 MonitoredPackage(String name, long durationMs, boolean hasPassedHealthCheck) { 1003 this(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck); 1004 } 1005 MonitoredPackage(String name, long durationMs, long healthCheckDurationMs, boolean hasPassedHealthCheck)1006 MonitoredPackage(String name, long durationMs, long healthCheckDurationMs, 1007 boolean hasPassedHealthCheck) { 1008 mName = name; 1009 mDurationMs = durationMs; 1010 mHealthCheckDurationMs = healthCheckDurationMs; 1011 mHasPassedHealthCheck = hasPassedHealthCheck; 1012 updateHealthCheckStateLocked(); 1013 } 1014 1015 /** Writes the salient fields to disk using {@code out}. */ 1016 @GuardedBy("mLock") writeLocked(XmlSerializer out)1017 public void writeLocked(XmlSerializer out) throws IOException { 1018 out.startTag(null, TAG_PACKAGE); 1019 out.attribute(null, ATTR_NAME, mName); 1020 out.attribute(null, ATTR_DURATION, String.valueOf(mDurationMs)); 1021 out.attribute(null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION, 1022 String.valueOf(mHealthCheckDurationMs)); 1023 out.attribute(null, ATTR_PASSED_HEALTH_CHECK, 1024 String.valueOf(mHasPassedHealthCheck)); 1025 out.endTag(null, TAG_PACKAGE); 1026 } 1027 1028 /** 1029 * Increment package failures or resets failure count depending on the last package failure. 1030 * 1031 * @return {@code true} if failure count exceeds a threshold, {@code false} otherwise 1032 */ 1033 @GuardedBy("mLock") onFailureLocked()1034 public boolean onFailureLocked() { 1035 final long now = SystemClock.uptimeMillis(); 1036 final long duration = now - mUptimeStartMs; 1037 if (duration > mTriggerFailureDurationMs) { 1038 // TODO(b/120598832): Reseting to 1 is not correct 1039 // because there may be more than 1 failure in the last trigger window from now 1040 // This is the RescueParty impl, will leave for now 1041 mFailures = 1; 1042 mUptimeStartMs = now; 1043 } else { 1044 mFailures++; 1045 } 1046 boolean failed = mFailures >= mTriggerFailureCount; 1047 if (failed) { 1048 mFailures = 0; 1049 } 1050 return failed; 1051 } 1052 1053 /** 1054 * Sets the initial health check duration. 1055 * 1056 * @return the new health check state 1057 */ 1058 @GuardedBy("mLock") setHealthCheckActiveLocked(long initialHealthCheckDurationMs)1059 public int setHealthCheckActiveLocked(long initialHealthCheckDurationMs) { 1060 if (initialHealthCheckDurationMs <= 0) { 1061 Slog.wtf(TAG, "Cannot set non-positive health check duration " 1062 + initialHealthCheckDurationMs + "ms for package " + mName 1063 + ". Using total duration " + mDurationMs + "ms instead"); 1064 initialHealthCheckDurationMs = mDurationMs; 1065 } 1066 if (mHealthCheckState == STATE_INACTIVE) { 1067 // Transitions to ACTIVE 1068 mHealthCheckDurationMs = initialHealthCheckDurationMs; 1069 } 1070 return updateHealthCheckStateLocked(); 1071 } 1072 1073 /** 1074 * Updates the monitoring durations of the package. 1075 * 1076 * @return the new health check state 1077 */ 1078 @GuardedBy("mLock") handleElapsedTimeLocked(long elapsedMs)1079 public int handleElapsedTimeLocked(long elapsedMs) { 1080 if (elapsedMs <= 0) { 1081 Slog.w(TAG, "Cannot handle non-positive elapsed time for package " + mName); 1082 return mHealthCheckState; 1083 } 1084 // Transitions to FAILED if now <= 0 and health check not passed 1085 mDurationMs -= elapsedMs; 1086 if (mHealthCheckState == STATE_ACTIVE) { 1087 // We only update health check durations if we have #setHealthCheckActiveLocked 1088 // This ensures we don't leave the INACTIVE state for an unexpected elapsed time 1089 // Transitions to FAILED if now <= 0 and health check not passed 1090 mHealthCheckDurationMs -= elapsedMs; 1091 } 1092 return updateHealthCheckStateLocked(); 1093 } 1094 1095 /** 1096 * Marks the health check as passed and transitions to {@link #STATE_PASSED} 1097 * if not yet {@link #STATE_FAILED}. 1098 * 1099 * @return the new health check state 1100 */ 1101 @GuardedBy("mLock") tryPassHealthCheckLocked()1102 public int tryPassHealthCheckLocked() { 1103 if (mHealthCheckState != STATE_FAILED) { 1104 // FAILED is a final state so only pass if we haven't failed 1105 // Transition to PASSED 1106 mHasPassedHealthCheck = true; 1107 } 1108 return updateHealthCheckStateLocked(); 1109 } 1110 1111 /** Returns the monitored package name. */ getName()1112 private String getName() { 1113 return mName; 1114 } 1115 1116 //TODO(b/120598832): IntDef 1117 /** 1118 * Returns the current health check state, any of {@link #STATE_ACTIVE}, 1119 * {@link #STATE_INACTIVE} or {@link #STATE_PASSED} 1120 */ 1121 @GuardedBy("mLock") getHealthCheckStateLocked()1122 public int getHealthCheckStateLocked() { 1123 return mHealthCheckState; 1124 } 1125 1126 /** 1127 * Returns the shortest duration before the package should be scheduled for a prune. 1128 * 1129 * @return the duration or {@link Long#MAX_VALUE} if the package should not be scheduled 1130 */ 1131 @GuardedBy("mLock") getShortestScheduleDurationMsLocked()1132 public long getShortestScheduleDurationMsLocked() { 1133 // Consider health check duration only if #isPendingHealthChecksLocked is true 1134 return Math.min(toPositive(mDurationMs), 1135 isPendingHealthChecksLocked() 1136 ? toPositive(mHealthCheckDurationMs) : Long.MAX_VALUE); 1137 } 1138 1139 /** 1140 * Returns {@code true} if the total duration left to monitor the package is less than or 1141 * equal to 0 {@code false} otherwise. 1142 */ 1143 @GuardedBy("mLock") isExpiredLocked()1144 public boolean isExpiredLocked() { 1145 return mDurationMs <= 0; 1146 } 1147 1148 /** 1149 * Returns {@code true} if the package, {@link #getName} is expecting health check results 1150 * {@code false} otherwise. 1151 */ 1152 @GuardedBy("mLock") isPendingHealthChecksLocked()1153 public boolean isPendingHealthChecksLocked() { 1154 return mHealthCheckState == STATE_ACTIVE || mHealthCheckState == STATE_INACTIVE; 1155 } 1156 1157 /** 1158 * Updates the health check state based on {@link #mHasPassedHealthCheck} 1159 * and {@link #mHealthCheckDurationMs}. 1160 * 1161 * @return the new health check state 1162 */ 1163 @GuardedBy("mLock") updateHealthCheckStateLocked()1164 private int updateHealthCheckStateLocked() { 1165 int oldState = mHealthCheckState; 1166 if (mHasPassedHealthCheck) { 1167 // Set final state first to avoid ambiguity 1168 mHealthCheckState = STATE_PASSED; 1169 } else if (mHealthCheckDurationMs <= 0 || mDurationMs <= 0) { 1170 // Set final state first to avoid ambiguity 1171 mHealthCheckState = STATE_FAILED; 1172 } else if (mHealthCheckDurationMs == Long.MAX_VALUE) { 1173 mHealthCheckState = STATE_INACTIVE; 1174 } else { 1175 mHealthCheckState = STATE_ACTIVE; 1176 } 1177 Slog.i(TAG, "Updated health check state for package " + mName + ": " 1178 + toString(oldState) + " -> " + toString(mHealthCheckState)); 1179 return mHealthCheckState; 1180 } 1181 1182 /** Returns a {@link String} representation of the current health check state. */ toString(int state)1183 private String toString(int state) { 1184 switch (state) { 1185 case STATE_ACTIVE: 1186 return "ACTIVE"; 1187 case STATE_INACTIVE: 1188 return "INACTIVE"; 1189 case STATE_PASSED: 1190 return "PASSED"; 1191 case STATE_FAILED: 1192 return "FAILED"; 1193 default: 1194 return "UNKNOWN"; 1195 } 1196 } 1197 1198 /** Returns {@code value} if it is greater than 0 or {@link Long#MAX_VALUE} otherwise. */ toPositive(long value)1199 private long toPositive(long value) { 1200 return value > 0 ? value : Long.MAX_VALUE; 1201 } 1202 } 1203 } 1204