1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.classifier; 18 19 import static com.android.systemui.classifier.Classifier.BACK_GESTURE; 20 import static com.android.systemui.classifier.Classifier.GENERIC; 21 import static com.android.systemui.classifier.Classifier.MEDIA_SEEKBAR; 22 import static com.android.systemui.classifier.FalsingManagerProxy.FALSING_SUCCESS; 23 import static com.android.systemui.classifier.FalsingModule.BRIGHT_LINE_GESTURE_CLASSIFERS; 24 25 import android.net.Uri; 26 import android.os.Build; 27 import android.util.IndentingPrintWriter; 28 import android.util.Log; 29 import android.view.accessibility.AccessibilityManager; 30 31 import androidx.annotation.NonNull; 32 33 import com.android.internal.logging.MetricsLogger; 34 import com.android.systemui.classifier.FalsingDataProvider.SessionListener; 35 import com.android.systemui.classifier.HistoryTracker.BeliefListener; 36 import com.android.systemui.dagger.qualifiers.TestHarness; 37 import com.android.systemui.flags.FeatureFlags; 38 import com.android.systemui.flags.Flags; 39 import com.android.systemui.plugins.FalsingManager; 40 import com.android.systemui.statusbar.policy.KeyguardStateController; 41 42 import java.io.PrintWriter; 43 import java.util.ArrayDeque; 44 import java.util.ArrayList; 45 import java.util.Collection; 46 import java.util.Collections; 47 import java.util.List; 48 import java.util.Queue; 49 import java.util.Set; 50 import java.util.StringJoiner; 51 import java.util.stream.Collectors; 52 53 import javax.inject.Inject; 54 import javax.inject.Named; 55 56 /** 57 * FalsingManager designed to make clear why a touch was rejected. 58 */ 59 public class BrightLineFalsingManager implements FalsingManager { 60 61 private static final String TAG = "FalsingManager"; 62 public static final boolean DEBUG = Log.isLoggable(TAG, Log.DEBUG); 63 64 private static final int RECENT_INFO_LOG_SIZE = 40; 65 private static final int RECENT_SWIPE_LOG_SIZE = 20; 66 private static final double TAP_CONFIDENCE_THRESHOLD = 0.7; 67 private static final double FALSE_BELIEF_THRESHOLD = 0.9; 68 69 private final FalsingDataProvider mDataProvider; 70 private final LongTapClassifier mLongTapClassifier; 71 private final SingleTapClassifier mSingleTapClassifier; 72 private final DoubleTapClassifier mDoubleTapClassifier; 73 private final HistoryTracker mHistoryTracker; 74 private final KeyguardStateController mKeyguardStateController; 75 private AccessibilityManager mAccessibilityManager; 76 private final boolean mTestHarness; 77 private final MetricsLogger mMetricsLogger; 78 private int mIsFalseTouchCalls; 79 private FeatureFlags mFeatureFlags; 80 private static final Queue<String> RECENT_INFO_LOG = 81 new ArrayDeque<>(RECENT_INFO_LOG_SIZE + 1); 82 private static final Queue<DebugSwipeRecord> RECENT_SWIPES = 83 new ArrayDeque<>(RECENT_SWIPE_LOG_SIZE + 1); 84 85 private final Collection<FalsingClassifier> mClassifiers; 86 private final List<FalsingBeliefListener> mFalsingBeliefListeners = new ArrayList<>(); 87 private List<FalsingTapListener> mFalsingTapListeners = new ArrayList<>(); 88 private ProximityEvent mLastProximityEvent; 89 90 private boolean mDestroyed; 91 92 private final SessionListener mSessionListener = new SessionListener() { 93 @Override 94 public void onSessionEnded() { 95 mLastProximityEvent = null; 96 mClassifiers.forEach(FalsingClassifier::onSessionEnded); 97 } 98 99 @Override 100 public void onSessionStarted() { 101 mClassifiers.forEach(FalsingClassifier::onSessionStarted); 102 } 103 }; 104 105 private final BeliefListener mBeliefListener = new BeliefListener() { 106 @Override 107 public void onBeliefChanged(double belief) { 108 logInfo(String.format( 109 "{belief=%s confidence=%s}", 110 mHistoryTracker.falseBelief(), 111 mHistoryTracker.falseConfidence())); 112 if (belief > FALSE_BELIEF_THRESHOLD) { 113 mFalsingBeliefListeners.forEach(FalsingBeliefListener::onFalse); 114 logInfo("Triggering False Event (Threshold: " + FALSE_BELIEF_THRESHOLD + ")"); 115 } 116 } 117 }; 118 119 private final FalsingDataProvider.GestureFinalizedListener mGestureFinalizedListener = 120 new FalsingDataProvider.GestureFinalizedListener() { 121 @Override 122 public void onGestureFinalized(long completionTimeMs) { 123 if (mPriorResults != null) { 124 boolean boolResult = mPriorResults.stream().anyMatch( 125 FalsingClassifier.Result::isFalse); 126 127 mPriorResults.forEach(result -> { 128 if (result.isFalse()) { 129 String reason = result.getReason(); 130 if (reason != null) { 131 logInfo(reason); 132 } 133 } 134 }); 135 136 if (Build.IS_ENG || Build.IS_USERDEBUG) { 137 // Copy motion events, as the results returned by 138 // #getRecentMotionEvents are recycled elsewhere. 139 RECENT_SWIPES.add(new DebugSwipeRecord( 140 boolResult, 141 mPriorInteractionType, 142 mDataProvider.getRecentMotionEvents().stream().map( 143 motionEvent -> new XYDt( 144 (int) motionEvent.getX(), 145 (int) motionEvent.getY(), 146 (int) (motionEvent.getEventTime() 147 - motionEvent.getDownTime()))) 148 .collect(Collectors.toList()))); 149 while (RECENT_SWIPES.size() > RECENT_INFO_LOG_SIZE) { 150 RECENT_SWIPES.remove(); 151 } 152 } 153 154 155 mHistoryTracker.addResults(mPriorResults, completionTimeMs); 156 mPriorResults = null; 157 mPriorInteractionType = Classifier.GENERIC; 158 } else { 159 // Gestures that were not classified get treated as a false. 160 // Gestures that look like simple taps are less likely to be false 161 // than swipes. They may simply be mis-clicks. 162 double penalty = mSingleTapClassifier.isTap( 163 mDataProvider.getRecentMotionEvents(), 0).isFalse() 164 ? 0.7 : 0.8; 165 mHistoryTracker.addResults( 166 Collections.singleton( 167 FalsingClassifier.Result.falsed( 168 penalty, getClass().getSimpleName(), 169 "unclassified")), 170 completionTimeMs); 171 } 172 } 173 }; 174 175 private Collection<FalsingClassifier.Result> mPriorResults; 176 private @Classifier.InteractionType int mPriorInteractionType = Classifier.GENERIC; 177 178 @Inject BrightLineFalsingManager( FalsingDataProvider falsingDataProvider, MetricsLogger metricsLogger, @Named(BRIGHT_LINE_GESTURE_CLASSIFERS) Set<FalsingClassifier> classifiers, SingleTapClassifier singleTapClassifier, LongTapClassifier longTapClassifier, DoubleTapClassifier doubleTapClassifier, HistoryTracker historyTracker, KeyguardStateController keyguardStateController, AccessibilityManager accessibilityManager, @TestHarness boolean testHarness, FeatureFlags featureFlags)179 public BrightLineFalsingManager( 180 FalsingDataProvider falsingDataProvider, 181 MetricsLogger metricsLogger, 182 @Named(BRIGHT_LINE_GESTURE_CLASSIFERS) Set<FalsingClassifier> classifiers, 183 SingleTapClassifier singleTapClassifier, LongTapClassifier longTapClassifier, 184 DoubleTapClassifier doubleTapClassifier, HistoryTracker historyTracker, 185 KeyguardStateController keyguardStateController, 186 AccessibilityManager accessibilityManager, 187 @TestHarness boolean testHarness, 188 FeatureFlags featureFlags) { 189 mDataProvider = falsingDataProvider; 190 mMetricsLogger = metricsLogger; 191 mClassifiers = classifiers; 192 mSingleTapClassifier = singleTapClassifier; 193 mLongTapClassifier = longTapClassifier; 194 mDoubleTapClassifier = doubleTapClassifier; 195 mHistoryTracker = historyTracker; 196 mKeyguardStateController = keyguardStateController; 197 mAccessibilityManager = accessibilityManager; 198 mTestHarness = testHarness; 199 mFeatureFlags = featureFlags; 200 201 mDataProvider.addSessionListener(mSessionListener); 202 mDataProvider.addGestureCompleteListener(mGestureFinalizedListener); 203 mHistoryTracker.addBeliefListener(mBeliefListener); 204 } 205 206 @Override isClassifierEnabled()207 public boolean isClassifierEnabled() { 208 return true; 209 } 210 211 @Override isFalseTouch(@lassifier.InteractionType int interactionType)212 public boolean isFalseTouch(@Classifier.InteractionType int interactionType) { 213 checkDestroyed(); 214 215 mPriorInteractionType = interactionType; 216 if (skipFalsing(interactionType)) { 217 mPriorResults = getPassedResult(1); 218 logDebug("Skipped falsing"); 219 return false; 220 } 221 222 final boolean[] localResult = {false}; 223 mPriorResults = mClassifiers.stream().map(falsingClassifier -> { 224 FalsingClassifier.Result r = falsingClassifier.classifyGesture( 225 interactionType, 226 mHistoryTracker.falseBelief(), 227 mHistoryTracker.falseConfidence()); 228 localResult[0] |= r.isFalse(); 229 230 return r; 231 }).collect(Collectors.toList()); 232 233 // check for false tap if it is a seekbar interaction 234 if (interactionType == MEDIA_SEEKBAR) { 235 localResult[0] &= isFalseTap(mFeatureFlags.isEnabled(Flags.MEDIA_FALSING_PENALTY) 236 ? FalsingManager.MODERATE_PENALTY : FalsingManager.LOW_PENALTY); 237 } 238 239 logDebug("False Gesture (type: " + interactionType + "): " + localResult[0]); 240 241 return localResult[0]; 242 } 243 244 @Override isSimpleTap()245 public boolean isSimpleTap() { 246 checkDestroyed(); 247 248 FalsingClassifier.Result result = mSingleTapClassifier.isTap( 249 mDataProvider.getRecentMotionEvents(), 0); 250 mPriorResults = Collections.singleton(result); 251 252 return !result.isFalse(); 253 } 254 checkDestroyed()255 private void checkDestroyed() { 256 if (mDestroyed) { 257 Log.wtf(TAG, "Tried to use FalsingManager after being destroyed!"); 258 } 259 } 260 261 @Override isFalseTap(@enalty int penalty)262 public boolean isFalseTap(@Penalty int penalty) { 263 checkDestroyed(); 264 265 if (skipFalsing(GENERIC)) { 266 mPriorResults = getPassedResult(1); 267 logDebug("Skipped falsing"); 268 return false; 269 } 270 271 double falsePenalty = 0; 272 switch(penalty) { 273 case NO_PENALTY: 274 falsePenalty = 0; 275 break; 276 case LOW_PENALTY: 277 falsePenalty = 0.1; 278 break; 279 case MODERATE_PENALTY: 280 falsePenalty = 0.3; 281 break; 282 case HIGH_PENALTY: 283 falsePenalty = 0.6; 284 break; 285 } 286 287 FalsingClassifier.Result singleTapResult = 288 mSingleTapClassifier.isTap(mDataProvider.getRecentMotionEvents().isEmpty() 289 ? mDataProvider.getPriorMotionEvents() 290 : mDataProvider.getRecentMotionEvents(), falsePenalty); 291 mPriorResults = Collections.singleton(singleTapResult); 292 293 if (!singleTapResult.isFalse()) { 294 if (mDataProvider.isJustUnlockedWithFace()) { 295 // Immediately pass if a face is detected. 296 mPriorResults = getPassedResult(1); 297 logDebug("False Single Tap: false (face detected)"); 298 return false; 299 } else if (!isFalseDoubleTap()) { 300 // We must check double tapping before other heuristics. This is because 301 // the double tap will fail if there's only been one tap. We don't want that 302 // failure to be recorded in mPriorResults. 303 logDebug("False Single Tap: false (double tapped)"); 304 return false; 305 } else if (mHistoryTracker.falseBelief() > TAP_CONFIDENCE_THRESHOLD) { 306 mPriorResults = Collections.singleton( 307 FalsingClassifier.Result.falsed( 308 0, getClass().getSimpleName(), "bad history")); 309 logDebug("False Single Tap: true (bad history)"); 310 mFalsingTapListeners.forEach(FalsingTapListener::onAdditionalTapRequired); 311 return true; 312 } else { 313 mPriorResults = getPassedResult(0.1); 314 logDebug("False Single Tap: false (default)"); 315 return false; 316 } 317 318 } else { 319 logDebug("False Single Tap: " + singleTapResult.isFalse() + " (simple)"); 320 return singleTapResult.isFalse(); 321 } 322 323 } 324 325 @Override isFalseLongTap(@enalty int penalty)326 public boolean isFalseLongTap(@Penalty int penalty) { 327 if (!mFeatureFlags.isEnabled(Flags.FALSING_FOR_LONG_TAPS)) { 328 return false; 329 } 330 331 checkDestroyed(); 332 333 if (skipFalsing(GENERIC)) { 334 mPriorResults = getPassedResult(1); 335 logDebug("Skipped falsing"); 336 return false; 337 } 338 339 double falsePenalty = 0; 340 switch(penalty) { 341 case NO_PENALTY: 342 falsePenalty = 0; 343 break; 344 case LOW_PENALTY: 345 falsePenalty = 0.1; 346 break; 347 case MODERATE_PENALTY: 348 falsePenalty = 0.3; 349 break; 350 case HIGH_PENALTY: 351 falsePenalty = 0.6; 352 break; 353 } 354 355 FalsingClassifier.Result longTapResult = 356 mLongTapClassifier.isTap(mDataProvider.getRecentMotionEvents().isEmpty() 357 ? mDataProvider.getPriorMotionEvents() 358 : mDataProvider.getRecentMotionEvents(), falsePenalty); 359 mPriorResults = Collections.singleton(longTapResult); 360 361 if (!longTapResult.isFalse()) { 362 if (mDataProvider.isJustUnlockedWithFace()) { 363 // Immediately pass if a face is detected. 364 mPriorResults = getPassedResult(1); 365 logDebug("False Long Tap: false (face detected)"); 366 } else { 367 mPriorResults = getPassedResult(0.1); 368 logDebug("False Long Tap: false (default)"); 369 } 370 return false; 371 } else { 372 logDebug("False Long Tap: " + longTapResult.isFalse() + " (simple)"); 373 return longTapResult.isFalse(); 374 } 375 } 376 377 @Override isFalseDoubleTap()378 public boolean isFalseDoubleTap() { 379 checkDestroyed(); 380 381 if (skipFalsing(GENERIC)) { 382 mPriorResults = getPassedResult(1); 383 logDebug("Skipped falsing"); 384 return false; 385 } 386 387 FalsingClassifier.Result result = mDoubleTapClassifier.classifyGesture( 388 Classifier.GENERIC, 389 mHistoryTracker.falseBelief(), 390 mHistoryTracker.falseConfidence()); 391 mPriorResults = Collections.singleton(result); 392 logDebug("False Double Tap: " + result.isFalse() + " reason=" + result.getReason()); 393 return result.isFalse(); 394 } 395 skipFalsing(@lassifier.InteractionType int interactionType)396 private boolean skipFalsing(@Classifier.InteractionType int interactionType) { 397 return interactionType == BACK_GESTURE 398 || !mKeyguardStateController.isShowing() 399 || mTestHarness 400 || mDataProvider.isJustUnlockedWithFace() 401 || mDataProvider.isDocked() 402 || mAccessibilityManager.isTouchExplorationEnabled() 403 || mDataProvider.isA11yAction() 404 || (mFeatureFlags.isEnabled(Flags.FALSING_OFF_FOR_UNFOLDED) 405 && mDataProvider.isUnfolded()); 406 } 407 408 @Override onProximityEvent(ProximityEvent proximityEvent)409 public void onProximityEvent(ProximityEvent proximityEvent) { 410 // TODO: some of these classifiers might allow us to abort early, meaning we don't have to 411 // make these calls. 412 mLastProximityEvent = proximityEvent; 413 mClassifiers.forEach((classifier) -> classifier.onProximityEvent(proximityEvent)); 414 } 415 416 @Override onSuccessfulUnlock()417 public void onSuccessfulUnlock() { 418 if (mIsFalseTouchCalls != 0) { 419 mMetricsLogger.histogram(FALSING_SUCCESS, mIsFalseTouchCalls); 420 mIsFalseTouchCalls = 0; 421 } 422 } 423 424 @Override isProximityNear()425 public boolean isProximityNear() { 426 return mLastProximityEvent != null && mLastProximityEvent.getCovered(); 427 } 428 429 @Override isUnlockingDisabled()430 public boolean isUnlockingDisabled() { 431 return false; 432 } 433 434 @Override shouldEnforceBouncer()435 public boolean shouldEnforceBouncer() { 436 return false; 437 } 438 439 @Override reportRejectedTouch()440 public Uri reportRejectedTouch() { 441 return null; 442 } 443 444 @Override isReportingEnabled()445 public boolean isReportingEnabled() { 446 return false; 447 } 448 449 @Override addFalsingBeliefListener(FalsingBeliefListener listener)450 public void addFalsingBeliefListener(FalsingBeliefListener listener) { 451 mFalsingBeliefListeners.add(listener); 452 } 453 454 @Override removeFalsingBeliefListener(FalsingBeliefListener listener)455 public void removeFalsingBeliefListener(FalsingBeliefListener listener) { 456 mFalsingBeliefListeners.remove(listener); 457 } 458 459 @Override addTapListener(FalsingTapListener listener)460 public void addTapListener(FalsingTapListener listener) { 461 mFalsingTapListeners.add(listener); 462 } 463 464 @Override removeTapListener(FalsingTapListener listener)465 public void removeTapListener(FalsingTapListener listener) { 466 mFalsingTapListeners.remove(listener); 467 } 468 469 @Override dump(@onNull PrintWriter pw, @NonNull String[] args)470 public void dump(@NonNull PrintWriter pw, @NonNull String[] args) { 471 IndentingPrintWriter ipw = new IndentingPrintWriter(pw, " "); 472 ipw.println("BRIGHTLINE FALSING MANAGER"); 473 ipw.print("classifierEnabled="); 474 ipw.println(isClassifierEnabled() ? 1 : 0); 475 ipw.print("mJustUnlockedWithFace="); 476 ipw.println(mDataProvider.isJustUnlockedWithFace() ? 1 : 0); 477 ipw.print("isDocked="); 478 ipw.println(mDataProvider.isDocked() ? 1 : 0); 479 ipw.print("width="); 480 ipw.println(mDataProvider.getWidthPixels()); 481 ipw.print("height="); 482 ipw.println(mDataProvider.getHeightPixels()); 483 ipw.println(); 484 if (RECENT_SWIPES.size() != 0) { 485 ipw.println("Recent swipes:"); 486 ipw.increaseIndent(); 487 for (DebugSwipeRecord record : RECENT_SWIPES) { 488 ipw.println(record.getString()); 489 ipw.println(); 490 } 491 ipw.decreaseIndent(); 492 } else { 493 ipw.println("No recent swipes"); 494 } 495 ipw.println(); 496 ipw.println("Recent falsing info:"); 497 ipw.increaseIndent(); 498 for (String msg : RECENT_INFO_LOG) { 499 ipw.println(msg); 500 } 501 ipw.println(); 502 } 503 504 @Override cleanupInternal()505 public void cleanupInternal() { 506 mDestroyed = true; 507 mDataProvider.removeSessionListener(mSessionListener); 508 mDataProvider.removeGestureCompleteListener(mGestureFinalizedListener); 509 mClassifiers.forEach(FalsingClassifier::cleanup); 510 mFalsingBeliefListeners.clear(); 511 mHistoryTracker.removeBeliefListener(mBeliefListener); 512 } 513 getPassedResult(double confidence)514 private static Collection<FalsingClassifier.Result> getPassedResult(double confidence) { 515 return Collections.singleton(FalsingClassifier.Result.passed(confidence)); 516 } 517 logDebug(String msg)518 static void logDebug(String msg) { 519 logDebug(msg, null); 520 } 521 logDebug(String msg, Throwable throwable)522 static void logDebug(String msg, Throwable throwable) { 523 if (DEBUG) { 524 Log.d(TAG, msg, throwable); 525 } 526 } 527 logVerbose(String msg)528 static void logVerbose(String msg) { 529 if (DEBUG) { 530 Log.v(TAG, msg); 531 } 532 } 533 logInfo(String msg)534 static void logInfo(String msg) { 535 Log.i(TAG, msg); 536 RECENT_INFO_LOG.add(msg); 537 while (RECENT_INFO_LOG.size() > RECENT_INFO_LOG_SIZE) { 538 RECENT_INFO_LOG.remove(); 539 } 540 } 541 logError(String msg)542 static void logError(String msg) { 543 Log.e(TAG, msg); 544 } 545 546 private static class DebugSwipeRecord { 547 private static final byte VERSION = 1; // opaque version number indicating format of data. 548 private final boolean mIsFalse; 549 private final int mInteractionType; 550 private final List<XYDt> mRecentMotionEvents; 551 DebugSwipeRecord(boolean isFalse, int interactionType, List<XYDt> recentMotionEvents)552 DebugSwipeRecord(boolean isFalse, int interactionType, 553 List<XYDt> recentMotionEvents) { 554 mIsFalse = isFalse; 555 mInteractionType = interactionType; 556 mRecentMotionEvents = recentMotionEvents; 557 } 558 getString()559 String getString() { 560 StringJoiner sj = new StringJoiner(","); 561 sj.add(Integer.toString(VERSION)) 562 .add(mIsFalse ? "1" : "0") 563 .add(Integer.toString(mInteractionType)); 564 for (XYDt event : mRecentMotionEvents) { 565 sj.add(event.toString()); 566 } 567 return sj.toString(); 568 } 569 } 570 571 private static class XYDt { 572 private final int mX; 573 private final int mY; 574 private final int mDT; 575 XYDt(int x, int y, int dT)576 XYDt(int x, int y, int dT) { 577 mX = x; 578 mY = y; 579 mDT = dT; 580 } 581 582 @Override toString()583 public String toString() { 584 return mX + "," + mY + "," + mDT; 585 } 586 } 587 } 588