1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.wm; 18 19 import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER; 20 21 import static com.android.server.wm.utils.RegionUtils.forEachRect; 22 23 import android.annotation.NonNull; 24 import android.graphics.Matrix; 25 import android.graphics.Rect; 26 import android.graphics.RectF; 27 import android.graphics.Region; 28 import android.os.Handler; 29 import android.os.IBinder; 30 import android.os.InputConfig; 31 import android.os.Looper; 32 import android.os.Message; 33 import android.util.Pair; 34 import android.util.Slog; 35 import android.util.SparseArray; 36 import android.view.IWindow; 37 import android.view.InputWindowHandle; 38 import android.view.MagnificationSpec; 39 import android.view.WindowInfo; 40 import android.view.WindowManager; 41 import android.window.WindowInfosListener; 42 43 import com.android.internal.annotations.GuardedBy; 44 45 import java.util.ArrayList; 46 import java.util.HashMap; 47 import java.util.List; 48 import java.util.Map; 49 50 /** 51 * This class is the accessibility windows population adapter. 52 */ 53 public final class AccessibilityWindowsPopulator extends WindowInfosListener { 54 55 private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName(); 56 // If the surface flinger callback is not coming within in 2 frames time, i.e. about 57 // 35ms, then assuming the windows become stable. 58 private static final int SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS = 35; 59 // To avoid the surface flinger callbacks always comes within in 2 frames, then no windows 60 // are reported to the A11y framework, and the animation duration time is 500ms, so setting 61 // this value as the max timeout value to force computing changed windows. 62 private static final int WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS = 500; 63 64 private static final float[] sTempFloats = new float[9]; 65 66 private final WindowManagerService mService; 67 private final AccessibilityController mAccessibilityController; 68 @GuardedBy("mLock") 69 private final SparseArray<List<InputWindowHandle>> mInputWindowHandlesOnDisplays = 70 new SparseArray<>(); 71 @GuardedBy("mLock") 72 private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>(); 73 @GuardedBy("mLock") 74 private final SparseArray<DisplayInfo> mDisplayInfos = new SparseArray<>(); 75 private final SparseArray<MagnificationSpec> mCurrentMagnificationSpec = new SparseArray<>(); 76 @GuardedBy("mLock") 77 private final SparseArray<MagnificationSpec> mPreviousMagnificationSpec = new SparseArray<>(); 78 @GuardedBy("mLock") 79 private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>(); 80 @GuardedBy("mLock") 81 private boolean mWindowsNotificationEnabled = false; 82 @GuardedBy("mLock") 83 private final Map<IBinder, Matrix> mWindowsTransformMatrixMap = new HashMap<>(); 84 private final Object mLock = new Object(); 85 private final Handler mHandler; 86 87 private final Matrix mTempMatrix1 = new Matrix(); 88 private final Matrix mTempMatrix2 = new Matrix(); 89 private final float[] mTempFloat1 = new float[9]; 90 private final float[] mTempFloat2 = new float[9]; 91 private final float[] mTempFloat3 = new float[9]; 92 AccessibilityWindowsPopulator(WindowManagerService service, AccessibilityController accessibilityController)93 AccessibilityWindowsPopulator(WindowManagerService service, 94 AccessibilityController accessibilityController) { 95 mService = service; 96 mAccessibilityController = accessibilityController; 97 mHandler = new MyHandler(mService.mH.getLooper()); 98 } 99 100 /** 101 * Gets the visible windows list with the window layer on the specified display. 102 * 103 * @param displayId The display. 104 * @param outWindows The visible windows list. The z-order of each window in the list 105 * is from the top to bottom. 106 */ populateVisibleWindowsOnScreenLocked(int displayId, List<AccessibilityWindow> outWindows)107 public void populateVisibleWindowsOnScreenLocked(int displayId, 108 List<AccessibilityWindow> outWindows) { 109 List<InputWindowHandle> inputWindowHandles; 110 final Matrix inverseMatrix = new Matrix(); 111 final Matrix displayMatrix = new Matrix(); 112 113 synchronized (mLock) { 114 inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId); 115 if (inputWindowHandles == null) { 116 outWindows.clear(); 117 118 return; 119 } 120 inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); 121 122 final DisplayInfo displayInfo = mDisplayInfos.get(displayId); 123 if (displayInfo != null) { 124 displayMatrix.set(displayInfo.mTransform); 125 } else { 126 Slog.w(TAG, "The displayInfo of this displayId (" + displayId + ") called " 127 + "back from the surface fligner is null"); 128 } 129 } 130 131 final DisplayContent dc = mService.mRoot.getDisplayContent(displayId); 132 final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP); 133 final IBinder pipMenuIBinder = 134 shellroot != null ? shellroot.getAccessibilityWindowToken() : null; 135 136 for (final InputWindowHandle windowHandle : inputWindowHandles) { 137 final AccessibilityWindow accessibilityWindow = 138 AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix, 139 pipMenuIBinder, displayMatrix); 140 141 outWindows.add(accessibilityWindow); 142 } 143 } 144 145 @Override onWindowInfosChanged(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos)146 public void onWindowInfosChanged(InputWindowHandle[] windowHandles, 147 DisplayInfo[] displayInfos) { 148 mHandler.post(() -> onWindowInfosChangedInternal(windowHandles, displayInfos)); 149 } 150 onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos)151 private void onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, 152 DisplayInfo[] displayInfos) { 153 final List<InputWindowHandle> tempVisibleWindows = new ArrayList<>(); 154 155 for (InputWindowHandle window : windowHandles) { 156 final boolean visible = (window.inputConfig & InputConfig.NOT_VISIBLE) == 0; 157 if (visible && window.getWindow() != null && !window.isClone) { 158 tempVisibleWindows.add(window); 159 } 160 } 161 final HashMap<IBinder, Matrix> windowsTransformMatrixMap = 162 getWindowsTransformMatrix(tempVisibleWindows); 163 164 synchronized (mLock) { 165 mWindowsTransformMatrixMap.clear(); 166 mWindowsTransformMatrixMap.putAll(windowsTransformMatrixMap); 167 168 mVisibleWindows.clear(); 169 mVisibleWindows.addAll(tempVisibleWindows); 170 171 mDisplayInfos.clear(); 172 for (final DisplayInfo displayInfo : displayInfos) { 173 mDisplayInfos.put(displayInfo.mDisplayId, displayInfo); 174 } 175 176 if (mWindowsNotificationEnabled) { 177 if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { 178 mHandler.sendEmptyMessageDelayed( 179 MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, 180 WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); 181 } 182 populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); 183 } 184 } 185 } 186 getWindowsTransformMatrix(List<InputWindowHandle> windows)187 private HashMap<IBinder, Matrix> getWindowsTransformMatrix(List<InputWindowHandle> windows) { 188 synchronized (mService.mGlobalLock) { 189 final HashMap<IBinder, Matrix> windowsTransformMatrixMap = new HashMap<>(); 190 191 for (InputWindowHandle inputWindowHandle : windows) { 192 final IWindow iWindow = inputWindowHandle.getWindow(); 193 final WindowState windowState = iWindow != null ? mService.mWindowMap.get( 194 iWindow.asBinder()) : null; 195 196 if (windowState != null && windowState.shouldMagnify()) { 197 final Matrix transformMatrix = new Matrix(); 198 windowState.getTransformationMatrix(sTempFloats, transformMatrix); 199 windowsTransformMatrixMap.put(iWindow.asBinder(), transformMatrix); 200 } 201 } 202 203 return windowsTransformMatrixMap; 204 } 205 } 206 207 /** 208 * Sets to notify the accessibilityController to compute changed windows on 209 * the display after populating the visible windows if the windows reported 210 * from the surface flinger changes. 211 * 212 * @param register {@code true} means starting windows population. 213 */ setWindowsNotification(boolean register)214 public void setWindowsNotification(boolean register) { 215 synchronized (mLock) { 216 if (mWindowsNotificationEnabled == register) { 217 return; 218 } 219 mWindowsNotificationEnabled = register; 220 if (mWindowsNotificationEnabled) { 221 Pair<InputWindowHandle[], DisplayInfo[]> info = register(); 222 onWindowInfosChangedInternal(info.first, info.second); 223 } else { 224 unregister(); 225 releaseResources(); 226 } 227 } 228 } 229 230 /** 231 * Sets the magnification spec for calculating the window bounds of all windows 232 * reported from the surface flinger in the magnifying. 233 * 234 * @param displayId The display Id. 235 * @param spec THe magnification spec. 236 */ setMagnificationSpec(int displayId, MagnificationSpec spec)237 public void setMagnificationSpec(int displayId, MagnificationSpec spec) { 238 synchronized (mLock) { 239 MagnificationSpec currentMagnificationSpec = mCurrentMagnificationSpec.get(displayId); 240 if (currentMagnificationSpec == null) { 241 currentMagnificationSpec = new MagnificationSpec(); 242 currentMagnificationSpec.setTo(spec); 243 mCurrentMagnificationSpec.put(displayId, currentMagnificationSpec); 244 245 return; 246 } 247 248 MagnificationSpec previousMagnificationSpec = mPreviousMagnificationSpec.get(displayId); 249 if (previousMagnificationSpec == null) { 250 previousMagnificationSpec = new MagnificationSpec(); 251 mPreviousMagnificationSpec.put(displayId, previousMagnificationSpec); 252 } 253 previousMagnificationSpec.setTo(currentMagnificationSpec); 254 currentMagnificationSpec.setTo(spec); 255 } 256 } 257 258 @GuardedBy("mLock") populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded()259 private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded() { 260 final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>(); 261 262 for (final InputWindowHandle windowHandle : mVisibleWindows) { 263 List<InputWindowHandle> inputWindowHandles = tempWindowHandleList.get( 264 windowHandle.displayId); 265 266 if (inputWindowHandles == null) { 267 inputWindowHandles = new ArrayList<>(); 268 tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); 269 } 270 inputWindowHandles.add(windowHandle); 271 } 272 findMagnificationSpecInverseMatrixIfNeeded(tempWindowHandleList); 273 274 final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); 275 getDisplaysForWindowsChanged(displayIdsForWindowsChanged, tempWindowHandleList, 276 mInputWindowHandlesOnDisplays); 277 278 // Clones all windows from the callback of the surface flinger. 279 mInputWindowHandlesOnDisplays.clear(); 280 for (int i = 0; i < tempWindowHandleList.size(); i++) { 281 final int displayId = tempWindowHandleList.keyAt(i); 282 mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); 283 } 284 285 if (!displayIdsForWindowsChanged.isEmpty()) { 286 if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { 287 mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, 288 displayIdsForWindowsChanged).sendToTarget(); 289 } 290 291 return; 292 } 293 mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); 294 mHandler.sendEmptyMessageDelayed(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE, 295 SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); 296 } 297 298 @GuardedBy("mLock") getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, SparseArray<List<InputWindowHandle>> newWindowsList, SparseArray<List<InputWindowHandle>> oldWindowsList)299 private static void getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, 300 SparseArray<List<InputWindowHandle>> newWindowsList, 301 SparseArray<List<InputWindowHandle>> oldWindowsList) { 302 for (int i = 0; i < newWindowsList.size(); i++) { 303 final int displayId = newWindowsList.keyAt(i); 304 final List<InputWindowHandle> newWindows = newWindowsList.get(displayId); 305 final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId); 306 307 if (hasWindowsChanged(newWindows, oldWindows)) { 308 outDisplayIdsForWindowsChanged.add(displayId); 309 } 310 } 311 } 312 313 @GuardedBy("mLock") hasWindowsChanged(List<InputWindowHandle> newWindows, List<InputWindowHandle> oldWindows)314 private static boolean hasWindowsChanged(List<InputWindowHandle> newWindows, 315 List<InputWindowHandle> oldWindows) { 316 if (oldWindows == null || oldWindows.size() != newWindows.size()) { 317 return true; 318 } 319 320 final int windowsCount = newWindows.size(); 321 // Since we always traverse windows from high to low layer, 322 // the old and new windows at the same index should be the 323 // same, otherwise something changed. 324 for (int i = 0; i < windowsCount; i++) { 325 final InputWindowHandle newWindow = newWindows.get(i); 326 final InputWindowHandle oldWindow = oldWindows.get(i); 327 328 if (!newWindow.getWindow().asBinder().equals(oldWindow.getWindow().asBinder())) { 329 return true; 330 } 331 } 332 333 return false; 334 } 335 336 @GuardedBy("mLock") findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> windowHandleList)337 private void findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> 338 windowHandleList) { 339 MagnificationSpec currentMagnificationSpec; 340 MagnificationSpec previousMagnificationSpec; 341 for (int i = 0; i < windowHandleList.size(); i++) { 342 final int displayId = windowHandleList.keyAt(i); 343 List<InputWindowHandle> inputWindowHandles = windowHandleList.get(displayId); 344 345 final MagnificationSpec currentSpec = mCurrentMagnificationSpec.get(displayId); 346 if (currentSpec == null) { 347 continue; 348 } 349 currentMagnificationSpec = new MagnificationSpec(); 350 currentMagnificationSpec.setTo(currentSpec); 351 352 final MagnificationSpec previousSpec = mPreviousMagnificationSpec.get(displayId); 353 354 if (previousSpec == null) { 355 final Matrix inverseMatrixForCurrentSpec = new Matrix(); 356 generateInverseMatrix(currentMagnificationSpec, inverseMatrixForCurrentSpec); 357 mMagnificationSpecInverseMatrix.put(displayId, inverseMatrixForCurrentSpec); 358 continue; 359 } 360 previousMagnificationSpec = new MagnificationSpec(); 361 previousMagnificationSpec.setTo(previousSpec); 362 363 generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(inputWindowHandles, 364 currentMagnificationSpec, previousMagnificationSpec); 365 } 366 } 367 368 @GuardedBy("mLock") generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, MagnificationSpec previousMagnificationSpec)369 private void generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( 370 List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, 371 MagnificationSpec previousMagnificationSpec) { 372 // To decrease the counts of holding the WindowManagerService#mGlogalLock in 373 // the method, getWindowTransformMatrix(), this for loop begins from the bottom 374 // to top of the z-order windows. 375 for (int index = inputWindowHandles.size() - 1; index >= 0; index--) { 376 final Matrix windowTransformMatrix = mTempMatrix2; 377 final InputWindowHandle windowHandle = inputWindowHandles.get(index); 378 final IBinder iBinder = windowHandle.getWindow().asBinder(); 379 380 if (getWindowTransformMatrix(iBinder, windowTransformMatrix)) { 381 generateMagnificationSpecInverseMatrix(windowHandle, currentMagnificationSpec, 382 previousMagnificationSpec, windowTransformMatrix); 383 384 break; 385 } 386 } 387 } 388 389 @GuardedBy("mLock") getWindowTransformMatrix(IBinder iBinder, Matrix outTransform)390 private boolean getWindowTransformMatrix(IBinder iBinder, Matrix outTransform) { 391 final Matrix windowMatrix = iBinder != null 392 ? mWindowsTransformMatrixMap.get(iBinder) : null; 393 394 if (windowMatrix == null) { 395 return false; 396 } 397 outTransform.set(windowMatrix); 398 399 return true; 400 } 401 402 /** 403 * Generates the inverse matrix based on the proper magnification spec. 404 * The magnification spec associated with the InputWindowHandle might not the current 405 * spec set by WM, which might be the previous one. To find the appropriate spec, 406 * we store two consecutive magnification specs, and found out which one is the proper 407 * one closing the identity matrix for generating the inverse matrix. 408 * 409 * @param inputWindowHandle The window from the surface flinger. 410 * @param currentMagnificationSpec The current magnification spec. 411 * @param previousMagnificationSpec The previous magnification spec. 412 * @param transformMatrix The transform matrix of the window doesn't consider the 413 * magnifying effect. 414 */ 415 @GuardedBy("mLock") generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, @NonNull MagnificationSpec currentMagnificationSpec, @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix)416 private void generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, 417 @NonNull MagnificationSpec currentMagnificationSpec, 418 @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix) { 419 420 final float[] identityMatrixFloatsForCurrentSpec = mTempFloat1; 421 computeIdentityMatrix(inputWindowHandle, currentMagnificationSpec, 422 transformMatrix, identityMatrixFloatsForCurrentSpec); 423 final float[] identityMatrixFloatsForPreviousSpec = mTempFloat2; 424 computeIdentityMatrix(inputWindowHandle, previousMagnificationSpec, 425 transformMatrix, identityMatrixFloatsForPreviousSpec); 426 427 Matrix inverseMatrixForMagnificationSpec = new Matrix(); 428 if (selectProperMagnificationSpecByComparingIdentityDegree( 429 identityMatrixFloatsForCurrentSpec, identityMatrixFloatsForPreviousSpec)) { 430 generateInverseMatrix(currentMagnificationSpec, 431 inverseMatrixForMagnificationSpec); 432 433 // Choosing the current spec means the previous spec is out of date, 434 // so removing it. And if the current spec is no magnifying, meaning 435 // the magnifying is done so removing the inverse matrix of this display. 436 mPreviousMagnificationSpec.remove(inputWindowHandle.displayId); 437 if (currentMagnificationSpec.isNop()) { 438 mCurrentMagnificationSpec.remove(inputWindowHandle.displayId); 439 mMagnificationSpecInverseMatrix.remove(inputWindowHandle.displayId); 440 return; 441 } 442 } else { 443 generateInverseMatrix(previousMagnificationSpec, 444 inverseMatrixForMagnificationSpec); 445 } 446 447 mMagnificationSpecInverseMatrix.put(inputWindowHandle.displayId, 448 inverseMatrixForMagnificationSpec); 449 } 450 451 /** 452 * Computes the identity matrix for generating the 453 * inverse matrix based on below formula under window is at the stable state: 454 * inputWindowHandle#transform * MagnificationSpecMatrix * WindowState#transform 455 * = IdentityMatrix 456 */ 457 @GuardedBy("mLock") computeIdentityMatrix(InputWindowHandle inputWindowHandle, @NonNull MagnificationSpec magnificationSpec, Matrix transformMatrix, float[] magnifyMatrixFloats)458 private void computeIdentityMatrix(InputWindowHandle inputWindowHandle, 459 @NonNull MagnificationSpec magnificationSpec, 460 Matrix transformMatrix, float[] magnifyMatrixFloats) { 461 final Matrix specMatrix = mTempMatrix1; 462 transformMagnificationSpecToMatrix(magnificationSpec, specMatrix); 463 464 final Matrix resultMatrix = new Matrix(inputWindowHandle.transform); 465 resultMatrix.preConcat(specMatrix); 466 resultMatrix.preConcat(transformMatrix); 467 468 resultMatrix.getValues(magnifyMatrixFloats); 469 } 470 471 /** 472 * @return true if selecting the magnification spec one, otherwise selecting the 473 * magnification spec two. 474 */ 475 @GuardedBy("mLock") selectProperMagnificationSpecByComparingIdentityDegree( float[] magnifyMatrixFloatsForSpecOne, float[] magnifyMatrixFloatsForSpecTwo)476 private boolean selectProperMagnificationSpecByComparingIdentityDegree( 477 float[] magnifyMatrixFloatsForSpecOne, 478 float[] magnifyMatrixFloatsForSpecTwo) { 479 final float[] IdentityMatrixValues = mTempFloat3; 480 Matrix.IDENTITY_MATRIX.getValues(IdentityMatrixValues); 481 482 final float scaleDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] 483 - magnifyMatrixFloatsForSpecOne[Matrix.MSCALE_X]); 484 final float scaleDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] 485 - magnifyMatrixFloatsForSpecTwo[Matrix.MSCALE_X]); 486 final float offsetXDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] 487 - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_X]); 488 final float offsetXDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] 489 - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_X]); 490 final float offsetYDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] 491 - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_Y]); 492 final float offsetYDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] 493 - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_Y]); 494 final float offsetDiffForSpecOne = offsetXDiffForSpecOne 495 + offsetYDiffForSpecOne; 496 final float offsetDiffForSpecTwo = offsetXDiffForSpecTwo 497 + offsetYDiffForSpecTwo; 498 499 return Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) > 0 500 || (Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) == 0 501 && Float.compare(offsetDiffForSpecTwo, offsetDiffForSpecOne) > 0); 502 } 503 504 @GuardedBy("mLock") generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix)505 private static void generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix) { 506 outMatrix.reset(); 507 508 final Matrix tempMatrix = new Matrix(); 509 transformMagnificationSpecToMatrix(spec, tempMatrix); 510 511 final boolean result = tempMatrix.invert(outMatrix); 512 if (!result) { 513 Slog.e(TAG, "Can't inverse the magnification spec matrix with the " 514 + "magnification spec = " + spec); 515 outMatrix.reset(); 516 } 517 } 518 519 @GuardedBy("mLock") transformMagnificationSpecToMatrix(MagnificationSpec spec, Matrix outMatrix)520 private static void transformMagnificationSpecToMatrix(MagnificationSpec spec, 521 Matrix outMatrix) { 522 outMatrix.reset(); 523 outMatrix.postScale(spec.scale, spec.scale); 524 outMatrix.postTranslate(spec.offsetX, spec.offsetY); 525 } 526 notifyWindowsChanged(@onNull List<Integer> displayIdsForWindowsChanged)527 private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) { 528 mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT); 529 530 for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) { 531 mAccessibilityController.performComputeChangedWindowsNot( 532 displayIdsForWindowsChanged.get(i), false); 533 } 534 } 535 forceUpdateWindows()536 private void forceUpdateWindows() { 537 final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); 538 539 synchronized (mLock) { 540 for (int i = 0; i < mInputWindowHandlesOnDisplays.size(); i++) { 541 final int displayId = mInputWindowHandlesOnDisplays.keyAt(i); 542 displayIdsForWindowsChanged.add(displayId); 543 } 544 } 545 notifyWindowsChanged(displayIdsForWindowsChanged); 546 } 547 548 @GuardedBy("mLock") releaseResources()549 private void releaseResources() { 550 mInputWindowHandlesOnDisplays.clear(); 551 mMagnificationSpecInverseMatrix.clear(); 552 mVisibleWindows.clear(); 553 mDisplayInfos.clear(); 554 mCurrentMagnificationSpec.clear(); 555 mPreviousMagnificationSpec.clear(); 556 mWindowsTransformMatrixMap.clear(); 557 mWindowsNotificationEnabled = false; 558 mHandler.removeCallbacksAndMessages(null); 559 } 560 561 private class MyHandler extends Handler { 562 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1; 563 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE = 2; 564 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT = 3; 565 MyHandler(Looper looper)566 MyHandler(Looper looper) { 567 super(looper, null, false); 568 } 569 570 @Override handleMessage(Message message)571 public void handleMessage(Message message) { 572 switch (message.what) { 573 case MESSAGE_NOTIFY_WINDOWS_CHANGED: { 574 final List<Integer> displayIdsForWindowsChanged = (List<Integer>) message.obj; 575 notifyWindowsChanged(displayIdsForWindowsChanged); 576 } break; 577 578 case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE: { 579 forceUpdateWindows(); 580 } break; 581 582 case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT: { 583 Slog.w(TAG, "Windows change within in 2 frames continuously over 500 ms " 584 + "and notify windows changed immediately"); 585 mHandler.removeMessages( 586 MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); 587 588 forceUpdateWindows(); 589 } break; 590 } 591 } 592 } 593 594 /** 595 * This class represents information about a window from the 596 * surface flinger to the accessibility framework. 597 */ 598 public static class AccessibilityWindow { 599 // Data 600 private IWindow mWindow; 601 private int mDisplayId; 602 @WindowManager.LayoutParams.WindowType 603 private int mType; 604 @InputWindowHandle.InputConfigFlags 605 private int mInputConfig; 606 private int mPrivateFlags; 607 private boolean mIsPIPMenu; 608 private boolean mIsFocused; 609 private boolean mShouldMagnify; 610 private boolean mIgnoreDuetoRecentsAnimation; 611 private final Region mTouchableRegionInScreen = new Region(); 612 private final Region mTouchableRegionInWindow = new Region(); 613 private final Region mLetterBoxBounds = new Region(); 614 private WindowInfo mWindowInfo; 615 616 617 /** 618 * Returns the instance after initializing the internal data. 619 * @param service The window manager service. 620 * @param inputWindowHandle The window from the surface flinger. 621 * @param magnificationInverseMatrix The magnification spec inverse matrix. 622 */ initializeData(WindowManagerService service, InputWindowHandle inputWindowHandle, Matrix magnificationInverseMatrix, IBinder pipIBinder, Matrix displayMatrix)623 public static AccessibilityWindow initializeData(WindowManagerService service, 624 InputWindowHandle inputWindowHandle, Matrix magnificationInverseMatrix, 625 IBinder pipIBinder, Matrix displayMatrix) { 626 final IWindow window = inputWindowHandle.getWindow(); 627 final WindowState windowState = window != null ? service.mWindowMap.get( 628 window.asBinder()) : null; 629 630 final AccessibilityWindow instance = new AccessibilityWindow(); 631 632 instance.mWindow = inputWindowHandle.getWindow(); 633 instance.mDisplayId = inputWindowHandle.displayId; 634 instance.mInputConfig = inputWindowHandle.inputConfig; 635 instance.mType = inputWindowHandle.layoutParamsType; 636 instance.mIsPIPMenu = inputWindowHandle.getWindow().asBinder().equals(pipIBinder); 637 638 // TODO (b/199357848): gets the private flag of the window from other way. 639 instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0; 640 // TODO (b/199358208) : using new way to implement the focused window. 641 instance.mIsFocused = windowState != null && windowState.isFocused(); 642 instance.mShouldMagnify = windowState == null || windowState.shouldMagnify(); 643 644 final RecentsAnimationController controller = service.getRecentsAnimationController(); 645 instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null 646 && controller.shouldIgnoreForAccessibility(windowState); 647 648 // TODO (b/199358388) : gets the letterbox bounds of the window from other way. 649 if (windowState != null && windowState.areAppWindowBoundsLetterboxed()) { 650 getLetterBoxBounds(windowState, instance.mLetterBoxBounds); 651 } 652 653 final Rect windowFrame = new Rect(inputWindowHandle.frameLeft, 654 inputWindowHandle.frameTop, inputWindowHandle.frameRight, 655 inputWindowHandle.frameBottom); 656 getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion, 657 instance.mTouchableRegionInWindow, windowFrame, magnificationInverseMatrix, 658 displayMatrix); 659 getUnMagnifiedTouchableRegion(instance.mShouldMagnify, 660 inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen, 661 magnificationInverseMatrix, displayMatrix); 662 instance.mWindowInfo = windowState != null 663 ? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance); 664 665 // Compute the transform matrix that will transform bounds from the window 666 // coordinates to screen coordinates. 667 final Matrix inverseTransform = new Matrix(); 668 inputWindowHandle.transform.invert(inverseTransform); 669 inverseTransform.postConcat(displayMatrix); 670 inverseTransform.getValues(instance.mWindowInfo.mTransformMatrix); 671 672 // Compute the magnification spec matrix. 673 final Matrix magnificationSpecMatrix = new Matrix(); 674 if (instance.shouldMagnify() && magnificationInverseMatrix != null 675 && !magnificationInverseMatrix.isIdentity()) { 676 if (magnificationInverseMatrix.invert(magnificationSpecMatrix)) { 677 magnificationSpecMatrix.getValues(sTempFloats); 678 final MagnificationSpec spec = instance.mWindowInfo.mMagnificationSpec; 679 spec.scale = sTempFloats[Matrix.MSCALE_X]; 680 spec.offsetX = sTempFloats[Matrix.MTRANS_X]; 681 spec.offsetY = sTempFloats[Matrix.MTRANS_Y]; 682 } else { 683 Slog.w(TAG, "can't find spec"); 684 } 685 } 686 return instance; 687 } 688 689 /** 690 * Returns the touchable region in the screen. 691 * @param outRegion The touchable region. 692 */ getTouchableRegionInScreen(Region outRegion)693 public void getTouchableRegionInScreen(Region outRegion) { 694 outRegion.set(mTouchableRegionInScreen); 695 } 696 697 /** 698 * Returns the touchable region in the window. 699 * @param outRegion The touchable region. 700 */ getTouchableRegionInWindow(Region outRegion)701 public void getTouchableRegionInWindow(Region outRegion) { 702 outRegion.set(mTouchableRegionInWindow); 703 } 704 705 /** 706 * @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}. 707 */ getType()708 public int getType() { 709 return mType; 710 } 711 712 /** 713 * @return the layout parameter private flag 714 * {@link android.view.WindowManager.LayoutParams#privateFlags}. 715 */ getPrivateFlag()716 public int getPrivateFlag() { 717 return mPrivateFlags; 718 } 719 720 /** 721 * @return the windowInfo {@link WindowInfo}. 722 */ getWindowInfo()723 public WindowInfo getWindowInfo() { 724 return mWindowInfo; 725 } 726 727 /** 728 * Gets the letter box bounds if activity bounds are letterboxed 729 * or letterboxed for display cutout. 730 * 731 * @return {@code true} there's a letter box bounds. 732 */ setLetterBoxBoundsIfNeeded(Region outBounds)733 public Boolean setLetterBoxBoundsIfNeeded(Region outBounds) { 734 if (mLetterBoxBounds.isEmpty()) { 735 return false; 736 } 737 738 outBounds.set(mLetterBoxBounds); 739 return true; 740 } 741 742 /** 743 * @return true if this window should be magnified. 744 */ shouldMagnify()745 public boolean shouldMagnify() { 746 return mShouldMagnify; 747 } 748 749 /** 750 * @return true if this window is focused. 751 */ isFocused()752 public boolean isFocused() { 753 return mIsFocused; 754 } 755 756 /** 757 * @return true if it's running the recent animation but not the target app. 758 */ ignoreRecentsAnimationForAccessibility()759 public boolean ignoreRecentsAnimationForAccessibility() { 760 return mIgnoreDuetoRecentsAnimation; 761 } 762 763 /** 764 * @return true if this window is the trusted overlay. 765 */ isTrustedOverlay()766 public boolean isTrustedOverlay() { 767 return (mInputConfig & InputConfig.TRUSTED_OVERLAY) != 0; 768 } 769 770 /** 771 * @return true if this window is touchable. 772 */ isTouchable()773 public boolean isTouchable() { 774 return (mInputConfig & InputConfig.NOT_TOUCHABLE) == 0; 775 } 776 777 /** 778 * @return true if this window is the navigation bar with the gesture mode. 779 */ isUntouchableNavigationBar()780 public boolean isUntouchableNavigationBar() { 781 if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) { 782 return false; 783 } 784 785 return mTouchableRegionInScreen.isEmpty(); 786 } 787 788 /** 789 * @return true if this window is PIP menu. 790 */ isPIPMenu()791 public boolean isPIPMenu() { 792 return mIsPIPMenu; 793 } 794 getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix)795 private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, 796 Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) { 797 // Some modal windows, like the activity with Theme.dialog, has the full screen 798 // as its touchable region, but its window frame is smaller than the touchable 799 // region. The region we report should be the touchable area in the window frame 800 // for the consistency and match developers expectation. 801 // So we need to make the intersection between the frame and touchable region to 802 // obtain the real touch region in the screen. 803 Region touchRegion = new Region(); 804 touchRegion.set(inRegion); 805 touchRegion.op(frame, Region.Op.INTERSECT); 806 807 getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix, 808 displayMatrix); 809 } 810 811 /** 812 * Gets the un-magnified touchable region. If this window can be magnified and magnifying, 813 * we will transform the input touchable region by applying the inverse matrix of the 814 * magnification spec to get the un-magnified touchable region. 815 * @param shouldMagnify The window can be magnified. 816 * @param inRegion The touchable region of this window. 817 * @param outRegion The un-magnified touchable region of this window. 818 * @param inverseMatrix The inverse matrix of the magnification spec. 819 * @param displayMatrix The display transform matrix which takes display coordinates to 820 * logical display coordinates. 821 */ getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, Region outRegion, Matrix inverseMatrix, Matrix displayMatrix)822 private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, 823 Region outRegion, Matrix inverseMatrix, Matrix displayMatrix) { 824 if ((!shouldMagnify || inverseMatrix.isIdentity()) && displayMatrix.isIdentity()) { 825 outRegion.set(inRegion); 826 return; 827 } 828 829 forEachRect(inRegion, rect -> { 830 // Move to origin as all transforms are captured by the matrix. 831 RectF windowFrame = new RectF(rect); 832 833 displayMatrix.mapRect(windowFrame); 834 inverseMatrix.mapRect(windowFrame); 835 // Union all rects. 836 outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, 837 (int) windowFrame.right, (int) windowFrame.bottom)); 838 }); 839 } 840 getWindowInfoForWindowlessWindows(AccessibilityWindow window)841 private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) { 842 WindowInfo windowInfo = WindowInfo.obtain(); 843 windowInfo.displayId = window.mDisplayId; 844 windowInfo.type = window.mType; 845 windowInfo.token = window.mWindow.asBinder(); 846 windowInfo.hasFlagWatchOutsideTouch = (window.mInputConfig 847 & InputConfig.WATCH_OUTSIDE_TOUCH) != 0; 848 windowInfo.inPictureInPicture = false; 849 850 // There only are two windowless windows now, one is split window, and the other 851 // one is PIP. 852 if (windowInfo.type == TYPE_DOCK_DIVIDER) { 853 windowInfo.title = "Splitscreen Divider"; 854 } else if (window.mIsPIPMenu) { 855 windowInfo.title = "Picture-in-Picture menu"; 856 // Set it to true to be consistent with the legacy implementation. 857 windowInfo.inPictureInPicture = true; 858 } 859 return windowInfo; 860 } 861 getLetterBoxBounds(WindowState windowState, Region outRegion)862 private static void getLetterBoxBounds(WindowState windowState, Region outRegion) { 863 final Rect letterboxInsets = windowState.mActivityRecord.getLetterboxInsets(); 864 final Rect nonLetterboxRect = new Rect(windowState.getBounds()); 865 866 nonLetterboxRect.inset(letterboxInsets); 867 outRegion.set(windowState.getBounds()); 868 outRegion.op(nonLetterboxRect, Region.Op.DIFFERENCE); 869 } 870 871 @Override toString()872 public String toString() { 873 String builder = "A11yWindow=[" + mWindow.asBinder() 874 + ", displayId=" + mDisplayId 875 + ", inputConfig=0x" + Integer.toHexString(mInputConfig) 876 + ", type=" + mType 877 + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) 878 + ", focused=" + mIsFocused 879 + ", shouldMagnify=" + mShouldMagnify 880 + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation 881 + ", isTrustedOverlay=" + isTrustedOverlay() 882 + ", regionInScreen=" + mTouchableRegionInScreen 883 + ", touchableRegion=" + mTouchableRegionInWindow 884 + ", letterBoxBounds=" + mLetterBoxBounds 885 + ", isPIPMenu=" + mIsPIPMenu 886 + ", windowInfo=" + mWindowInfo 887 + "]"; 888 889 return builder; 890 } 891 } 892 } 893