1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.autofillservice.cts; 18 19 import static android.autofillservice.cts.Timeouts.FILL_TIMEOUT; 20 21 import static com.google.common.truth.Truth.assertWithMessage; 22 23 import android.app.assist.AssistStructure.ViewNode; 24 import android.content.Context; 25 import android.graphics.Canvas; 26 import android.graphics.Color; 27 import android.graphics.Paint; 28 import android.graphics.Paint.Style; 29 import android.graphics.Rect; 30 import android.os.Bundle; 31 import android.text.Editable; 32 import android.text.TextUtils; 33 import android.text.TextWatcher; 34 import android.util.AttributeSet; 35 import android.util.DisplayMetrics; 36 import android.util.Log; 37 import android.util.Pair; 38 import android.util.SparseArray; 39 import android.view.MotionEvent; 40 import android.view.View; 41 import android.view.ViewStructure; 42 import android.view.ViewStructure.HtmlInfo; 43 import android.view.WindowManager; 44 import android.view.accessibility.AccessibilityEvent; 45 import android.view.accessibility.AccessibilityManager; 46 import android.view.accessibility.AccessibilityNodeInfo; 47 import android.view.accessibility.AccessibilityNodeProvider; 48 import android.view.autofill.AutofillId; 49 import android.view.autofill.AutofillManager; 50 import android.view.autofill.AutofillValue; 51 52 import java.util.ArrayList; 53 import java.util.Arrays; 54 import java.util.concurrent.CountDownLatch; 55 import java.util.concurrent.TimeUnit; 56 57 class VirtualContainerView extends View { 58 59 private static final String TAG = "VirtualContainerView"; 60 private static final int LOGIN_BUTTON_VIRTUAL_ID = 666; 61 62 static final String LABEL_CLASS = "my.readonly.view"; 63 static final String TEXT_CLASS = "my.editable.view"; 64 static final String ID_URL_BAR = "my_url_bar"; 65 static final String ID_URL_BAR2 = "my_url_bar2"; 66 67 private final ArrayList<Line> mLines = new ArrayList<>(); 68 private final SparseArray<Item> mItems = new SparseArray<>(); 69 private AutofillManager mAfm; 70 final AutofillId mLoginButtonId; 71 72 private Line mFocusedLine; 73 private int mNextChildId; 74 75 private Paint mTextPaint; 76 private int mTextHeight; 77 private int mTopMargin; 78 private int mLeftMargin; 79 private int mVerticalGap; 80 private int mLineLength; 81 private int mFocusedColor; 82 private int mUnfocusedColor; 83 private boolean mSync = true; 84 private boolean mOverrideDispatchProvideAutofillStructure = false; 85 86 private boolean mCompatMode = false; 87 private AccessibilityDelegate mAccessibilityDelegate; 88 private AccessibilityNodeProvider mAccessibilityNodeProvider; 89 90 /** 91 * Enum defining how the view communicate visibility changes to the framework 92 */ 93 enum VisibilityIntegrationMode { 94 NOTIFY_AFM, 95 OVERRIDE_IS_VISIBLE_TO_USER 96 } 97 98 private VisibilityIntegrationMode mVisibilityIntegrationMode; 99 VirtualContainerView(Context context, AttributeSet attrs)100 public VirtualContainerView(Context context, AttributeSet attrs) { 101 super(context, attrs); 102 103 setAutofillManager(context); 104 105 mTextPaint = new Paint(); 106 107 mUnfocusedColor = Color.BLACK; 108 mFocusedColor = Color.RED; 109 mTextPaint.setStyle(Style.FILL); 110 DisplayMetrics metrics = new DisplayMetrics(); 111 WindowManager wm = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); 112 wm.getDefaultDisplay().getMetrics(metrics); 113 mTopMargin = metrics.heightPixels * 5 / 100; 114 mLeftMargin = metrics.widthPixels * 5 / 100; 115 mTextHeight = metrics.widthPixels * 5 / 100; // adjust text size with display width 116 mVerticalGap = metrics.heightPixels / 100; 117 118 mLineLength = mTextHeight + mVerticalGap; 119 mTextPaint.setTextSize(mTextHeight); 120 Log.d(TAG, "Text height: " + mTextHeight); 121 mLoginButtonId = new AutofillId(getAutofillId(), LOGIN_BUTTON_VIRTUAL_ID); 122 } 123 setAutofillManager(Context context)124 public void setAutofillManager(Context context) { 125 mAfm = context.getSystemService(AutofillManager.class); 126 Log.d(TAG, "Set AFM from " + context); 127 } 128 129 @Override autofill(SparseArray<AutofillValue> values)130 public void autofill(SparseArray<AutofillValue> values) { 131 Log.d(TAG, "autofill: " + values); 132 if (mCompatMode) { 133 Log.v(TAG, "using super.autofill() on compat mode"); 134 super.autofill(values); 135 return; 136 } 137 for (int i = 0; i < values.size(); i++) { 138 final int id = values.keyAt(i); 139 final AutofillValue value = values.valueAt(i); 140 final Item item = getItem(id); 141 item.autofill(value.getTextValue()); 142 } 143 postInvalidate(); 144 } 145 146 @Override onDraw(Canvas canvas)147 protected void onDraw(Canvas canvas) { 148 super.onDraw(canvas); 149 150 Log.d(TAG, "onDraw: " + mLines.size() + " lines; canvas:" + canvas); 151 float x; 152 float y = mTopMargin + mLineLength; 153 for (int i = 0; i < mLines.size(); i++) { 154 x = mLeftMargin; 155 final Line line = mLines.get(i); 156 if (!line.visible) { 157 continue; 158 } 159 Log.v(TAG, "Drawing '" + line + "' at " + x + "x" + y); 160 mTextPaint.setColor(line.focused ? mFocusedColor : mUnfocusedColor); 161 final String readOnlyText = line.label.text + ": ["; 162 final String writeText = line.text.text + "]"; 163 // Paints the label first... 164 canvas.drawText(readOnlyText, x, y, mTextPaint); 165 // ...then paints the edit text and sets the proper boundary 166 final float deltaX = mTextPaint.measureText(readOnlyText); 167 x += deltaX; 168 line.bounds.set((int) x, (int) (y - mLineLength), 169 (int) (x + mTextPaint.measureText(writeText)), (int) y); 170 Log.d(TAG, "setBounds(" + x + ", " + y + "): " + line.bounds); 171 canvas.drawText(writeText, x, y, mTextPaint); 172 y += mLineLength; 173 } 174 } 175 176 @Override onTouchEvent(MotionEvent event)177 public boolean onTouchEvent(MotionEvent event) { 178 final int y = (int) event.getY(); 179 Log.d(TAG, "You can touch this: y=" + y + ", range=" + mLineLength + ", top=" + mTopMargin); 180 int lowerY = mTopMargin; 181 int upperY = -1; 182 for (int i = 0; i < mLines.size(); i++) { 183 upperY = lowerY + mLineLength; 184 final Line line = mLines.get(i); 185 Log.d(TAG, "Line " + i + " ranges from " + lowerY + " to " + upperY); 186 if (lowerY <= y && y <= upperY) { 187 if (mFocusedLine != null) { 188 Log.d(TAG, "Removing focus from " + mFocusedLine); 189 mFocusedLine.changeFocus(false); 190 } 191 Log.d(TAG, "Changing focus to " + line); 192 mFocusedLine = line; 193 mFocusedLine.changeFocus(true); 194 invalidate(); 195 break; 196 } 197 lowerY += mLineLength; 198 } 199 return super.onTouchEvent(event); 200 } 201 202 @Override dispatchProvideAutofillStructure(ViewStructure structure, int flags)203 public void dispatchProvideAutofillStructure(ViewStructure structure, int flags) { 204 if (mOverrideDispatchProvideAutofillStructure) { 205 Log.d(TAG, "Overriding dispatchProvideAutofillStructure()"); 206 structure.setAutofillId(getAutofillId()); 207 onProvideAutofillVirtualStructure(structure, flags); 208 } else { 209 super.dispatchProvideAutofillStructure(structure, flags); 210 } 211 } 212 213 @Override onProvideAutofillVirtualStructure(ViewStructure structure, int flags)214 public void onProvideAutofillVirtualStructure(ViewStructure structure, int flags) { 215 Log.d(TAG, "onProvideAutofillVirtualStructure(): flags = " + flags); 216 super.onProvideAutofillVirtualStructure(structure, flags); 217 218 if (mCompatMode) { 219 Log.v(TAG, "using super.onProvideAutofillVirtualStructure() on compat mode"); 220 return; 221 } 222 223 final String packageName = getContext().getPackageName(); 224 structure.setClassName(getClass().getName()); 225 final int childrenSize = mItems.size(); 226 int index = structure.addChildCount(childrenSize); 227 final String syncMsg = mSync ? "" : " (async)"; 228 for (int i = 0; i < childrenSize; i++) { 229 final Item item = mItems.valueAt(i); 230 Log.d(TAG, "Adding new child" + syncMsg + " at index " + index + ": " + item); 231 final ViewStructure child = mSync 232 ? structure.newChild(index) 233 : structure.asyncNewChild(index); 234 child.setAutofillId(structure.getAutofillId(), item.id); 235 child.setDataIsSensitive(item.sensitive); 236 if (item.editable) { 237 child.setInputType(item.line.inputType); 238 } 239 index++; 240 child.setClassName(item.className); 241 // Must set "fake" idEntry because that's what the test cases use to find nodes. 242 child.setId(1000 + index, packageName, "id", item.resourceId); 243 child.setText(item.text); 244 if (TextUtils.getTrimmedLength(item.text) > 0) { 245 // TODO: Must checked trimmed length because input fields use 8 empty spaces to 246 // set width 247 child.setAutofillValue(AutofillValue.forText(item.text)); 248 } 249 child.setFocused(item.line.focused); 250 child.setHtmlInfo(child.newHtmlInfoBuilder("TAGGY") 251 .addAttribute("a1", "v1") 252 .addAttribute("a2", "v2") 253 .addAttribute("a1", "v2") 254 .build()); 255 child.setAutofillHints(new String[] {"c", "a", "a", "b", "a", "a"}); 256 257 if (!mSync) { 258 Log.d(TAG, "Commiting virtual child"); 259 child.asyncCommit(); 260 } 261 } 262 } 263 264 @Override isVisibleToUserForAutofill(int virtualId)265 public boolean isVisibleToUserForAutofill(int virtualId) { 266 boolean callSuper = true; 267 if (mVisibilityIntegrationMode == null) { 268 Log.w(TAG, "isVisibleToUserForAutofill(): mVisibilityIntegrationMode not set"); 269 } else { 270 callSuper = mVisibilityIntegrationMode == VisibilityIntegrationMode.NOTIFY_AFM; 271 } 272 final boolean isVisible; 273 if (callSuper) { 274 isVisible = super.isVisibleToUserForAutofill(virtualId); 275 Log.d(TAG, "isVisibleToUserForAutofill(" + virtualId + ") using super: " + isVisible); 276 } else { 277 final Item item = getItem(virtualId); 278 isVisible = item.line.visible; 279 Log.d(TAG, "isVisibleToUserForAutofill(" + virtualId + ") set by test: " + isVisible); 280 } 281 return isVisible; 282 } 283 284 /** 285 * Emulates clicking the login button. 286 */ clickLogin()287 void clickLogin() { 288 Log.d(TAG, "clickLogin()"); 289 if (mCompatMode) { 290 sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_CLICKED, LOGIN_BUTTON_VIRTUAL_ID); 291 } else { 292 mAfm.notifyViewClicked(this, LOGIN_BUTTON_VIRTUAL_ID); 293 } 294 } 295 getItem(int id)296 private Item getItem(int id) { 297 final Item item = mItems.get(id); 298 assertWithMessage("No item for id %s", id).that(item).isNotNull(); 299 return item; 300 } 301 onProvideAutofillCompatModeAccessibilityNodeInfo()302 private AccessibilityNodeInfo onProvideAutofillCompatModeAccessibilityNodeInfo() { 303 final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(); 304 305 final String packageName = getContext().getPackageName(); 306 node.setPackageName(packageName); 307 node.setClassName(getClass().getName()); 308 309 final int childrenSize = mItems.size(); 310 for (int i = 0; i < childrenSize; i++) { 311 final Item item = mItems.valueAt(i); 312 final int id = i + 1; 313 Log.d(TAG, "Adding new A11Y child with id " + id + ": " + item); 314 315 node.addChild(this, id); 316 } 317 318 return node; 319 } 320 onProvideAutofillCompatModeAccessibilityNodeInfoForLoginButton()321 private AccessibilityNodeInfo onProvideAutofillCompatModeAccessibilityNodeInfoForLoginButton() { 322 final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(); 323 node.setSource(this, LOGIN_BUTTON_VIRTUAL_ID); 324 node.setPackageName(getContext().getPackageName()); 325 // TODO(b/37566627): ideally this button should be visible / drawn in the canvas and contain 326 // more properties like boundaries, class name, text etc... 327 return node; 328 } 329 assertHtmlInfo(ViewNode node)330 static void assertHtmlInfo(ViewNode node) { 331 final String name = node.getText().toString(); 332 final HtmlInfo info = node.getHtmlInfo(); 333 assertWithMessage("no HTML info on %s", name).that(info).isNotNull(); 334 assertWithMessage("wrong HTML tag on %s", name).that(info.getTag()).isEqualTo("TAGGY"); 335 assertWithMessage("wrong attributes on %s", name).that(info.getAttributes()) 336 .containsExactly( 337 new Pair<>("a1", "v1"), 338 new Pair<>("a2", "v2"), 339 new Pair<>("a1", "v2")); 340 } 341 addLine(String labelId, String label, String textId, String text, int inputType)342 Line addLine(String labelId, String label, String textId, String text, int inputType) { 343 final Line line = new Line(labelId, label, textId, text, inputType); 344 Log.d(TAG, "addLine: " + line); 345 mLines.add(line); 346 mItems.put(line.label.id, line.label); 347 mItems.put(line.text.id, line.text); 348 return line; 349 } 350 setSync(boolean sync)351 void setSync(boolean sync) { 352 mSync = sync; 353 } 354 setCompatMode(boolean compatMode)355 void setCompatMode(boolean compatMode) { 356 mCompatMode = compatMode; 357 358 if (mCompatMode) { 359 setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_YES); 360 mAccessibilityNodeProvider = new AccessibilityNodeProvider() { 361 @Override 362 public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { 363 Log.d(TAG, "createAccessibilityNodeInfo(): id=" + virtualViewId); 364 switch (virtualViewId) { 365 case AccessibilityNodeProvider.HOST_VIEW_ID: 366 return onProvideAutofillCompatModeAccessibilityNodeInfo(); 367 case LOGIN_BUTTON_VIRTUAL_ID: 368 return onProvideAutofillCompatModeAccessibilityNodeInfoForLoginButton(); 369 default: 370 final Item item = getItem(virtualViewId); 371 return item.provideAccessibilityNodeInfo(VirtualContainerView.this, 372 getContext()); 373 } 374 } 375 376 @Override 377 public boolean performAction(int virtualViewId, int action, Bundle arguments) { 378 if (action == AccessibilityNodeInfo.ACTION_SET_TEXT) { 379 final CharSequence text = arguments.getCharSequence( 380 AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE); 381 final Item item = getItem(virtualViewId); 382 item.autofill(text); 383 return true; 384 } 385 386 return false; 387 } 388 }; 389 mAccessibilityDelegate = new AccessibilityDelegate() { 390 @Override 391 public AccessibilityNodeProvider getAccessibilityNodeProvider(View host) { 392 return mAccessibilityNodeProvider; 393 } 394 }; 395 396 setAccessibilityDelegate(mAccessibilityDelegate); 397 } 398 } 399 setOverrideDispatchProvideAutofillStructure(boolean flag)400 void setOverrideDispatchProvideAutofillStructure(boolean flag) { 401 mOverrideDispatchProvideAutofillStructure = flag; 402 } 403 sendAccessibilityEvent(int eventType, int virtualId)404 private void sendAccessibilityEvent(int eventType, int virtualId) { 405 final AccessibilityEvent event = AccessibilityEvent.obtain(); 406 event.setEventType(eventType); 407 event.setSource(VirtualContainerView.this, virtualId); 408 event.setEnabled(true); 409 event.setPackageName(getContext().getPackageName()); 410 Log.v(TAG, "sendAccessibilityEvent(" + eventType + ", " + virtualId + "): " + event); 411 getContext().getSystemService(AccessibilityManager.class).sendAccessibilityEvent(event); 412 } 413 414 final class Line { 415 416 final Item label; 417 final Item text; 418 // Boundaries of the text field, relative to the CustomView 419 final Rect bounds = new Rect(); 420 // Boundaries of the text field, relative to the screen 421 Rect absBounds; 422 423 private boolean focused; 424 private boolean visible = true; 425 private final int inputType; 426 Line(String labelId, String label, String textId, String text, int inputType)427 private Line(String labelId, String label, String textId, String text, int inputType) { 428 this.label = new Item(this, ++mNextChildId, labelId, label, false, false); 429 this.text = new Item(this, ++mNextChildId, textId, text, true, true); 430 this.inputType = inputType; 431 } 432 changeFocus(boolean focused)433 void changeFocus(boolean focused) { 434 this.focused = focused; 435 436 if (focused) { 437 absBounds = getAbsCoordinates(); 438 Log.v(TAG, "Setting absBounds for " + text.id + " on focus change: " + absBounds); 439 } 440 441 if (mCompatMode) { 442 sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_FOCUSED, text.id); 443 return; 444 } 445 446 if (focused) { 447 Log.d(TAG, "focus gained on " + text.id + "; absBounds=" + absBounds); 448 mAfm.notifyViewEntered(VirtualContainerView.this, text.id, absBounds); 449 } else { 450 Log.d(TAG, "focus lost on " + text.id); 451 mAfm.notifyViewExited(VirtualContainerView.this, text.id); 452 } 453 } 454 setVisibilityIntegrationMode(VisibilityIntegrationMode mode)455 void setVisibilityIntegrationMode(VisibilityIntegrationMode mode) { 456 mVisibilityIntegrationMode = mode; 457 } 458 changeVisibility(boolean visible)459 void changeVisibility(boolean visible) { 460 if (mVisibilityIntegrationMode == null) { 461 throw new IllegalStateException("must call setVisibilityIntegrationMode() first"); 462 } 463 if (this.visible == visible) { 464 return; 465 } 466 this.visible = visible; 467 Log.d(TAG, "visibility changed view: " + text.id + "; visible:" + visible 468 + "; integrationMode: " + mVisibilityIntegrationMode); 469 if (mVisibilityIntegrationMode == VisibilityIntegrationMode.NOTIFY_AFM) { 470 mAfm.notifyViewVisibilityChanged(VirtualContainerView.this, text.id, visible); 471 } 472 invalidate(); 473 } 474 getAbsCoordinates()475 Rect getAbsCoordinates() { 476 // Must offset the boundaries so they're relative to the CustomView. 477 final int offset[] = new int[2]; 478 getLocationOnScreen(offset); 479 final Rect absBounds = new Rect(bounds.left + offset[0], 480 bounds.top + offset[1], 481 bounds.right + offset[0], bounds.bottom + offset[1]); 482 Log.v(TAG, "getAbsCoordinates() for " + text.id + ": bounds=" + bounds 483 + " offset: " + Arrays.toString(offset) + " absBounds: " + absBounds); 484 return absBounds; 485 } 486 setText(String value)487 void setText(String value) { 488 text.text = value; 489 final AutofillManager autofillManager = 490 getContext().getSystemService(AutofillManager.class); 491 if (mCompatMode) { 492 sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED, text.id); 493 } else { 494 if (autofillManager != null) { 495 autofillManager.notifyValueChanged(VirtualContainerView.this, text.id, 496 AutofillValue.forText(text.text)); 497 } 498 } 499 invalidate(); 500 } 501 setTextChangedListener(TextWatcher listener)502 void setTextChangedListener(TextWatcher listener) { 503 text.listener = listener; 504 } 505 506 @Override toString()507 public String toString() { 508 return "Label: " + label + " Text: " + text + " Focused: " + focused 509 + " Visible: " + visible; 510 } 511 512 final class OneTimeLineWatcher implements TextWatcher { 513 private final CountDownLatch latch; 514 private final CharSequence expected; 515 OneTimeLineWatcher(CharSequence expectedValue)516 OneTimeLineWatcher(CharSequence expectedValue) { 517 this.expected = expectedValue; 518 this.latch = new CountDownLatch(1); 519 } 520 521 @Override beforeTextChanged(CharSequence s, int start, int count, int after)522 public void beforeTextChanged(CharSequence s, int start, int count, int after) { 523 } 524 525 @Override onTextChanged(CharSequence s, int start, int before, int count)526 public void onTextChanged(CharSequence s, int start, int before, int count) { 527 latch.countDown(); 528 } 529 530 @Override afterTextChanged(Editable s)531 public void afterTextChanged(Editable s) { 532 } 533 assertAutoFilled()534 void assertAutoFilled() throws Exception { 535 final boolean set = latch.await(FILL_TIMEOUT.ms(), TimeUnit.MILLISECONDS); 536 assertWithMessage("Timeout (%s ms) on Line %s", FILL_TIMEOUT.ms(), label) 537 .that(set).isTrue(); 538 final String actual = text.text.toString(); 539 assertWithMessage("Wrong auto-fill value on Line %s", label) 540 .that(actual).isEqualTo(expected.toString()); 541 } 542 } 543 } 544 545 static final class Item { 546 private final Line line; 547 final int id; 548 private final String resourceId; 549 private CharSequence text; 550 private final boolean editable; 551 private final boolean sensitive; 552 private final String className; 553 private TextWatcher listener; 554 Item(Line line, int id, String resourceId, CharSequence text, boolean editable, boolean sensitive)555 Item(Line line, int id, String resourceId, CharSequence text, boolean editable, 556 boolean sensitive) { 557 this.line = line; 558 this.id = id; 559 this.resourceId = resourceId; 560 this.text = text; 561 this.editable = editable; 562 this.sensitive = sensitive; 563 this.className = editable ? TEXT_CLASS : LABEL_CLASS; 564 } 565 provideAccessibilityNodeInfo(View parent, Context context)566 AccessibilityNodeInfo provideAccessibilityNodeInfo(View parent, Context context) { 567 final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(); 568 node.setSource(parent, id); 569 node.setPackageName(context.getPackageName()); 570 node.setClassName(className); 571 node.setEditable(editable); 572 node.setViewIdResourceName(resourceId); 573 node.setVisibleToUser(true); 574 node.setInputType(line.inputType); 575 if (line.absBounds != null) { 576 node.setBoundsInScreen(line.absBounds); 577 } 578 if (TextUtils.getTrimmedLength(text) > 0) { 579 // TODO: Must checked trimmed length because input fields use 8 empty spaces to 580 // set width 581 node.setText(text); 582 } 583 return node; 584 } 585 autofill(CharSequence value)586 private void autofill(CharSequence value) { 587 if (!editable) { 588 Log.w(TAG, "Item for id " + id + " is not editable: " + this); 589 return; 590 } 591 text = value; 592 if (listener != null) { 593 Log.d(TAG, "Notify listener: " + text); 594 listener.onTextChanged(text, 0, 0, 0); 595 } 596 } 597 598 @Override toString()599 public String toString() { 600 return id + "/" + resourceId + ": " + text + (editable ? " (editable)" : " (read-only)" 601 + (sensitive ? " (sensitive)" : " (sanitized")); 602 } 603 } 604 } 605