1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.internal.widget.remotecompose.player.platform; 17 18 import android.content.Context; 19 import android.graphics.Bitmap; 20 import android.graphics.Canvas; 21 import android.graphics.Color; 22 import android.graphics.Paint; 23 import android.graphics.Point; 24 import android.graphics.Rect; 25 import android.util.AttributeSet; 26 import android.view.Choreographer; 27 import android.view.MotionEvent; 28 import android.view.VelocityTracker; 29 import android.view.View; 30 import android.widget.FrameLayout; 31 import android.widget.TextView; 32 33 import com.android.internal.widget.remotecompose.core.CoreDocument; 34 import com.android.internal.widget.remotecompose.core.RemoteContext; 35 import com.android.internal.widget.remotecompose.core.operations.Header; 36 import com.android.internal.widget.remotecompose.core.operations.RootContentBehavior; 37 import com.android.internal.widget.remotecompose.core.operations.Theme; 38 import com.android.internal.widget.remotecompose.player.RemoteComposeDocument; 39 40 import java.util.Set; 41 42 /** Internal view handling the actual painting / interactions */ 43 public class RemoteComposeCanvas extends FrameLayout implements View.OnAttachStateChangeListener { 44 45 static final boolean USE_VIEW_AREA_CLICK = true; // Use views to represent click areas 46 static final float DEFAULT_FRAME_RATE = 60f; 47 static final float POST_TO_NEXT_FRAME_THRESHOLD = 60f; 48 49 RemoteComposeDocument mDocument = null; 50 int mTheme = Theme.LIGHT; 51 boolean mInActionDown = false; 52 int mDebug = 0; 53 boolean mHasClickAreas = false; 54 Point mActionDownPoint = new Point(0, 0); 55 AndroidRemoteContext mARContext = new AndroidRemoteContext(); 56 float mDensity = Float.NaN; 57 long mStart = System.nanoTime(); 58 59 long mLastFrameDelay = 1; 60 float mMaxFrameRate = DEFAULT_FRAME_RATE; // frames per seconds 61 long mMaxFrameDelay = (long) (1000 / mMaxFrameRate); 62 63 long mLastFrameCall = System.currentTimeMillis(); 64 65 private Choreographer mChoreographer; 66 private Choreographer.FrameCallback mFrameCallback = 67 new Choreographer.FrameCallback() { 68 @Override 69 public void doFrame(long frameTimeNanos) { 70 mARContext.currentTime = frameTimeNanos / 1000000; 71 mARContext.setDebug(mDebug); 72 postInvalidateOnAnimation(); 73 } 74 }; 75 RemoteComposeCanvas(Context context)76 public RemoteComposeCanvas(Context context) { 77 super(context); 78 addOnAttachStateChangeListener(this); 79 } 80 RemoteComposeCanvas(Context context, AttributeSet attrs)81 public RemoteComposeCanvas(Context context, AttributeSet attrs) { 82 super(context, attrs); 83 addOnAttachStateChangeListener(this); 84 } 85 RemoteComposeCanvas(Context context, AttributeSet attrs, int defStyleAttr)86 public RemoteComposeCanvas(Context context, AttributeSet attrs, int defStyleAttr) { 87 super(context, attrs, defStyleAttr); 88 setBackgroundColor(Color.WHITE); 89 addOnAttachStateChangeListener(this); 90 } 91 setDebug(int value)92 public void setDebug(int value) { 93 if (mDebug != value) { 94 mDebug = value; 95 if (USE_VIEW_AREA_CLICK) { 96 for (int i = 0; i < getChildCount(); i++) { 97 View child = getChildAt(i); 98 if (child instanceof ClickAreaView) { 99 ((ClickAreaView) child).setDebug(mDebug == 1); 100 } 101 } 102 } 103 invalidate(); 104 } 105 } 106 setDocument(RemoteComposeDocument value)107 public void setDocument(RemoteComposeDocument value) { 108 mDocument = value; 109 mMaxFrameRate = DEFAULT_FRAME_RATE; 110 mDocument.initializeContext(mARContext); 111 mDisable = false; 112 mARContext.setDocLoadTime(); 113 mARContext.setAnimationEnabled(true); 114 mARContext.setDensity(mDensity); 115 mARContext.setUseChoreographer(true); 116 setContentDescription(mDocument.getDocument().getContentDescription()); 117 118 updateClickAreas(); 119 requestLayout(); 120 mARContext.loadFloat(RemoteContext.ID_TOUCH_EVENT_TIME, -Float.MAX_VALUE); 121 mARContext.loadFloat(RemoteContext.ID_FONT_SIZE, getDefaultTextSize()); 122 123 invalidate(); 124 Integer fps = (Integer) mDocument.getDocument().getProperty(Header.DOC_DESIRED_FPS); 125 if (fps != null && fps > 0) { 126 mMaxFrameRate = fps; 127 mMaxFrameDelay = (long) (1000 / mMaxFrameRate); 128 } 129 } 130 131 @Override onViewAttachedToWindow(View view)132 public void onViewAttachedToWindow(View view) { 133 if (mChoreographer == null) { 134 mChoreographer = Choreographer.getInstance(); 135 mChoreographer.postFrameCallback(mFrameCallback); 136 } 137 mDensity = getContext().getResources().getDisplayMetrics().density; 138 mARContext.setDensity(mDensity); 139 if (mDocument == null) { 140 return; 141 } 142 updateClickAreas(); 143 } 144 updateClickAreas()145 private void updateClickAreas() { 146 if (USE_VIEW_AREA_CLICK && mDocument != null) { 147 mHasClickAreas = false; 148 Set<CoreDocument.ClickAreaRepresentation> clickAreas = 149 mDocument.getDocument().getClickAreas(); 150 removeAllViews(); 151 for (CoreDocument.ClickAreaRepresentation area : clickAreas) { 152 ClickAreaView viewArea = 153 new ClickAreaView( 154 getContext(), 155 mDebug == 1, 156 area.getId(), 157 area.getContentDescription(), 158 area.getMetadata()); 159 int w = (int) area.width(); 160 int h = (int) area.height(); 161 FrameLayout.LayoutParams param = new FrameLayout.LayoutParams(w, h); 162 param.width = w; 163 param.height = h; 164 param.leftMargin = (int) area.getLeft(); 165 param.topMargin = (int) area.getTop(); 166 viewArea.setOnClickListener( 167 view1 -> 168 mDocument 169 .getDocument() 170 .performClick( 171 mARContext, area.getId(), area.getMetadata())); 172 addView(viewArea, param); 173 } 174 if (!clickAreas.isEmpty()) { 175 mHasClickAreas = true; 176 } 177 } 178 } 179 setHapticEngine(CoreDocument.HapticEngine engine)180 public void setHapticEngine(CoreDocument.HapticEngine engine) { 181 mDocument.getDocument().setHapticEngine(engine); 182 } 183 184 @Override onViewDetachedFromWindow(View view)185 public void onViewDetachedFromWindow(View view) { 186 if (mChoreographer != null) { 187 mChoreographer.removeFrameCallback(mFrameCallback); 188 mChoreographer = null; 189 } 190 removeAllViews(); 191 } 192 getNamedColors()193 public String[] getNamedColors() { 194 return mDocument.getNamedColors(); 195 } 196 197 /** 198 * Gets a array of Names of the named variables of a specific type defined in the loaded doc. 199 * 200 * @param type the type of variable NamedVariable.COLOR_TYPE, STRING_TYPE, etc 201 * @return array of name or null 202 */ getNamedVariables(int type)203 public String[] getNamedVariables(int type) { 204 return mDocument.getNamedVariables(type); 205 } 206 207 /** 208 * set the color associated with this name. 209 * 210 * @param colorName Name of color typically "android.xxx" 211 * @param colorValue "the argb value" 212 */ setColor(String colorName, int colorValue)213 public void setColor(String colorName, int colorValue) { 214 mARContext.setNamedColorOverride(colorName, colorValue); 215 } 216 217 /** 218 * set the value of a long associated with this name. 219 * 220 * @param name Name of color typically "android.xxx" 221 * @param value the long value 222 */ setLong(String name, long value)223 public void setLong(String name, long value) { 224 mARContext.setNamedLong(name, value); 225 } 226 getDocument()227 public RemoteComposeDocument getDocument() { 228 return mDocument; 229 } 230 setLocalString(String name, String content)231 public void setLocalString(String name, String content) { 232 mARContext.setNamedStringOverride(name, content); 233 if (mDocument != null) { 234 mDocument.invalidate(); 235 } 236 } 237 clearLocalString(String name)238 public void clearLocalString(String name) { 239 mARContext.clearNamedStringOverride(name); 240 if (mDocument != null) { 241 mDocument.invalidate(); 242 } 243 } 244 setLocalInt(String name, int content)245 public void setLocalInt(String name, int content) { 246 mARContext.setNamedIntegerOverride(name, content); 247 if (mDocument != null) { 248 mDocument.invalidate(); 249 } 250 } 251 clearLocalInt(String name)252 public void clearLocalInt(String name) { 253 mARContext.clearNamedIntegerOverride(name); 254 if (mDocument != null) { 255 mDocument.invalidate(); 256 } 257 } 258 259 /** 260 * Set a local named color 261 * 262 * @param name 263 * @param content 264 */ setLocalColor(String name, int content)265 public void setLocalColor(String name, int content) { 266 mARContext.setNamedColorOverride(name, content); 267 if (mDocument != null) { 268 mDocument.invalidate(); 269 } 270 } 271 272 /** 273 * Clear a local named color 274 * 275 * @param name 276 */ clearLocalColor(String name)277 public void clearLocalColor(String name) { 278 mARContext.clearNamedDataOverride(name); 279 if (mDocument != null) { 280 mDocument.invalidate(); 281 } 282 } 283 setLocalFloat(String name, Float content)284 public void setLocalFloat(String name, Float content) { 285 mARContext.setNamedFloatOverride(name, content); 286 if (mDocument != null) { 287 mDocument.invalidate(); 288 } 289 } 290 clearLocalFloat(String name)291 public void clearLocalFloat(String name) { 292 mARContext.clearNamedFloatOverride(name); 293 if (mDocument != null) { 294 mDocument.invalidate(); 295 } 296 } 297 setLocalBitmap(String name, Bitmap content)298 public void setLocalBitmap(String name, Bitmap content) { 299 mARContext.setNamedDataOverride(name, content); 300 if (mDocument != null) { 301 mDocument.invalidate(); 302 } 303 } 304 clearLocalBitmap(String name)305 public void clearLocalBitmap(String name) { 306 mARContext.clearNamedDataOverride(name); 307 if (mDocument != null) { 308 mDocument.invalidate(); 309 } 310 } 311 hasSensorListeners(int[] ids)312 public int hasSensorListeners(int[] ids) { 313 int count = 0; 314 for (int id = RemoteContext.ID_ACCELERATION_X; id <= RemoteContext.ID_LIGHT; id++) { 315 if (mARContext.mRemoteComposeState.hasListener(id)) { 316 ids[count++] = id; 317 } 318 } 319 return count; 320 } 321 322 /** 323 * set a float externally 324 * 325 * @param id 326 * @param value 327 */ setExternalFloat(int id, float value)328 public void setExternalFloat(int id, float value) { 329 mARContext.loadFloat(id, value); 330 } 331 332 /** 333 * Returns true if the document supports drag touch events 334 * 335 * @return true if draggable content, false otherwise 336 */ isDraggable()337 public boolean isDraggable() { 338 if (mDocument == null) { 339 return false; 340 } 341 return mDocument.getDocument().hasTouchListener(); 342 } 343 344 /** 345 * Check shaders and disable them 346 * 347 * @param shaderControl the callback to validate the shader 348 */ checkShaders(CoreDocument.ShaderControl shaderControl)349 public void checkShaders(CoreDocument.ShaderControl shaderControl) { 350 mDocument.getDocument().checkShaders(mARContext, shaderControl); 351 } 352 353 /** 354 * Set to true to use the choreographer 355 * 356 * @param value 357 */ setUseChoreographer(boolean value)358 public void setUseChoreographer(boolean value) { 359 mARContext.setUseChoreographer(value); 360 } 361 getRemoteContext()362 public RemoteContext getRemoteContext() { 363 return mARContext; 364 } 365 366 public interface ClickCallbacks { click(int id, String metadata)367 void click(int id, String metadata); 368 } 369 addIdActionListener(ClickCallbacks callback)370 public void addIdActionListener(ClickCallbacks callback) { 371 if (mDocument == null) { 372 return; 373 } 374 mDocument.getDocument().addIdActionListener((id, metadata) -> callback.click(id, metadata)); 375 } 376 getTheme()377 public int getTheme() { 378 return mTheme; 379 } 380 setTheme(int theme)381 public void setTheme(int theme) { 382 this.mTheme = theme; 383 } 384 385 private VelocityTracker mVelocityTracker = null; 386 onTouchEvent(MotionEvent event)387 public boolean onTouchEvent(MotionEvent event) { 388 int index = event.getActionIndex(); 389 int action = event.getActionMasked(); 390 int pointerId = event.getPointerId(index); 391 if (USE_VIEW_AREA_CLICK && mHasClickAreas) { 392 return super.onTouchEvent(event); 393 } 394 switch (event.getActionMasked()) { 395 case MotionEvent.ACTION_DOWN: 396 mActionDownPoint.x = (int) event.getX(); 397 mActionDownPoint.y = (int) event.getY(); 398 CoreDocument doc = mDocument.getDocument(); 399 if (doc.hasTouchListener()) { 400 mARContext.loadFloat( 401 RemoteContext.ID_TOUCH_EVENT_TIME, mARContext.getAnimationTime()); 402 mInActionDown = true; 403 if (mVelocityTracker == null) { 404 mVelocityTracker = VelocityTracker.obtain(); 405 } else { 406 mVelocityTracker.clear(); 407 } 408 mVelocityTracker.addMovement(event); 409 doc.touchDown(mARContext, event.getX(), event.getY()); 410 invalidate(); 411 return true; 412 } 413 return false; 414 415 case MotionEvent.ACTION_CANCEL: 416 mInActionDown = false; 417 doc = mDocument.getDocument(); 418 if (doc.hasTouchListener()) { 419 mVelocityTracker.computeCurrentVelocity(1000); 420 float dx = mVelocityTracker.getXVelocity(pointerId); 421 float dy = mVelocityTracker.getYVelocity(pointerId); 422 doc.touchCancel(mARContext, event.getX(), event.getY(), dx, dy); 423 invalidate(); 424 return true; 425 } 426 return false; 427 428 case MotionEvent.ACTION_UP: 429 mInActionDown = false; 430 performClick(); 431 doc = mDocument.getDocument(); 432 if (doc.hasTouchListener()) { 433 mARContext.loadFloat( 434 RemoteContext.ID_TOUCH_EVENT_TIME, mARContext.getAnimationTime()); 435 mVelocityTracker.computeCurrentVelocity(1000); 436 float dx = mVelocityTracker.getXVelocity(pointerId); 437 float dy = mVelocityTracker.getYVelocity(pointerId); 438 doc.touchUp(mARContext, event.getX(), event.getY(), dx, dy); 439 invalidate(); 440 return true; 441 } 442 return false; 443 444 case MotionEvent.ACTION_MOVE: 445 if (mInActionDown) { 446 if (mVelocityTracker != null) { 447 mARContext.loadFloat( 448 RemoteContext.ID_TOUCH_EVENT_TIME, mARContext.getAnimationTime()); 449 mVelocityTracker.addMovement(event); 450 doc = mDocument.getDocument(); 451 boolean repaint = doc.touchDrag(mARContext, event.getX(), event.getY()); 452 if (repaint) { 453 invalidate(); 454 } 455 } 456 return true; 457 } 458 return false; 459 } 460 return false; 461 } 462 463 @Override performClick()464 public boolean performClick() { 465 if (USE_VIEW_AREA_CLICK && mHasClickAreas) { 466 return super.performClick(); 467 } 468 mDocument 469 .getDocument() 470 .onClick(mARContext, (float) mActionDownPoint.x, (float) mActionDownPoint.y); 471 super.performClick(); 472 invalidate(); 473 return true; 474 } 475 measureDimension(int measureSpec, int intrinsicSize)476 public int measureDimension(int measureSpec, int intrinsicSize) { 477 int result = intrinsicSize; 478 int mode = MeasureSpec.getMode(measureSpec); 479 int size = MeasureSpec.getSize(measureSpec); 480 switch (mode) { 481 case MeasureSpec.EXACTLY: 482 result = size; 483 break; 484 case MeasureSpec.AT_MOST: 485 result = Integer.min(size, intrinsicSize); 486 break; 487 case MeasureSpec.UNSPECIFIED: 488 result = intrinsicSize; 489 } 490 return result; 491 } 492 493 private static final float[] sScaleOutput = new float[2]; 494 495 @Override onMeasure(int widthMeasureSpec, int heightMeasureSpec)496 protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { 497 super.onMeasure(widthMeasureSpec, heightMeasureSpec); 498 if (mDocument == null) { 499 return; 500 } 501 int preWidth = getWidth(); 502 int preHeight = getHeight(); 503 int w = measureDimension(widthMeasureSpec, mDocument.getWidth()); 504 int h = measureDimension(heightMeasureSpec, mDocument.getHeight()); 505 506 if (!USE_VIEW_AREA_CLICK) { 507 if (mDocument.getDocument().getContentSizing() == RootContentBehavior.SIZING_SCALE) { 508 mDocument.getDocument().computeScale(w, h, sScaleOutput); 509 w = (int) (mDocument.getWidth() * sScaleOutput[0]); 510 h = (int) (mDocument.getHeight() * sScaleOutput[1]); 511 } 512 } 513 setMeasuredDimension(w, h); 514 if (preWidth != w || preHeight != h) { 515 mDocument.getDocument().invalidateMeasure(); 516 } 517 } 518 519 private int mCount; 520 private long mTime = System.nanoTime(); 521 private long mDuration; 522 private boolean mEvalTime = false; // turn on to measure eval time 523 private float mLastAnimationTime = 0.1f; // set to random non 0 number 524 private boolean mDisable = false; 525 526 /** 527 * This returns the amount of time in ms the player used to evalueate a pass it is averaged over 528 * a number of evaluations. 529 * 530 * @return time in ms 531 */ getEvalTime()532 public float getEvalTime() { 533 if (!mEvalTime) { 534 mEvalTime = true; 535 return 0.0f; 536 } 537 double avg = mDuration / (double) mCount; 538 if (mCount > 100) { 539 mDuration /= 2; 540 mCount /= 2; 541 } 542 return (float) (avg * 1E-6); // ms 543 } 544 545 @Override onDraw(Canvas canvas)546 protected void onDraw(Canvas canvas) { 547 super.onDraw(canvas); 548 if (mDocument == null) { 549 return; 550 } 551 if (mDisable) { 552 drawDisable(canvas); 553 return; 554 } 555 try { 556 557 long start = mEvalTime ? System.nanoTime() : 0; // measure execut of commands 558 559 float animationTime = (System.nanoTime() - mStart) * 1E-9f; 560 mARContext.setAnimationTime(animationTime); 561 mARContext.loadFloat(RemoteContext.ID_ANIMATION_TIME, animationTime); 562 float loopTime = animationTime - mLastAnimationTime; 563 mARContext.loadFloat(RemoteContext.ID_ANIMATION_DELTA_TIME, loopTime); 564 mLastAnimationTime = animationTime; 565 mARContext.setAnimationEnabled(true); 566 mARContext.currentTime = System.currentTimeMillis(); 567 mARContext.setDebug(mDebug); 568 float density = getContext().getResources().getDisplayMetrics().density; 569 mARContext.useCanvas(canvas); 570 mARContext.mWidth = getWidth(); 571 mARContext.mHeight = getHeight(); 572 mDocument.paint(mARContext, mTheme); 573 if (mDebug == 1) { 574 mCount++; 575 if (System.nanoTime() - mTime > 1000000000L) { 576 System.out.println(" count " + mCount + " fps"); 577 mCount = 0; 578 mTime = System.nanoTime(); 579 } 580 } 581 int nextFrame = mDocument.needsRepaint(); 582 if (nextFrame > 0) { 583 if (mMaxFrameRate >= POST_TO_NEXT_FRAME_THRESHOLD) { 584 mLastFrameDelay = nextFrame; 585 } else { 586 mLastFrameDelay = Math.max(mMaxFrameDelay, nextFrame); 587 } 588 if (mChoreographer != null) { 589 if (mDebug == 1) { 590 System.err.println( 591 "RC : POST CHOREOGRAPHER WITH " 592 + mLastFrameDelay 593 + " (nextFrame was " 594 + nextFrame 595 + ", max delay " 596 + mMaxFrameDelay 597 + ", " 598 + " max framerate is " 599 + mMaxFrameRate 600 + ")"); 601 } 602 mChoreographer.postFrameCallbackDelayed(mFrameCallback, mLastFrameDelay); 603 } 604 if (!mARContext.useChoreographer()) { 605 invalidate(); 606 } 607 } else { 608 if (mChoreographer != null) { 609 mChoreographer.removeFrameCallback(mFrameCallback); 610 } 611 } 612 if (mEvalTime) { 613 mDuration += System.nanoTime() - start; 614 mCount++; 615 } 616 } catch (Exception ex) { 617 mARContext.getLastOpCount(); 618 mDisable = true; 619 invalidate(); 620 } 621 if (mDebug == 1) { 622 long frameDelay = System.currentTimeMillis() - mLastFrameCall; 623 System.err.println( 624 "RC : Delay since last frame " 625 + frameDelay 626 + " ms (" 627 + (1000f / (float) frameDelay) 628 + " fps)"); 629 mLastFrameCall = System.currentTimeMillis(); 630 } 631 } 632 drawDisable(Canvas canvas)633 private void drawDisable(Canvas canvas) { 634 Rect rect = new Rect(); 635 canvas.drawColor(Color.BLACK); 636 Paint paint = new Paint(); 637 paint.setTextSize(128f); 638 paint.setColor(Color.RED); 639 int w = getWidth(); 640 int h = getHeight(); 641 642 String str = "⚠"; 643 paint.getTextBounds(str, 0, 1, rect); 644 645 float x = w / 2f - rect.width() / 2f - rect.left; 646 float y = h / 2f + rect.height() / 2f - rect.bottom; 647 648 canvas.drawText(str, x, y, paint); 649 } 650 getDefaultTextSize()651 private float getDefaultTextSize() { 652 return new TextView(getContext()).getTextSize(); 653 } 654 } 655