1 /* 2 * libjingle 3 * Copyright 2015 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 package org.webrtc; 29 30 import android.content.Context; 31 import android.content.res.Resources.NotFoundException; 32 import android.graphics.Point; 33 import android.opengl.GLES20; 34 import android.os.Handler; 35 import android.os.HandlerThread; 36 import android.util.AttributeSet; 37 import android.view.SurfaceHolder; 38 import android.view.SurfaceView; 39 40 import org.webrtc.Logging; 41 42 import java.util.concurrent.CountDownLatch; 43 44 import javax.microedition.khronos.egl.EGLContext; 45 46 /** 47 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView. 48 * renderFrame() is asynchronous to avoid blocking the calling thread. 49 * This class is thread safe and handles access from potentially four different threads: 50 * Interaction from the main app in init, release, setMirror, and setScalingtype. 51 * Interaction from C++ webrtc::VideoRendererInterface in renderFrame and canApplyRotation. 52 * Interaction from the Activity lifecycle in surfaceCreated, surfaceChanged, and surfaceDestroyed. 53 * Interaction with the layout framework in onMeasure and onSizeChanged. 54 */ 55 public class SurfaceViewRenderer extends SurfaceView 56 implements SurfaceHolder.Callback, VideoRenderer.Callbacks { 57 private static final String TAG = "SurfaceViewRenderer"; 58 59 // Dedicated render thread. 60 private HandlerThread renderThread; 61 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized 62 // on |handlerLock|. 63 private final Object handlerLock = new Object(); 64 private Handler renderThreadHandler; 65 66 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed 67 // from the render thread. 68 private EglBase eglBase; 69 private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader(); 70 private RendererCommon.GlDrawer drawer; 71 // Texture ids for YUV frames. Allocated on first arrival of a YUV frame. 72 private int[] yuvTextures = null; 73 74 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|. 75 private final Object frameLock = new Object(); 76 private VideoRenderer.I420Frame pendingFrame; 77 78 // These variables are synchronized on |layoutLock|. 79 private final Object layoutLock = new Object(); 80 // These dimension values are used to keep track of the state in these functions: onMeasure(), 81 // onLayout(), and surfaceChanged(). A new layout is triggered with requestLayout(). This happens 82 // internally when the incoming frame size changes. requestLayout() can also be triggered 83 // externally. The layout change is a two pass process: first onMeasure() is called in a top-down 84 // traversal of the View tree, followed by an onLayout() pass that is also top-down. During the 85 // onLayout() pass, each parent is responsible for positioning its children using the sizes 86 // computed in the measure pass. 87 // |desiredLayoutsize| is the layout size we have requested in onMeasure() and are waiting for to 88 // take effect. 89 private Point desiredLayoutSize = new Point(); 90 // |layoutSize|/|surfaceSize| is the actual current layout/surface size. They are updated in 91 // onLayout() and surfaceChanged() respectively. 92 private final Point layoutSize = new Point(); 93 // TODO(magjed): Enable hardware scaler with SurfaceHolder.setFixedSize(). This will decouple 94 // layout and surface size. 95 private final Point surfaceSize = new Point(); 96 // |isSurfaceCreated| keeps track of the current status in surfaceCreated()/surfaceDestroyed(). 97 private boolean isSurfaceCreated; 98 // Last rendered frame dimensions, or 0 if no frame has been rendered yet. 99 private int frameWidth; 100 private int frameHeight; 101 private int frameRotation; 102 // |scalingType| determines how the video will fill the allowed layout area in onMeasure(). 103 private RendererCommon.ScalingType scalingType = RendererCommon.ScalingType.SCALE_ASPECT_BALANCED; 104 // If true, mirrors the video stream horizontally. 105 private boolean mirror; 106 // Callback for reporting renderer events. 107 private RendererCommon.RendererEvents rendererEvents; 108 109 // These variables are synchronized on |statisticsLock|. 110 private final Object statisticsLock = new Object(); 111 // Total number of video frames received in renderFrame() call. 112 private int framesReceived; 113 // Number of video frames dropped by renderFrame() because previous frame has not been rendered 114 // yet. 115 private int framesDropped; 116 // Number of rendered video frames. 117 private int framesRendered; 118 // Time in ns when the first video frame was rendered. 119 private long firstFrameTimeNs; 120 // Time in ns spent in renderFrameOnRenderThread() function. 121 private long renderTimeNs; 122 123 // Runnable for posting frames to render thread. 124 private final Runnable renderFrameRunnable = new Runnable() { 125 @Override public void run() { 126 renderFrameOnRenderThread(); 127 } 128 }; 129 // Runnable for clearing Surface to black. 130 private final Runnable makeBlackRunnable = new Runnable() { 131 @Override public void run() { 132 makeBlack(); 133 } 134 }; 135 136 /** 137 * Standard View constructor. In order to render something, you must first call init(). 138 */ SurfaceViewRenderer(Context context)139 public SurfaceViewRenderer(Context context) { 140 super(context); 141 getHolder().addCallback(this); 142 } 143 144 /** 145 * Standard View constructor. In order to render something, you must first call init(). 146 */ SurfaceViewRenderer(Context context, AttributeSet attrs)147 public SurfaceViewRenderer(Context context, AttributeSet attrs) { 148 super(context, attrs); 149 getHolder().addCallback(this); 150 } 151 152 /** 153 * Initialize this class, sharing resources with |sharedContext|. It is allowed to call init() to 154 * reinitialize the renderer after a previous init()/release() cycle. 155 */ init( EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents)156 public void init( 157 EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents) { 158 init(sharedContext, rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawer()); 159 } 160 161 /** 162 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used 163 * for drawing frames on the EGLSurface. This class is responsible for calling release() on 164 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous 165 * init()/release() cycle. 166 */ init(EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents, int[] configAttributes, RendererCommon.GlDrawer drawer)167 public void init(EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents, 168 int[] configAttributes, RendererCommon.GlDrawer drawer) { 169 synchronized (handlerLock) { 170 if (renderThreadHandler != null) { 171 throw new IllegalStateException(getResourceName() + "Already initialized"); 172 } 173 Logging.d(TAG, getResourceName() + "Initializing."); 174 this.rendererEvents = rendererEvents; 175 this.drawer = drawer; 176 renderThread = new HandlerThread(TAG); 177 renderThread.start(); 178 eglBase = EglBase.create(sharedContext, configAttributes); 179 renderThreadHandler = new Handler(renderThread.getLooper()); 180 } 181 tryCreateEglSurface(); 182 } 183 184 /** 185 * Create and make an EGLSurface current if both init() and surfaceCreated() have been called. 186 */ tryCreateEglSurface()187 public void tryCreateEglSurface() { 188 // |renderThreadHandler| is only created after |eglBase| is created in init(), so the 189 // following code will only execute if eglBase != null. 190 runOnRenderThread(new Runnable() { 191 @Override public void run() { 192 synchronized (layoutLock) { 193 if (isSurfaceCreated && !eglBase.hasSurface()) { 194 eglBase.createSurface(getHolder().getSurface()); 195 eglBase.makeCurrent(); 196 // Necessary for YUV frames with odd width. 197 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); 198 } 199 } 200 } 201 }); 202 } 203 204 /** 205 * Block until any pending frame is returned and all GL resources released, even if an interrupt 206 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function 207 * should be called before the Activity is destroyed and the EGLContext is still valid. If you 208 * don't call this function, the GL resources might leak. 209 */ release()210 public void release() { 211 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1); 212 synchronized (handlerLock) { 213 if (renderThreadHandler == null) { 214 Logging.d(TAG, getResourceName() + "Already released"); 215 return; 216 } 217 // Release EGL and GL resources on render thread. 218 // TODO(magjed): This might not be necessary - all OpenGL resources are automatically deleted 219 // when the EGL context is lost. It might be dangerous to delete them manually in 220 // Activity.onDestroy(). 221 renderThreadHandler.postAtFrontOfQueue(new Runnable() { 222 @Override public void run() { 223 drawer.release(); 224 drawer = null; 225 if (yuvTextures != null) { 226 GLES20.glDeleteTextures(3, yuvTextures, 0); 227 yuvTextures = null; 228 } 229 // Clear last rendered image to black. 230 makeBlack(); 231 eglBase.release(); 232 eglBase = null; 233 eglCleanupBarrier.countDown(); 234 } 235 }); 236 // Don't accept any more frames or messages to the render thread. 237 renderThreadHandler = null; 238 } 239 // Make sure the EGL/GL cleanup posted above is executed. 240 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier); 241 renderThread.quit(); 242 synchronized (frameLock) { 243 if (pendingFrame != null) { 244 VideoRenderer.renderFrameDone(pendingFrame); 245 pendingFrame = null; 246 } 247 } 248 // The |renderThread| cleanup is not safe to cancel and we need to wait until it's done. 249 ThreadUtils.joinUninterruptibly(renderThread); 250 renderThread = null; 251 // Reset statistics and event reporting. 252 synchronized (layoutLock) { 253 frameWidth = 0; 254 frameHeight = 0; 255 frameRotation = 0; 256 rendererEvents = null; 257 } 258 resetStatistics(); 259 } 260 261 /** 262 * Reset statistics. This will reset the logged statistics in logStatistics(), and 263 * RendererEvents.onFirstFrameRendered() will be called for the next frame. 264 */ resetStatistics()265 public void resetStatistics() { 266 synchronized (statisticsLock) { 267 framesReceived = 0; 268 framesDropped = 0; 269 framesRendered = 0; 270 firstFrameTimeNs = 0; 271 renderTimeNs = 0; 272 } 273 } 274 275 /** 276 * Set if the video stream should be mirrored or not. 277 */ setMirror(final boolean mirror)278 public void setMirror(final boolean mirror) { 279 synchronized (layoutLock) { 280 this.mirror = mirror; 281 } 282 } 283 284 /** 285 * Set how the video will fill the allowed layout area. 286 */ setScalingType(RendererCommon.ScalingType scalingType)287 public void setScalingType(RendererCommon.ScalingType scalingType) { 288 synchronized (layoutLock) { 289 this.scalingType = scalingType; 290 } 291 } 292 293 // VideoRenderer.Callbacks interface. 294 @Override renderFrame(VideoRenderer.I420Frame frame)295 public void renderFrame(VideoRenderer.I420Frame frame) { 296 synchronized (statisticsLock) { 297 ++framesReceived; 298 } 299 synchronized (handlerLock) { 300 if (renderThreadHandler == null) { 301 Logging.d(TAG, getResourceName() 302 + "Dropping frame - Not initialized or already released."); 303 VideoRenderer.renderFrameDone(frame); 304 return; 305 } 306 synchronized (frameLock) { 307 if (pendingFrame != null) { 308 // Drop old frame. 309 synchronized (statisticsLock) { 310 ++framesDropped; 311 } 312 VideoRenderer.renderFrameDone(pendingFrame); 313 } 314 pendingFrame = frame; 315 updateFrameDimensionsAndReportEvents(frame); 316 renderThreadHandler.post(renderFrameRunnable); 317 } 318 } 319 } 320 321 // Returns desired layout size given current measure specification and video aspect ratio. getDesiredLayoutSize(int widthSpec, int heightSpec)322 private Point getDesiredLayoutSize(int widthSpec, int heightSpec) { 323 synchronized (layoutLock) { 324 final int maxWidth = getDefaultSize(Integer.MAX_VALUE, widthSpec); 325 final int maxHeight = getDefaultSize(Integer.MAX_VALUE, heightSpec); 326 final Point size = 327 RendererCommon.getDisplaySize(scalingType, frameAspectRatio(), maxWidth, maxHeight); 328 if (MeasureSpec.getMode(widthSpec) == MeasureSpec.EXACTLY) { 329 size.x = maxWidth; 330 } 331 if (MeasureSpec.getMode(heightSpec) == MeasureSpec.EXACTLY) { 332 size.y = maxHeight; 333 } 334 return size; 335 } 336 } 337 338 // View layout interface. 339 @Override onMeasure(int widthSpec, int heightSpec)340 protected void onMeasure(int widthSpec, int heightSpec) { 341 synchronized (layoutLock) { 342 if (frameWidth == 0 || frameHeight == 0) { 343 super.onMeasure(widthSpec, heightSpec); 344 return; 345 } 346 desiredLayoutSize = getDesiredLayoutSize(widthSpec, heightSpec); 347 if (desiredLayoutSize.x != getMeasuredWidth() || desiredLayoutSize.y != getMeasuredHeight()) { 348 // Clear the surface asap before the layout change to avoid stretched video and other 349 // render artifacs. Don't wait for it to finish because the IO thread should never be 350 // blocked, so it's a best-effort attempt. 351 synchronized (handlerLock) { 352 if (renderThreadHandler != null) { 353 renderThreadHandler.postAtFrontOfQueue(makeBlackRunnable); 354 } 355 } 356 } 357 setMeasuredDimension(desiredLayoutSize.x, desiredLayoutSize.y); 358 } 359 } 360 361 @Override onLayout(boolean changed, int left, int top, int right, int bottom)362 protected void onLayout(boolean changed, int left, int top, int right, int bottom) { 363 synchronized (layoutLock) { 364 layoutSize.x = right - left; 365 layoutSize.y = bottom - top; 366 } 367 // Might have a pending frame waiting for a layout of correct size. 368 runOnRenderThread(renderFrameRunnable); 369 } 370 371 // SurfaceHolder.Callback interface. 372 @Override surfaceCreated(final SurfaceHolder holder)373 public void surfaceCreated(final SurfaceHolder holder) { 374 Logging.d(TAG, getResourceName() + "Surface created."); 375 synchronized (layoutLock) { 376 isSurfaceCreated = true; 377 } 378 tryCreateEglSurface(); 379 } 380 381 @Override surfaceDestroyed(SurfaceHolder holder)382 public void surfaceDestroyed(SurfaceHolder holder) { 383 Logging.d(TAG, getResourceName() + "Surface destroyed."); 384 synchronized (layoutLock) { 385 isSurfaceCreated = false; 386 surfaceSize.x = 0; 387 surfaceSize.y = 0; 388 } 389 runOnRenderThread(new Runnable() { 390 @Override public void run() { 391 eglBase.releaseSurface(); 392 } 393 }); 394 } 395 396 @Override surfaceChanged(SurfaceHolder holder, int format, int width, int height)397 public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { 398 Logging.d(TAG, getResourceName() + "Surface changed: " + width + "x" + height); 399 synchronized (layoutLock) { 400 surfaceSize.x = width; 401 surfaceSize.y = height; 402 } 403 // Might have a pending frame waiting for a surface of correct size. 404 runOnRenderThread(renderFrameRunnable); 405 } 406 407 /** 408 * Private helper function to post tasks safely. 409 */ runOnRenderThread(Runnable runnable)410 private void runOnRenderThread(Runnable runnable) { 411 synchronized (handlerLock) { 412 if (renderThreadHandler != null) { 413 renderThreadHandler.post(runnable); 414 } 415 } 416 } 417 getResourceName()418 private String getResourceName() { 419 try { 420 return getResources().getResourceEntryName(getId()) + ": "; 421 } catch (NotFoundException e) { 422 return ""; 423 } 424 } 425 makeBlack()426 private void makeBlack() { 427 if (Thread.currentThread() != renderThread) { 428 throw new IllegalStateException(getResourceName() + "Wrong thread."); 429 } 430 if (eglBase != null && eglBase.hasSurface()) { 431 GLES20.glClearColor(0, 0, 0, 0); 432 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 433 eglBase.swapBuffers(); 434 } 435 } 436 437 /** 438 * Requests new layout if necessary. Returns true if layout and surface size are consistent. 439 */ checkConsistentLayout()440 private boolean checkConsistentLayout() { 441 if (Thread.currentThread() != renderThread) { 442 throw new IllegalStateException(getResourceName() + "Wrong thread."); 443 } 444 synchronized (layoutLock) { 445 // Return false while we are in the middle of a layout change. 446 return layoutSize.equals(desiredLayoutSize) && surfaceSize.equals(layoutSize); 447 } 448 } 449 450 /** 451 * Renders and releases |pendingFrame|. 452 */ renderFrameOnRenderThread()453 private void renderFrameOnRenderThread() { 454 if (Thread.currentThread() != renderThread) { 455 throw new IllegalStateException(getResourceName() + "Wrong thread."); 456 } 457 // Fetch and render |pendingFrame|. 458 final VideoRenderer.I420Frame frame; 459 synchronized (frameLock) { 460 if (pendingFrame == null) { 461 return; 462 } 463 frame = pendingFrame; 464 pendingFrame = null; 465 } 466 if (eglBase == null || !eglBase.hasSurface()) { 467 Logging.d(TAG, getResourceName() + "No surface to draw on"); 468 VideoRenderer.renderFrameDone(frame); 469 return; 470 } 471 if (!checkConsistentLayout()) { 472 // Output intermediate black frames while the layout is updated. 473 makeBlack(); 474 VideoRenderer.renderFrameDone(frame); 475 return; 476 } 477 // After a surface size change, the EGLSurface might still have a buffer of the old size in the 478 // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet 479 // changed. Such a buffer will be rendered incorrectly, so flush it with a black frame. 480 synchronized (layoutLock) { 481 if (eglBase.surfaceWidth() != surfaceSize.x || eglBase.surfaceHeight() != surfaceSize.y) { 482 makeBlack(); 483 } 484 } 485 486 final long startTimeNs = System.nanoTime(); 487 final float[] texMatrix; 488 synchronized (layoutLock) { 489 final float[] rotatedSamplingMatrix = 490 RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree); 491 final float[] layoutMatrix = RendererCommon.getLayoutMatrix( 492 mirror, frameAspectRatio(), (float) layoutSize.x / layoutSize.y); 493 texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix); 494 } 495 496 // TODO(magjed): glClear() shouldn't be necessary since every pixel is covered anyway, but it's 497 // a workaround for bug 5147. Performance will be slightly worse. 498 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 499 if (frame.yuvFrame) { 500 // Make sure YUV textures are allocated. 501 if (yuvTextures == null) { 502 yuvTextures = new int[3]; 503 for (int i = 0; i < 3; i++) { 504 yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); 505 } 506 } 507 yuvUploader.uploadYuvData( 508 yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes); 509 drawer.drawYuv(yuvTextures, texMatrix, 0, 0, surfaceSize.x, surfaceSize.y); 510 } else { 511 drawer.drawOes(frame.textureId, texMatrix, 0, 0, surfaceSize.x, surfaceSize.y); 512 } 513 514 eglBase.swapBuffers(); 515 VideoRenderer.renderFrameDone(frame); 516 synchronized (statisticsLock) { 517 if (framesRendered == 0) { 518 firstFrameTimeNs = startTimeNs; 519 synchronized (layoutLock) { 520 Logging.d(TAG, getResourceName() + "Reporting first rendered frame."); 521 if (rendererEvents != null) { 522 rendererEvents.onFirstFrameRendered(); 523 } 524 } 525 } 526 ++framesRendered; 527 renderTimeNs += (System.nanoTime() - startTimeNs); 528 if (framesRendered % 300 == 0) { 529 logStatistics(); 530 } 531 } 532 } 533 534 // Return current frame aspect ratio, taking rotation into account. frameAspectRatio()535 private float frameAspectRatio() { 536 synchronized (layoutLock) { 537 if (frameWidth == 0 || frameHeight == 0) { 538 return 0.0f; 539 } 540 return (frameRotation % 180 == 0) ? (float) frameWidth / frameHeight 541 : (float) frameHeight / frameWidth; 542 } 543 } 544 545 // Update frame dimensions and report any changes to |rendererEvents|. updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame)546 private void updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame) { 547 synchronized (layoutLock) { 548 if (frameWidth != frame.width || frameHeight != frame.height 549 || frameRotation != frame.rotationDegree) { 550 Logging.d(TAG, getResourceName() + "Reporting frame resolution changed to " 551 + frame.width + "x" + frame.height + " with rotation " + frame.rotationDegree); 552 if (rendererEvents != null) { 553 rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree); 554 } 555 frameWidth = frame.width; 556 frameHeight = frame.height; 557 frameRotation = frame.rotationDegree; 558 post(new Runnable() { 559 @Override public void run() { 560 requestLayout(); 561 } 562 }); 563 } 564 } 565 } 566 logStatistics()567 private void logStatistics() { 568 synchronized (statisticsLock) { 569 Logging.d(TAG, getResourceName() + "Frames received: " 570 + framesReceived + ". Dropped: " + framesDropped + ". Rendered: " + framesRendered); 571 if (framesReceived > 0 && framesRendered > 0) { 572 final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs; 573 Logging.d(TAG, getResourceName() + "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + 574 " ms. FPS: " + framesRendered * 1e9 / timeSinceFirstFrameNs); 575 Logging.d(TAG, getResourceName() + "Average render time: " 576 + (int) (renderTimeNs / (1000 * framesRendered)) + " us."); 577 } 578 } 579 } 580 } 581