1 /* 2 * Copyright 2016 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 package org.webrtc; 12 13 import static org.junit.Assert.assertEquals; 14 import static org.junit.Assert.assertFalse; 15 import static org.junit.Assert.assertNotNull; 16 import static org.junit.Assert.assertNull; 17 import static org.junit.Assert.assertTrue; 18 import static org.junit.Assert.fail; 19 20 import android.graphics.Bitmap; 21 import android.graphics.SurfaceTexture; 22 import android.opengl.GLES11Ext; 23 import android.opengl.GLES20; 24 import android.support.test.InstrumentationRegistry; 25 import androidx.test.filters.SmallTest; 26 import java.nio.ByteBuffer; 27 import java.util.ArrayList; 28 import java.util.Arrays; 29 import java.util.concurrent.CountDownLatch; 30 import org.junit.After; 31 import org.junit.Before; 32 import org.junit.Test; 33 34 // EmptyActivity is needed for the surface. 35 public class EglRendererTest { 36 private final static String TAG = "EglRendererTest"; 37 private final static int RENDER_WAIT_MS = 1000; 38 private final static int SURFACE_WAIT_MS = 1000; 39 private final static int TEST_FRAME_WIDTH = 4; 40 private final static int TEST_FRAME_HEIGHT = 4; 41 private final static int REMOVE_FRAME_LISTENER_RACY_NUM_TESTS = 10; 42 // Some arbitrary frames. 43 private final static byte[][][] TEST_FRAMES_DATA = { 44 { 45 new byte[] { 46 -99, -93, -88, -83, -78, -73, -68, -62, -56, -52, -46, -41, -36, -31, -26, -20}, 47 new byte[] {110, 113, 116, 118}, new byte[] {31, 45, 59, 73}, 48 }, 49 { 50 new byte[] { 51 -108, -103, -98, -93, -87, -82, -77, -72, -67, -62, -56, -50, -45, -40, -35, -30}, 52 new byte[] {120, 123, 125, -127}, new byte[] {87, 100, 114, 127}, 53 }, 54 { 55 new byte[] { 56 -117, -112, -107, -102, -97, -92, -87, -81, -75, -71, -65, -60, -55, -50, -44, -39}, 57 new byte[] {113, 116, 118, 120}, new byte[] {45, 59, 73, 87}, 58 }, 59 }; 60 private final static ByteBuffer[][] TEST_FRAMES = 61 copyTestDataToDirectByteBuffers(TEST_FRAMES_DATA); 62 63 private static class TestFrameListener implements EglRenderer.FrameListener { 64 final private ArrayList<Bitmap> bitmaps = new ArrayList<Bitmap>(); 65 boolean bitmapReceived; 66 Bitmap storedBitmap; 67 68 @Override 69 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. 70 @SuppressWarnings("NoSynchronizedMethodCheck") onFrame(Bitmap bitmap)71 public synchronized void onFrame(Bitmap bitmap) { 72 if (bitmapReceived) { 73 fail("Unexpected bitmap was received."); 74 } 75 76 bitmapReceived = true; 77 storedBitmap = bitmap; 78 notify(); 79 } 80 81 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. 82 @SuppressWarnings("NoSynchronizedMethodCheck") waitForBitmap(int timeoutMs)83 public synchronized boolean waitForBitmap(int timeoutMs) throws InterruptedException { 84 final long endTimeMs = System.currentTimeMillis() + timeoutMs; 85 while (!bitmapReceived) { 86 final long waitTimeMs = endTimeMs - System.currentTimeMillis(); 87 if (waitTimeMs < 0) { 88 return false; 89 } 90 wait(timeoutMs); 91 } 92 return true; 93 } 94 95 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. 96 @SuppressWarnings("NoSynchronizedMethodCheck") resetAndGetBitmap()97 public synchronized Bitmap resetAndGetBitmap() { 98 bitmapReceived = false; 99 return storedBitmap; 100 } 101 } 102 103 final TestFrameListener testFrameListener = new TestFrameListener(); 104 105 EglRenderer eglRenderer; 106 CountDownLatch surfaceReadyLatch = new CountDownLatch(1); 107 int oesTextureId; 108 SurfaceTexture surfaceTexture; 109 110 @Before setUp()111 public void setUp() throws Exception { 112 PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions 113 .builder(InstrumentationRegistry.getTargetContext()) 114 .setNativeLibraryName(TestConstants.NATIVE_LIBRARY) 115 .createInitializationOptions()); 116 eglRenderer = new EglRenderer("TestRenderer: "); 117 eglRenderer.init(null /* sharedContext */, EglBase.CONFIG_RGBA, new GlRectDrawer()); 118 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); 119 surfaceTexture = new SurfaceTexture(oesTextureId); 120 surfaceTexture.setDefaultBufferSize(1 /* width */, 1 /* height */); 121 eglRenderer.createEglSurface(surfaceTexture); 122 } 123 124 @After tearDown()125 public void tearDown() { 126 surfaceTexture.release(); 127 GLES20.glDeleteTextures(1 /* n */, new int[] {oesTextureId}, 0 /* offset */); 128 eglRenderer.release(); 129 } 130 131 /** Checks the bitmap is not null and the correct size. */ checkBitmap(Bitmap bitmap, float scale)132 private static void checkBitmap(Bitmap bitmap, float scale) { 133 assertNotNull(bitmap); 134 assertEquals((int) (TEST_FRAME_WIDTH * scale), bitmap.getWidth()); 135 assertEquals((int) (TEST_FRAME_HEIGHT * scale), bitmap.getHeight()); 136 } 137 138 /** 139 * Does linear sampling on U/V plane of test data. 140 * 141 * @param data Plane data to be sampled from. 142 * @param planeWidth Width of the plane data. This is also assumed to be the stride. 143 * @param planeHeight Height of the plane data. 144 * @param x X-coordinate in range [0, 1]. 145 * @param y Y-coordinate in range [0, 1]. 146 */ linearSample( ByteBuffer plane, int planeWidth, int planeHeight, float x, float y)147 private static float linearSample( 148 ByteBuffer plane, int planeWidth, int planeHeight, float x, float y) { 149 final int stride = planeWidth; 150 151 final float coordX = x * planeWidth; 152 final float coordY = y * planeHeight; 153 154 int lowIndexX = (int) Math.floor(coordX - 0.5f); 155 int lowIndexY = (int) Math.floor(coordY - 0.5f); 156 int highIndexX = lowIndexX + 1; 157 int highIndexY = lowIndexY + 1; 158 159 final float highWeightX = coordX - lowIndexX - 0.5f; 160 final float highWeightY = coordY - lowIndexY - 0.5f; 161 final float lowWeightX = 1f - highWeightX; 162 final float lowWeightY = 1f - highWeightY; 163 164 // Clamp on the edges. 165 lowIndexX = Math.max(0, lowIndexX); 166 lowIndexY = Math.max(0, lowIndexY); 167 highIndexX = Math.min(planeWidth - 1, highIndexX); 168 highIndexY = Math.min(planeHeight - 1, highIndexY); 169 170 float lowYValue = (plane.get(lowIndexY * stride + lowIndexX) & 0xFF) * lowWeightX 171 + (plane.get(lowIndexY * stride + highIndexX) & 0xFF) * highWeightX; 172 float highYValue = (plane.get(highIndexY * stride + lowIndexX) & 0xFF) * lowWeightX 173 + (plane.get(highIndexY * stride + highIndexX) & 0xFF) * highWeightX; 174 175 return lowWeightY * lowYValue + highWeightY * highYValue; 176 } 177 saturatedFloatToByte(float c)178 private static byte saturatedFloatToByte(float c) { 179 return (byte) Math.round(255f * Math.max(0f, Math.min(1f, c))); 180 } 181 182 /** 183 * Converts test data YUV frame to expected RGBA frame. Tries to match the behavior of OpenGL 184 * YUV drawer shader. Does linear sampling on the U- and V-planes. 185 * 186 * @param yuvFrame Array of size 3 containing Y-, U-, V-planes for image of size 187 * (TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT). U- and V-planes should be half the size 188 * of the Y-plane. 189 */ convertYUVFrameToRGBA(ByteBuffer[] yuvFrame)190 private static byte[] convertYUVFrameToRGBA(ByteBuffer[] yuvFrame) { 191 final byte[] argbFrame = new byte[TEST_FRAME_WIDTH * TEST_FRAME_HEIGHT * 4]; 192 final int argbStride = TEST_FRAME_WIDTH * 4; 193 final int yStride = TEST_FRAME_WIDTH; 194 195 final int vStride = TEST_FRAME_WIDTH / 2; 196 197 for (int y = 0; y < TEST_FRAME_HEIGHT; y++) { 198 for (int x = 0; x < TEST_FRAME_WIDTH; x++) { 199 final float yC = ((yuvFrame[0].get(y * yStride + x) & 0xFF) - 16f) / 219f; 200 final float uC = (linearSample(yuvFrame[1], TEST_FRAME_WIDTH / 2, TEST_FRAME_HEIGHT / 2, 201 (x + 0.5f) / TEST_FRAME_WIDTH, (y + 0.5f) / TEST_FRAME_HEIGHT) 202 - 16f) 203 / 224f 204 - 0.5f; 205 final float vC = (linearSample(yuvFrame[2], TEST_FRAME_WIDTH / 2, TEST_FRAME_HEIGHT / 2, 206 (x + 0.5f) / TEST_FRAME_WIDTH, (y + 0.5f) / TEST_FRAME_HEIGHT) 207 - 16f) 208 / 224f 209 - 0.5f; 210 final float rC = yC + 1.403f * vC; 211 final float gC = yC - 0.344f * uC - 0.714f * vC; 212 final float bC = yC + 1.77f * uC; 213 214 argbFrame[y * argbStride + x * 4 + 0] = saturatedFloatToByte(rC); 215 argbFrame[y * argbStride + x * 4 + 1] = saturatedFloatToByte(gC); 216 argbFrame[y * argbStride + x * 4 + 2] = saturatedFloatToByte(bC); 217 argbFrame[y * argbStride + x * 4 + 3] = (byte) 255; 218 } 219 } 220 221 return argbFrame; 222 } 223 224 /** Checks that the bitmap content matches the test frame with the given index. */ 225 // TODO(titovartem) make correct fix during webrtc:9175 226 @SuppressWarnings("ByteBufferBackingArray") checkBitmapContent(Bitmap bitmap, int frame)227 private static void checkBitmapContent(Bitmap bitmap, int frame) { 228 checkBitmap(bitmap, 1f); 229 230 byte[] expectedRGBA = convertYUVFrameToRGBA(TEST_FRAMES[frame]); 231 ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(bitmap.getByteCount()); 232 bitmap.copyPixelsToBuffer(bitmapBuffer); 233 234 for (int i = 0; i < expectedRGBA.length; i++) { 235 int expected = expectedRGBA[i] & 0xFF; 236 int value = bitmapBuffer.get(i) & 0xFF; 237 // Due to unknown conversion differences check value matches +-1. 238 if (Math.abs(value - expected) > 1) { 239 Logging.d(TAG, "Expected bitmap content: " + Arrays.toString(expectedRGBA)); 240 Logging.d(TAG, "Bitmap content: " + Arrays.toString(bitmapBuffer.array())); 241 fail("Frame doesn't match original frame on byte " + i + ". Expected: " + expected 242 + " Result: " + value); 243 } 244 } 245 } 246 247 /** Tells eglRenderer to render test frame with given index. */ feedFrame(int i)248 private void feedFrame(int i) { 249 final VideoFrame.I420Buffer buffer = JavaI420Buffer.wrap(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, 250 TEST_FRAMES[i][0], TEST_FRAME_WIDTH, TEST_FRAMES[i][1], TEST_FRAME_WIDTH / 2, 251 TEST_FRAMES[i][2], TEST_FRAME_WIDTH / 2, null /* releaseCallback */); 252 final VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, 0 /* timestamp */); 253 eglRenderer.onFrame(frame); 254 frame.release(); 255 } 256 257 @Test 258 @SmallTest testAddFrameListener()259 public void testAddFrameListener() throws Exception { 260 eglRenderer.addFrameListener(testFrameListener, 0f /* scaleFactor */); 261 feedFrame(0); 262 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 263 assertNull(testFrameListener.resetAndGetBitmap()); 264 eglRenderer.addFrameListener(testFrameListener, 0f /* scaleFactor */); 265 feedFrame(1); 266 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 267 assertNull(testFrameListener.resetAndGetBitmap()); 268 feedFrame(2); 269 // Check we get no more bitmaps than two. 270 assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 271 } 272 273 @Test 274 @SmallTest testAddFrameListenerBitmap()275 public void testAddFrameListenerBitmap() throws Exception { 276 eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); 277 feedFrame(0); 278 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 279 checkBitmapContent(testFrameListener.resetAndGetBitmap(), 0); 280 eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); 281 feedFrame(1); 282 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 283 checkBitmapContent(testFrameListener.resetAndGetBitmap(), 1); 284 } 285 286 @Test 287 @SmallTest testAddFrameListenerBitmapScale()288 public void testAddFrameListenerBitmapScale() throws Exception { 289 for (int i = 0; i < 3; ++i) { 290 float scale = i * 0.5f + 0.5f; 291 eglRenderer.addFrameListener(testFrameListener, scale); 292 feedFrame(i); 293 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 294 checkBitmap(testFrameListener.resetAndGetBitmap(), scale); 295 } 296 } 297 298 /** 299 * Checks that the frame listener will not be called with a frame that was delivered before the 300 * frame listener was added. 301 */ 302 @Test 303 @SmallTest testFrameListenerNotCalledWithOldFrames()304 public void testFrameListenerNotCalledWithOldFrames() throws Exception { 305 feedFrame(0); 306 eglRenderer.addFrameListener(testFrameListener, 0f); 307 // Check the old frame does not trigger frame listener. 308 assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 309 } 310 311 /** Checks that the frame listener will not be called after it is removed. */ 312 @Test 313 @SmallTest testRemoveFrameListenerNotRacy()314 public void testRemoveFrameListenerNotRacy() throws Exception { 315 for (int i = 0; i < REMOVE_FRAME_LISTENER_RACY_NUM_TESTS; i++) { 316 feedFrame(0); 317 eglRenderer.addFrameListener(testFrameListener, 0f); 318 eglRenderer.removeFrameListener(testFrameListener); 319 feedFrame(1); 320 } 321 // Check the frame listener hasn't triggered. 322 assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 323 } 324 325 @Test 326 @SmallTest testFrameListenersFpsReduction()327 public void testFrameListenersFpsReduction() throws Exception { 328 // Test that normal frame listeners receive frames while the renderer is paused. 329 eglRenderer.pauseVideo(); 330 eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); 331 feedFrame(0); 332 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 333 checkBitmapContent(testFrameListener.resetAndGetBitmap(), 0); 334 335 // Test that frame listeners with FPS reduction applied receive frames while the renderer is not 336 // paused. 337 eglRenderer.disableFpsReduction(); 338 eglRenderer.addFrameListener( 339 testFrameListener, 1f /* scaleFactor */, null, true /* applyFpsReduction */); 340 feedFrame(1); 341 assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 342 checkBitmapContent(testFrameListener.resetAndGetBitmap(), 1); 343 344 // Test that frame listeners with FPS reduction applied will not receive frames while the 345 // renderer is paused. 346 eglRenderer.pauseVideo(); 347 eglRenderer.addFrameListener( 348 testFrameListener, 1f /* scaleFactor */, null, true /* applyFpsReduction */); 349 feedFrame(1); 350 assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); 351 } 352 copyTestDataToDirectByteBuffers(byte[][][] testData)353 private static ByteBuffer[][] copyTestDataToDirectByteBuffers(byte[][][] testData) { 354 final ByteBuffer[][] result = new ByteBuffer[testData.length][]; 355 356 for (int i = 0; i < testData.length; i++) { 357 result[i] = new ByteBuffer[testData[i].length]; 358 for (int j = 0; j < testData[i].length; j++) { 359 result[i][j] = ByteBuffer.allocateDirect(testData[i][j].length); 360 result[i][j].put(testData[i][j]); 361 result[i][j].rewind(); 362 } 363 } 364 return result; 365 } 366 } 367