1 /* 2 * Copyright 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 package org.webrtc; 12 13 import static com.google.common.truth.Truth.assertThat; 14 import static java.util.concurrent.TimeUnit.SECONDS; 15 import static org.mockito.ArgumentMatchers.any; 16 import static org.mockito.ArgumentMatchers.anyInt; 17 import static org.mockito.ArgumentMatchers.anyLong; 18 import static org.mockito.Mockito.spy; 19 import static org.mockito.Mockito.times; 20 import static org.mockito.Mockito.verify; 21 22 import android.media.MediaCodec; 23 import android.media.MediaCodecInfo; 24 import android.media.MediaFormat; 25 import android.os.Bundle; 26 import androidx.test.runner.AndroidJUnit4; 27 import java.nio.ByteBuffer; 28 import java.util.HashMap; 29 import java.util.Map; 30 import org.junit.Before; 31 import org.junit.Test; 32 import org.junit.runner.RunWith; 33 import org.mockito.ArgumentCaptor; 34 import org.mockito.Mock; 35 import org.mockito.MockitoAnnotations; 36 import org.robolectric.annotation.Config; 37 import org.webrtc.EncodedImage; 38 import org.webrtc.EncodedImage.FrameType; 39 import org.webrtc.FakeMediaCodecWrapper.State; 40 import org.webrtc.VideoCodecStatus; 41 import org.webrtc.VideoEncoder; 42 import org.webrtc.VideoEncoder.BitrateAllocation; 43 import org.webrtc.VideoEncoder.CodecSpecificInfo; 44 import org.webrtc.VideoEncoder.EncodeInfo; 45 import org.webrtc.VideoEncoder.Settings; 46 import org.webrtc.VideoFrame; 47 import org.webrtc.VideoFrame.Buffer; 48 import org.webrtc.VideoFrame.I420Buffer; 49 50 @RunWith(AndroidJUnit4.class) 51 @Config(manifest = Config.NONE) 52 public class HardwareVideoEncoderTest { 53 private static final VideoEncoder.Settings TEST_ENCODER_SETTINGS = new Settings( 54 /* numberOfCores= */ 1, 55 /* width= */ 640, 56 /* height= */ 480, 57 /* startBitrate= */ 10000, 58 /* maxFramerate= */ 30, 59 /* numberOfSimulcastStreams= */ 1, 60 /* automaticResizeOn= */ true, 61 /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)); 62 private static final long POLL_DELAY_MS = 10; 63 private static final long DELIVER_ENCODED_IMAGE_DELAY_MS = 10; 64 private static final EncodeInfo ENCODE_INFO_KEY_FRAME = 65 new EncodeInfo(new FrameType[] {FrameType.VideoFrameKey}); 66 private static final EncodeInfo ENCODE_INFO_DELTA_FRAME = 67 new EncodeInfo(new FrameType[] {FrameType.VideoFrameDelta}); 68 69 private static class TestEncoder extends HardwareVideoEncoder { 70 private final Object deliverEncodedImageLock = new Object(); 71 private boolean deliverEncodedImageDone = true; 72 TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat, Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs, BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext)73 TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, 74 VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat, 75 Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs, 76 BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext) { 77 super(mediaCodecWrapperFactory, codecName, codecType, surfaceColorFormat, yuvColorFormat, 78 params, keyFrameIntervalSec, forceKeyFrameIntervalMs, bitrateAdjuster, sharedContext); 79 } 80 waitDeliverEncodedImage()81 public void waitDeliverEncodedImage() throws InterruptedException { 82 synchronized (deliverEncodedImageLock) { 83 deliverEncodedImageDone = false; 84 deliverEncodedImageLock.notifyAll(); 85 while (!deliverEncodedImageDone) { 86 deliverEncodedImageLock.wait(); 87 } 88 } 89 } 90 91 @SuppressWarnings("WaitNotInLoop") // This method is called inside a loop. 92 @Override deliverEncodedImage()93 protected void deliverEncodedImage() { 94 synchronized (deliverEncodedImageLock) { 95 if (deliverEncodedImageDone) { 96 try { 97 deliverEncodedImageLock.wait(DELIVER_ENCODED_IMAGE_DELAY_MS); 98 } catch (InterruptedException e) { 99 Thread.currentThread().interrupt(); 100 return; 101 } 102 } 103 if (deliverEncodedImageDone) { 104 return; 105 } 106 super.deliverEncodedImage(); 107 deliverEncodedImageDone = true; 108 deliverEncodedImageLock.notifyAll(); 109 } 110 } 111 112 @Override fillInputBuffer(ByteBuffer buffer, Buffer videoFrameBuffer)113 protected void fillInputBuffer(ByteBuffer buffer, Buffer videoFrameBuffer) { 114 I420Buffer i420Buffer = videoFrameBuffer.toI420(); 115 buffer.put(i420Buffer.getDataY()); 116 buffer.put(i420Buffer.getDataU()); 117 buffer.put(i420Buffer.getDataV()); 118 buffer.flip(); 119 i420Buffer.release(); 120 } 121 } 122 123 private class TestEncoderBuilder { 124 private VideoCodecMimeType codecType = VideoCodecMimeType.VP8; 125 private BitrateAdjuster bitrateAdjuster = new BaseBitrateAdjuster(); 126 setCodecType(VideoCodecMimeType codecType)127 public TestEncoderBuilder setCodecType(VideoCodecMimeType codecType) { 128 this.codecType = codecType; 129 return this; 130 } 131 setBitrateAdjuster(BitrateAdjuster bitrateAdjuster)132 public TestEncoderBuilder setBitrateAdjuster(BitrateAdjuster bitrateAdjuster) { 133 this.bitrateAdjuster = bitrateAdjuster; 134 return this; 135 } 136 build()137 public TestEncoder build() { 138 return new TestEncoder((String name) 139 -> fakeMediaCodecWrapper, 140 "org.webrtc.testencoder", codecType, 141 /* surfaceColorFormat= */ null, 142 /* yuvColorFormat= */ MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar, 143 /* params= */ new HashMap<>(), 144 /* keyFrameIntervalSec= */ 0, 145 /* forceKeyFrameIntervalMs= */ 0, bitrateAdjuster, 146 /* sharedContext= */ null); 147 } 148 } 149 createTestVideoFrame(long timestampNs)150 private VideoFrame createTestVideoFrame(long timestampNs) { 151 byte[] i420 = CodecTestHelper.generateRandomData( 152 TEST_ENCODER_SETTINGS.width * TEST_ENCODER_SETTINGS.height * 3 / 2); 153 final VideoFrame.I420Buffer testBuffer = 154 CodecTestHelper.wrapI420(TEST_ENCODER_SETTINGS.width, TEST_ENCODER_SETTINGS.height, i420); 155 return new VideoFrame(testBuffer, /* rotation= */ 0, timestampNs); 156 } 157 158 @Mock VideoEncoder.Callback mockEncoderCallback; 159 private FakeMediaCodecWrapper fakeMediaCodecWrapper; 160 161 @Before setUp()162 public void setUp() { 163 MockitoAnnotations.initMocks(this); 164 MediaFormat inputFormat = new MediaFormat(); 165 MediaFormat outputFormat = new MediaFormat(); 166 // TODO(sakal): Add more details to output format as needed. 167 fakeMediaCodecWrapper = spy(new FakeMediaCodecWrapper(inputFormat, outputFormat)); 168 } 169 170 @Test testInit()171 public void testInit() { 172 // Set-up. 173 HardwareVideoEncoder encoder = 174 new TestEncoderBuilder().setCodecType(VideoCodecMimeType.VP8).build(); 175 176 // Test. 177 assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback)) 178 .isEqualTo(VideoCodecStatus.OK); 179 180 // Verify. 181 assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.EXECUTING_RUNNING); 182 183 MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat(); 184 assertThat(mediaFormat).isNotNull(); 185 assertThat(mediaFormat.getInteger(MediaFormat.KEY_WIDTH)) 186 .isEqualTo(TEST_ENCODER_SETTINGS.width); 187 assertThat(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) 188 .isEqualTo(TEST_ENCODER_SETTINGS.height); 189 assertThat(mediaFormat.getString(MediaFormat.KEY_MIME)) 190 .isEqualTo(VideoCodecMimeType.VP8.mimeType()); 191 192 assertThat(fakeMediaCodecWrapper.getConfiguredFlags()) 193 .isEqualTo(MediaCodec.CONFIGURE_FLAG_ENCODE); 194 } 195 196 @Test testEncodeByteBuffer()197 public void testEncodeByteBuffer() { 198 // Set-up. 199 HardwareVideoEncoder encoder = new TestEncoderBuilder().build(); 200 encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback); 201 202 // Test. 203 byte[] i420 = CodecTestHelper.generateRandomData( 204 TEST_ENCODER_SETTINGS.width * TEST_ENCODER_SETTINGS.height * 3 / 2); 205 final VideoFrame.I420Buffer testBuffer = 206 CodecTestHelper.wrapI420(TEST_ENCODER_SETTINGS.width, TEST_ENCODER_SETTINGS.height, i420); 207 final VideoFrame testFrame = 208 new VideoFrame(testBuffer, /* rotation= */ 0, /* timestampNs= */ 0); 209 assertThat(encoder.encode(testFrame, new EncodeInfo(new FrameType[] {FrameType.VideoFrameKey}))) 210 .isEqualTo(VideoCodecStatus.OK); 211 212 // Verify. 213 ArgumentCaptor<Integer> indexCaptor = ArgumentCaptor.forClass(Integer.class); 214 ArgumentCaptor<Integer> offsetCaptor = ArgumentCaptor.forClass(Integer.class); 215 ArgumentCaptor<Integer> sizeCaptor = ArgumentCaptor.forClass(Integer.class); 216 verify(fakeMediaCodecWrapper) 217 .queueInputBuffer(indexCaptor.capture(), offsetCaptor.capture(), sizeCaptor.capture(), 218 anyLong(), anyInt()); 219 ByteBuffer buffer = fakeMediaCodecWrapper.getInputBuffer(indexCaptor.getValue()); 220 CodecTestHelper.assertEqualContents( 221 i420, buffer, offsetCaptor.getValue(), sizeCaptor.getValue()); 222 } 223 224 @Test testDeliversOutputData()225 public void testDeliversOutputData() throws InterruptedException { 226 // Set-up. 227 TestEncoder encoder = new TestEncoderBuilder().build(); 228 encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback); 229 encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME); 230 231 // Test. 232 byte[] outputData = CodecTestHelper.generateRandomData(100); 233 fakeMediaCodecWrapper.addOutputData(outputData, 234 /* presentationTimestampUs= */ 0, 235 /* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME); 236 237 encoder.waitDeliverEncodedImage(); 238 239 // Verify. 240 ArgumentCaptor<EncodedImage> videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class); 241 verify(mockEncoderCallback) 242 .onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class)); 243 244 EncodedImage videoFrame = videoFrameCaptor.getValue(); 245 assertThat(videoFrame).isNotNull(); 246 assertThat(videoFrame.encodedWidth).isEqualTo(TEST_ENCODER_SETTINGS.width); 247 assertThat(videoFrame.encodedHeight).isEqualTo(TEST_ENCODER_SETTINGS.height); 248 assertThat(videoFrame.rotation).isEqualTo(0); 249 assertThat(videoFrame.captureTimeNs).isEqualTo(42); 250 assertThat(videoFrame.frameType).isEqualTo(FrameType.VideoFrameKey); 251 CodecTestHelper.assertEqualContents( 252 outputData, videoFrame.buffer, /* offset= */ 0, videoFrame.buffer.capacity()); 253 } 254 255 @Test testRelease()256 public void testRelease() { 257 // Set-up. 258 HardwareVideoEncoder encoder = new TestEncoderBuilder().build(); 259 encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback); 260 261 // Test. 262 assertThat(encoder.release()).isEqualTo(VideoCodecStatus.OK); 263 264 // Verify. 265 assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED); 266 } 267 268 @Test testReleaseMultipleTimes()269 public void testReleaseMultipleTimes() { 270 // Set-up. 271 HardwareVideoEncoder encoder = new TestEncoderBuilder().build(); 272 encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback); 273 274 // Test. 275 assertThat(encoder.release()).isEqualTo(VideoCodecStatus.OK); 276 assertThat(encoder.release()).isEqualTo(VideoCodecStatus.OK); 277 278 // Verify. 279 assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED); 280 } 281 282 @Test testFramerateWithFramerateBitrateAdjuster()283 public void testFramerateWithFramerateBitrateAdjuster() { 284 // Enable FramerateBitrateAdjuster and initialize encoder with frame rate 15fps. Vefity that our 285 // initial frame rate setting is ignored and media encoder is initialized with 30fps 286 // (FramerateBitrateAdjuster default). 287 HardwareVideoEncoder encoder = 288 new TestEncoderBuilder().setBitrateAdjuster(new FramerateBitrateAdjuster()).build(); 289 encoder.initEncode( 290 new Settings( 291 /* numberOfCores= */ 1, 292 /* width= */ 640, 293 /* height= */ 480, 294 /* startBitrate= */ 10000, 295 /* maxFramerate= */ 15, 296 /* numberOfSimulcastStreams= */ 1, 297 /* automaticResizeOn= */ true, 298 /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)), 299 mockEncoderCallback); 300 301 MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat(); 302 assertThat(mediaFormat.getFloat(MediaFormat.KEY_FRAME_RATE)).isEqualTo(30.0f); 303 } 304 305 @Test testBitrateWithFramerateBitrateAdjuster()306 public void testBitrateWithFramerateBitrateAdjuster() throws InterruptedException { 307 // Enable FramerateBitrateAdjuster and change frame rate while encoding video. Verify that 308 // bitrate setting passed to media encoder is adjusted to compensate for changes in frame rate. 309 TestEncoder encoder = 310 new TestEncoderBuilder().setBitrateAdjuster(new FramerateBitrateAdjuster()).build(); 311 encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback); 312 313 encoder.encode(createTestVideoFrame(/* timestampNs= */ 0), ENCODE_INFO_KEY_FRAME); 314 315 // Reduce frame rate by half. 316 BitrateAllocation bitrateAllocation = new BitrateAllocation( 317 /* bitratesBbs= */ new int[][] {new int[] {TEST_ENCODER_SETTINGS.startBitrate}}); 318 encoder.setRateAllocation(bitrateAllocation, TEST_ENCODER_SETTINGS.maxFramerate / 2); 319 320 // Generate output to trigger bitrate update in encoder wrapper. 321 fakeMediaCodecWrapper.addOutputData( 322 CodecTestHelper.generateRandomData(100), /* presentationTimestampUs= */ 0, /* flags= */ 0); 323 encoder.waitDeliverEncodedImage(); 324 325 // Frame rate has been reduced by half. Verify that bitrate doubled. 326 ArgumentCaptor<Bundle> bundleCaptor = ArgumentCaptor.forClass(Bundle.class); 327 verify(fakeMediaCodecWrapper, times(2)).setParameters(bundleCaptor.capture()); 328 Bundle params = bundleCaptor.getAllValues().get(1); 329 assertThat(params.containsKey(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE)).isTrue(); 330 assertThat(params.getInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE)) 331 .isEqualTo(TEST_ENCODER_SETTINGS.startBitrate * 2); 332 } 333 334 @Test testTimestampsWithFramerateBitrateAdjuster()335 public void testTimestampsWithFramerateBitrateAdjuster() throws InterruptedException { 336 // Enable FramerateBitrateAdjuster and change frame rate while encoding video. Verify that 337 // encoder ignores changes in frame rate and calculates frame timestamps based on fixed frame 338 // rate 30fps. 339 TestEncoder encoder = 340 new TestEncoderBuilder().setBitrateAdjuster(new FramerateBitrateAdjuster()).build(); 341 encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback); 342 343 encoder.encode(createTestVideoFrame(/* timestampNs= */ 0), ENCODE_INFO_KEY_FRAME); 344 345 // Reduce frametate by half. 346 BitrateAllocation bitrateAllocation = new BitrateAllocation( 347 /* bitratesBbs= */ new int[][] {new int[] {TEST_ENCODER_SETTINGS.startBitrate}}); 348 encoder.setRateAllocation(bitrateAllocation, TEST_ENCODER_SETTINGS.maxFramerate / 2); 349 350 // Encoder is allowed to buffer up to 2 frames. Generate output to avoid frame dropping. 351 fakeMediaCodecWrapper.addOutputData( 352 CodecTestHelper.generateRandomData(100), /* presentationTimestampUs= */ 0, /* flags= */ 0); 353 encoder.waitDeliverEncodedImage(); 354 355 encoder.encode(createTestVideoFrame(/* timestampNs= */ 1), ENCODE_INFO_DELTA_FRAME); 356 encoder.encode(createTestVideoFrame(/* timestampNs= */ 2), ENCODE_INFO_DELTA_FRAME); 357 358 ArgumentCaptor<Long> timestampCaptor = ArgumentCaptor.forClass(Long.class); 359 verify(fakeMediaCodecWrapper, times(3)) 360 .queueInputBuffer( 361 /* index= */ anyInt(), 362 /* offset= */ anyInt(), 363 /* size= */ anyInt(), timestampCaptor.capture(), 364 /* flags= */ anyInt()); 365 366 long frameDurationMs = SECONDS.toMicros(1) / 30; 367 assertThat(timestampCaptor.getAllValues()) 368 .containsExactly(0L, frameDurationMs, 2 * frameDurationMs); 369 } 370 } 371