1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.cts; 18 19 import android.media.MediaCodec; 20 import android.media.MediaExtractor; 21 import android.media.MediaFormat; 22 import android.util.DisplayMetrics; 23 import android.util.Log; 24 import android.view.Surface; 25 import android.view.SurfaceView; 26 import android.view.ViewGroup; 27 28 import androidx.test.filters.LargeTest; 29 import androidx.test.rule.ActivityTestRule; 30 31 import org.junit.Ignore; 32 import org.junit.Rule; 33 import org.junit.Test; 34 import org.junit.runner.RunWith; 35 import org.junit.runners.Parameterized; 36 37 import java.io.IOException; 38 import java.util.ArrayList; 39 import java.util.Arrays; 40 import java.util.Collection; 41 import java.util.HashSet; 42 import java.util.List; 43 import java.util.Set; 44 45 import static org.junit.Assert.assertTrue; 46 import static org.junit.Assert.fail; 47 48 @RunWith(Parameterized.class) 49 public class CodecDecoderSurfaceTest extends CodecDecoderTestBase { 50 private static final String LOG_TAG = CodecDecoderSurfaceTest.class.getSimpleName(); 51 52 private final String mReconfigFile; 53 private SurfaceView mSurfaceView; 54 CodecDecoderSurfaceTest(String mime, String testFile, String reconfigFile)55 public CodecDecoderSurfaceTest(String mime, String testFile, String reconfigFile) { 56 super(mime, testFile); 57 mReconfigFile = reconfigFile; 58 } 59 setScreenParams(int width, int height, boolean noStretch)60 private void setScreenParams(int width, int height, boolean noStretch) { 61 ViewGroup.LayoutParams lp = mSurfaceView.getLayoutParams(); 62 final DisplayMetrics dm = mActivityRule.getActivity().getResources().getDisplayMetrics(); 63 if (noStretch && width <= dm.widthPixels && height <= dm.heightPixels) { 64 lp.width = width; 65 lp.height = height; 66 } else { 67 int a = dm.widthPixels * height / width; 68 if (a <= dm.heightPixels) { 69 lp.width = dm.widthPixels; 70 lp.height = a; 71 } else { 72 lp.width = dm.heightPixels * width / height; 73 lp.height = dm.heightPixels; 74 } 75 } 76 assertTrue(lp.width <= dm.widthPixels); 77 assertTrue(lp.height <= dm.heightPixels); 78 mActivityRule.getActivity().runOnUiThread(() -> mSurfaceView.setLayoutParams(lp)); 79 } 80 dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)81 void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) { 82 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 83 mSawOutputEOS = true; 84 } 85 if (ENABLE_LOGS) { 86 Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " + 87 info.size + " timestamp: " + info.presentationTimeUs); 88 } 89 if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 90 mOutputBuff.saveOutPTS(info.presentationTimeUs); 91 mOutputCount++; 92 } 93 mCodec.releaseOutputBuffer(bufferIndex, mSurface != null); 94 } 95 decodeAndSavePts(String file, String decoder, long pts, int mode, int frameLimit)96 private void decodeAndSavePts(String file, String decoder, long pts, int mode, int frameLimit) 97 throws IOException, InterruptedException { 98 mOutputBuff = new OutputManager(); 99 mCodec = MediaCodec.createByCodecName(decoder); 100 MediaFormat format = setUpSource(file); 101 configureCodec(format, false, true, false); 102 mCodec.start(); 103 mExtractor.seekTo(pts, mode); 104 doWork(frameLimit); 105 queueEOS(); 106 waitForAllOutputs(); 107 mCodec.stop(); 108 mCodec.release(); 109 mExtractor.release(); 110 } 111 112 @Rule 113 public ActivityTestRule<CodecTestActivity> mActivityRule = 114 new ActivityTestRule<>(CodecTestActivity.class); 115 setUpSurface()116 public void setUpSurface() { 117 CodecTestActivity activity = mActivityRule.getActivity(); 118 mSurfaceView = activity.findViewById(R.id.surface); 119 mSurface = mSurfaceView.getHolder().getSurface(); 120 } 121 tearDownSurface()122 public void tearDownSurface() { 123 if (mSurface != null) { 124 mSurface.release(); 125 mSurface = null; 126 } 127 } 128 129 @Parameterized.Parameters(name = "{index}({0})") input()130 public static Collection<Object[]> input() { 131 Set<String> list = new HashSet<>(); 132 if (isHandheld() || isTv() || isAutomotive()) { 133 // sec 2.2.2, 2.3.2, 2.5.2 134 list.add(MediaFormat.MIMETYPE_VIDEO_AVC); 135 list.add(MediaFormat.MIMETYPE_VIDEO_MPEG4); 136 list.add(MediaFormat.MIMETYPE_VIDEO_H263); 137 list.add(MediaFormat.MIMETYPE_VIDEO_VP8); 138 list.add(MediaFormat.MIMETYPE_VIDEO_VP9); 139 } 140 if (isHandheld()) { 141 // sec 2.2.2 142 list.add(MediaFormat.MIMETYPE_VIDEO_HEVC); 143 } 144 if (isTv()) { 145 // sec 2.3.2 146 list.add(MediaFormat.MIMETYPE_VIDEO_HEVC); 147 list.add(MediaFormat.MIMETYPE_VIDEO_MPEG2); 148 } 149 ArrayList<String> cddRequiredMimeList = new ArrayList<>(list); 150 final List<Object[]> exhaustiveArgsList = Arrays.asList(new Object[][]{ 151 {MediaFormat.MIMETYPE_VIDEO_MPEG2, "bbb_340x280_768kbps_30fps_mpeg2.mp4", 152 "bbb_520x390_1mbps_30fps_mpeg2.mp4"}, 153 {MediaFormat.MIMETYPE_VIDEO_MPEG2, 154 "bbb_512x288_30fps_1mbps_mpeg2_interlaced_nob_2fields.mp4", 155 "bbb_520x390_1mbps_30fps_mpeg2.mp4"}, 156 {MediaFormat.MIMETYPE_VIDEO_MPEG2, 157 "bbb_512x288_30fps_1mbps_mpeg2_interlaced_nob_1field.ts", 158 "bbb_520x390_1mbps_30fps_mpeg2.mp4"}, 159 {MediaFormat.MIMETYPE_VIDEO_AVC, "bbb_340x280_768kbps_30fps_avc.mp4", 160 "bbb_520x390_1mbps_30fps_avc.mp4"}, 161 {MediaFormat.MIMETYPE_VIDEO_AVC, "bbb_360x640_768kbps_30fps_avc.mp4", 162 "bbb_520x390_1mbps_30fps_avc.mp4"}, 163 {MediaFormat.MIMETYPE_VIDEO_AVC, "bbb_160x1024_1500kbps_30fps_avc.mp4", 164 "bbb_520x390_1mbps_30fps_avc.mp4"}, 165 {MediaFormat.MIMETYPE_VIDEO_AVC, "bbb_1280x120_1500kbps_30fps_avc.mp4", 166 "bbb_340x280_768kbps_30fps_avc.mp4"}, 167 {MediaFormat.MIMETYPE_VIDEO_HEVC, "bbb_520x390_1mbps_30fps_hevc.mp4", 168 "bbb_340x280_768kbps_30fps_hevc.mp4"}, 169 {MediaFormat.MIMETYPE_VIDEO_MPEG4, "bbb_128x96_64kbps_12fps_mpeg4.mp4", 170 "bbb_176x144_192kbps_15fps_mpeg4.mp4"}, 171 {MediaFormat.MIMETYPE_VIDEO_H263, "bbb_176x144_128kbps_15fps_h263.3gp", 172 "bbb_176x144_192kbps_10fps_h263.3gp"}, 173 {MediaFormat.MIMETYPE_VIDEO_VP8, "bbb_340x280_768kbps_30fps_vp8.webm", 174 "bbb_520x390_1mbps_30fps_vp8.webm"}, 175 {MediaFormat.MIMETYPE_VIDEO_VP9, "bbb_340x280_768kbps_30fps_vp9.webm", 176 "bbb_520x390_1mbps_30fps_vp9.webm"}, 177 {MediaFormat.MIMETYPE_VIDEO_VP9, 178 "bbb_340x280_768kbps_30fps_split_non_display_frame_vp9.webm", 179 "bbb_520x390_1mbps_30fps_split_non_display_frame_vp9.webm"}, 180 {MediaFormat.MIMETYPE_VIDEO_AV1, "bbb_340x280_768kbps_30fps_av1.mp4", 181 "bbb_520x390_1mbps_30fps_av1.mp4"}, 182 }); 183 return prepareParamList(cddRequiredMimeList, exhaustiveArgsList, false); 184 } 185 186 /** 187 * Tests decoder for codec is in sync and async mode with surface. 188 * In these scenarios, Timestamp and it's ordering is verified. 189 */ 190 @LargeTest 191 @Test(timeout = PER_TEST_TIMEOUT_LARGE_TEST_MS) testSimpleDecodeToSurface()192 public void testSimpleDecodeToSurface() throws IOException, InterruptedException { 193 ArrayList<String> listOfDecoders = selectCodecs(mMime, null, null, false); 194 if (listOfDecoders.isEmpty()) { 195 fail("no suitable codecs found for mime: " + mMime); 196 } 197 boolean[] boolStates = {true, false}; 198 OutputManager ref; 199 OutputManager test = new OutputManager(); 200 final long pts = 0; 201 final int mode = MediaExtractor.SEEK_TO_CLOSEST_SYNC; 202 for (String decoder : listOfDecoders) { 203 decodeAndSavePts(mTestFile, decoder, pts, mode, Integer.MAX_VALUE); 204 ref = mOutputBuff; 205 assertTrue("input pts list and output pts list are not identical", 206 ref.isOutPtsListIdenticalToInpPtsList(false)); 207 MediaFormat format = setUpSource(mTestFile); 208 mCodec = MediaCodec.createByCodecName(decoder); 209 setUpSurface(); 210 setScreenParams(getWidth(format), getHeight(format), true); 211 for (boolean isAsync : boolStates) { 212 String log = String.format("codec: %s, file: %s, mode: %s:: ", decoder, mTestFile, 213 (isAsync ? "async" : "sync")); 214 mOutputBuff = test; 215 mOutputBuff.reset(); 216 mExtractor.seekTo(pts, mode); 217 configureCodec(format, isAsync, true, false); 218 mCodec.start(); 219 doWork(Integer.MAX_VALUE); 220 queueEOS(); 221 waitForAllOutputs(); 222 /* TODO(b/147348711) */ 223 if (false) mCodec.stop(); 224 else mCodec.reset(); 225 assertTrue(log + " unexpected error", !mAsyncHandle.hasSeenError()); 226 assertTrue(log + "no input sent", 0 != mInputCount); 227 assertTrue(log + "output received", 0 != mOutputCount); 228 assertTrue(log + "decoder output is flaky", ref.equals(test)); 229 } 230 mCodec.release(); 231 mExtractor.release(); 232 mSurface = null; 233 } 234 tearDownSurface(); 235 } 236 237 /** 238 * Tests flush when codec is in sync and async mode with surface. In these scenarios, 239 * Timestamp and the ordering is verified. 240 */ 241 @Ignore("TODO(b/147576107)") 242 @LargeTest 243 @Test(timeout = PER_TEST_TIMEOUT_LARGE_TEST_MS) testFlush()244 public void testFlush() throws IOException, InterruptedException { 245 MediaFormat format = setUpSource(mTestFile); 246 mExtractor.release(); 247 ArrayList<String> listOfDecoders = selectCodecs(mMime, null, null, false); 248 if (listOfDecoders.isEmpty()) { 249 fail("no suitable codecs found for mime: " + mMime); 250 } 251 mCsdBuffers.clear(); 252 for (int i = 0; ; i++) { 253 String csdKey = "csd-" + i; 254 if (format.containsKey(csdKey)) { 255 mCsdBuffers.add(format.getByteBuffer(csdKey)); 256 } else break; 257 } 258 final long pts = 500000; 259 final int mode = MediaExtractor.SEEK_TO_CLOSEST_SYNC; 260 boolean[] boolStates = {true, false}; 261 OutputManager test = new OutputManager(); 262 for (String decoder : listOfDecoders) { 263 decodeAndSavePts(mTestFile, decoder, pts, mode, Integer.MAX_VALUE); 264 OutputManager ref = mOutputBuff; 265 assertTrue("input pts list and output pts list are not identical", 266 ref.isOutPtsListIdenticalToInpPtsList(false)); 267 mOutputBuff = test; 268 setUpSource(mTestFile); 269 mCodec = MediaCodec.createByCodecName(decoder); 270 setUpSurface(); 271 setScreenParams(getWidth(format), getHeight(format), false); 272 for (boolean isAsync : boolStates) { 273 String log = String.format("decoder: %s, input file: %s, mode: %s:: ", decoder, 274 mTestFile, (isAsync ? "async" : "sync")); 275 mExtractor.seekTo(0, mode); 276 configureCodec(format, isAsync, true, false); 277 mCodec.start(); 278 279 /* test flush in running state before queuing input */ 280 flushCodec(); 281 if (mIsCodecInAsyncMode) mCodec.start(); 282 queueCodecConfig(); /* flushed codec too soon after start, resubmit csd */ 283 284 doWork(1); 285 flushCodec(); 286 if (mIsCodecInAsyncMode) mCodec.start(); 287 queueCodecConfig(); /* flushed codec too soon after start, resubmit csd */ 288 289 mExtractor.seekTo(0, mode); 290 test.reset(); 291 doWork(23); 292 assertTrue(log + " pts is not strictly increasing", 293 test.isPtsStrictlyIncreasing(mPrevOutputPts)); 294 295 /* test flush in running state */ 296 flushCodec(); 297 if (mIsCodecInAsyncMode) mCodec.start(); 298 test.reset(); 299 mExtractor.seekTo(pts, mode); 300 doWork(Integer.MAX_VALUE); 301 queueEOS(); 302 waitForAllOutputs(); 303 assertTrue(log + " unexpected error", !mAsyncHandle.hasSeenError()); 304 assertTrue(log + "no input sent", 0 != mInputCount); 305 assertTrue(log + "output received", 0 != mOutputCount); 306 assertTrue(log + "decoder output is flaky", ref.equals(test)); 307 308 /* test flush in eos state */ 309 flushCodec(); 310 if (mIsCodecInAsyncMode) mCodec.start(); 311 test.reset(); 312 mExtractor.seekTo(pts, mode); 313 doWork(Integer.MAX_VALUE); 314 queueEOS(); 315 waitForAllOutputs(); 316 /* TODO(b/147348711) */ 317 if (false) mCodec.stop(); 318 else mCodec.reset(); 319 assertTrue(log + " unexpected error", !mAsyncHandle.hasSeenError()); 320 assertTrue(log + "no input sent", 0 != mInputCount); 321 assertTrue(log + "output received", 0 != mOutputCount); 322 assertTrue(log + "decoder output is flaky", ref.equals(test)); 323 } 324 mCodec.release(); 325 mExtractor.release(); 326 mSurface = null; 327 } 328 tearDownSurface(); 329 } 330 331 /** 332 * Tests reconfigure when codec is in sync and async mode with surface. In these scenarios, 333 * Timestamp and the ordering is verified. 334 */ 335 @Ignore("TODO(b/148523403)") 336 @LargeTest 337 @Test(timeout = PER_TEST_TIMEOUT_LARGE_TEST_MS) testReconfigure()338 public void testReconfigure() throws IOException, InterruptedException { 339 MediaFormat format = setUpSource(mTestFile); 340 mExtractor.release(); 341 MediaFormat newFormat = setUpSource(mReconfigFile); 342 mExtractor.release(); 343 ArrayList<String> listOfDecoders = selectCodecs(mMime, null, null, false); 344 if (listOfDecoders.isEmpty()) { 345 fail("no suitable codecs found for mime: " + mMime); 346 } 347 final long pts = 500000; 348 final int mode = MediaExtractor.SEEK_TO_CLOSEST_SYNC; 349 boolean[] boolStates = {true, false}; 350 OutputManager test = new OutputManager(); 351 for (String decoder : listOfDecoders) { 352 decodeAndSavePts(mTestFile, decoder, pts, mode, Integer.MAX_VALUE); 353 OutputManager ref = mOutputBuff; 354 decodeAndSavePts(mReconfigFile, decoder, pts, mode, Integer.MAX_VALUE); 355 OutputManager configRef = mOutputBuff; 356 assertTrue("input pts list and reference pts list are not identical", 357 ref.isOutPtsListIdenticalToInpPtsList(false)); 358 assertTrue("input pts list and reconfig ref output pts list are not identical", 359 configRef.isOutPtsListIdenticalToInpPtsList(false)); 360 mOutputBuff = test; 361 mCodec = MediaCodec.createByCodecName(decoder); 362 setUpSurface(); 363 setScreenParams(getWidth(format), getHeight(format), false); 364 for (boolean isAsync : boolStates) { 365 setUpSource(mTestFile); 366 String log = String.format("decoder: %s, input file: %s, mode: %s:: ", decoder, 367 mTestFile, (isAsync ? "async" : "sync")); 368 mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC); 369 configureCodec(format, isAsync, true, false); 370 371 /* test reconfigure in stopped state */ 372 reConfigureCodec(format, !isAsync, false, false); 373 mCodec.start(); 374 375 /* test reconfigure in running state before queuing input */ 376 reConfigureCodec(format, !isAsync, false, false); 377 mCodec.start(); 378 doWork(23); 379 380 /* test reconfigure codec in running state */ 381 reConfigureCodec(format, isAsync, true, false); 382 mCodec.start(); 383 test.reset(); 384 mExtractor.seekTo(pts, mode); 385 doWork(Integer.MAX_VALUE); 386 queueEOS(); 387 waitForAllOutputs(); 388 /* TODO(b/147348711) */ 389 if (false) mCodec.stop(); 390 else mCodec.reset(); 391 assertTrue(log + " unexpected error", !mAsyncHandle.hasSeenError()); 392 assertTrue(log + "no input sent", 0 != mInputCount); 393 assertTrue(log + "output received", 0 != mOutputCount); 394 assertTrue(log + "decoder output is flaky", ref.equals(test)); 395 396 /* test reconfigure codec at eos state */ 397 reConfigureCodec(format, !isAsync, false, false); 398 mCodec.start(); 399 test.reset(); 400 mExtractor.seekTo(pts, mode); 401 doWork(Integer.MAX_VALUE); 402 queueEOS(); 403 waitForAllOutputs(); 404 /* TODO(b/147348711) */ 405 if (false) mCodec.stop(); 406 else mCodec.reset(); 407 assertTrue(log + " unexpected error", !mAsyncHandle.hasSeenError()); 408 assertTrue(log + "no input sent", 0 != mInputCount); 409 assertTrue(log + "output received", 0 != mOutputCount); 410 assertTrue(log + "decoder output is flaky", ref.equals(test)); 411 mExtractor.release(); 412 413 /* test reconfigure codec for new file */ 414 setUpSource(mReconfigFile); 415 log = String.format("decoder: %s, input file: %s, mode: %s:: ", decoder, 416 mReconfigFile, (isAsync ? "async" : "sync")); 417 setScreenParams(getWidth(newFormat), getHeight(newFormat), true); 418 reConfigureCodec(newFormat, isAsync, false, false); 419 mCodec.start(); 420 test.reset(); 421 mExtractor.seekTo(pts, mode); 422 doWork(Integer.MAX_VALUE); 423 queueEOS(); 424 waitForAllOutputs(); 425 /* TODO(b/147348711) */ 426 if (false) mCodec.stop(); 427 else mCodec.reset(); 428 assertTrue(log + " unexpected error", !mAsyncHandle.hasSeenError()); 429 assertTrue(log + "no input sent", 0 != mInputCount); 430 assertTrue(log + "output received", 0 != mOutputCount); 431 assertTrue(log + "decoder output is flaky", configRef.equals(test)); 432 mExtractor.release(); 433 } 434 mCodec.release(); 435 mSurface = null; 436 } 437 tearDownSurface(); 438 } 439 nativeTestSimpleDecode(String decoder, Surface surface, String mime, String testFile, String refFile, float rmsError)440 private native boolean nativeTestSimpleDecode(String decoder, Surface surface, String mime, 441 String testFile, String refFile, float rmsError); 442 443 @LargeTest 444 @Test(timeout = PER_TEST_TIMEOUT_LARGE_TEST_MS) testSimpleDecodeToSurfaceNative()445 public void testSimpleDecodeToSurfaceNative() throws IOException { 446 ArrayList<String> listOfDecoders = selectCodecs(mMime, null, null, false); 447 if (listOfDecoders.isEmpty()) { 448 fail("no suitable codecs found for mime: " + mMime); 449 } 450 MediaFormat format = setUpSource(mTestFile); 451 mExtractor.release(); 452 setUpSurface(); 453 setScreenParams(getWidth(format), getHeight(format), false); 454 for (String decoder : listOfDecoders) { 455 assertTrue(nativeTestSimpleDecode(decoder, mSurface, mMime, mInpPrefix + mTestFile, 456 mInpPrefix + mReconfigFile, -1.0f)); 457 } 458 tearDownSurface(); 459 } 460 nativeTestFlush(String decoder, Surface surface, String mime, String testFile)461 private native boolean nativeTestFlush(String decoder, Surface surface, String mime, 462 String testFile); 463 464 @LargeTest 465 @Test(timeout = PER_TEST_TIMEOUT_LARGE_TEST_MS) testFlushNative()466 public void testFlushNative() throws IOException { 467 ArrayList<String> listOfDecoders = selectCodecs(mMime, null, null, false); 468 if (listOfDecoders.isEmpty()) { 469 fail("no suitable codecs found for mime: " + mMime); 470 } 471 MediaFormat format = setUpSource(mTestFile); 472 mExtractor.release(); 473 setUpSurface(); 474 setScreenParams(getWidth(format), getHeight(format), true); 475 for (String decoder : listOfDecoders) { 476 assertTrue(nativeTestFlush(decoder, mSurface, mMime, mInpPrefix + mTestFile)); 477 } 478 tearDownSurface(); 479 } 480 } 481