1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.cts; 18 19 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 20 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; 21 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010; 22 import static android.mediav2.common.cts.CodecEncoderTestBase.ACCEPTABLE_WIRELESS_TX_QUALITY; 23 import static android.mediav2.common.cts.CodecEncoderTestBase.colorFormatToString; 24 import static android.mediav2.common.cts.CodecEncoderTestBase.getMuxerFormatForMediaType; 25 import static android.mediav2.common.cts.CodecEncoderTestBase.getTempFilePath; 26 import static android.mediav2.common.cts.CodecTestBase.PROFILE_HLG_MAP; 27 import static android.mediav2.common.cts.CodecTestBase.VNDK_IS_AT_LEAST_T; 28 import static android.mediav2.common.cts.CodecTestBase.VNDK_IS_BEFORE_U; 29 import static android.mediav2.common.cts.CodecTestBase.hasSupportForColorFormat; 30 import static android.mediav2.common.cts.CodecTestBase.isDefaultCodec; 31 import static android.mediav2.common.cts.CodecTestBase.isHardwareAcceleratedCodec; 32 import static android.mediav2.common.cts.CodecTestBase.isSoftwareCodec; 33 import static android.mediav2.common.cts.CodecTestBase.isVendorCodec; 34 35 import static org.junit.Assert.assertEquals; 36 import static org.junit.Assert.assertFalse; 37 import static org.junit.Assert.assertNotEquals; 38 import static org.junit.Assert.assertTrue; 39 import static org.junit.Assert.fail; 40 import static org.junit.Assume.assumeFalse; 41 import static org.junit.Assume.assumeTrue; 42 43 import android.media.MediaCodec; 44 import android.media.MediaExtractor; 45 import android.media.MediaFormat; 46 import android.media.MediaMuxer; 47 import android.mediav2.common.cts.CodecAsyncHandler; 48 import android.mediav2.common.cts.CodecEncoderTestBase; 49 import android.mediav2.common.cts.CodecTestBase; 50 import android.mediav2.common.cts.EncoderConfigParams; 51 import android.mediav2.common.cts.OutputManager; 52 import android.util.Log; 53 import android.util.Pair; 54 import android.view.Surface; 55 56 import androidx.test.filters.LargeTest; 57 import androidx.test.platform.app.InstrumentationRegistry; 58 59 import com.android.compatibility.common.util.ApiTest; 60 import com.android.compatibility.common.util.CddTest; 61 import com.android.compatibility.common.util.Preconditions; 62 63 import org.junit.After; 64 import org.junit.Assume; 65 import org.junit.Before; 66 import org.junit.Rule; 67 import org.junit.Test; 68 import org.junit.rules.TestName; 69 import org.junit.runner.RunWith; 70 import org.junit.runners.Parameterized; 71 72 import java.io.File; 73 import java.io.IOException; 74 import java.nio.ByteBuffer; 75 import java.util.ArrayList; 76 import java.util.Arrays; 77 import java.util.Collection; 78 import java.util.List; 79 import java.util.Objects; 80 import java.util.stream.IntStream; 81 82 /** 83 * Test mediacodec api, video encoders and their interactions in surface mode. 84 * <p> 85 * The test decodes an input clip to surface. This decoded output is fed as input to encoder. 86 * Assuming no frame drops, the test expects, 87 * <ul> 88 * <li>The number of encoded frames to be identical to number of frames present in input clip 89 * .</li> 90 * <li>As encoders are expected to give consistent output for a given input and configuration 91 * parameters, the test checks for consistency across runs. For now, this attribute is not 92 * strictly enforced in this test.</li> 93 * <li>The encoder output timestamps list should be identical to decoder input timestamp list 94 * .</li> 95 * </ul> 96 * <p> 97 * The output of encoder is further verified by computing PSNR to check for obvious visual 98 * artifacts. 99 * <p> 100 * The test runs mediacodec in synchronous and asynchronous mode. 101 */ 102 @RunWith(Parameterized.class) 103 public class CodecEncoderSurfaceTest { 104 private static final String LOG_TAG = CodecEncoderSurfaceTest.class.getSimpleName(); 105 private static final String MEDIA_DIR = WorkDir.getMediaDirString(); 106 private static final boolean ENABLE_LOGS = false; 107 108 private final String mEncoderName; 109 private final String mEncMediaType; 110 private final String mDecoderName; 111 private final String mTestFileMediaType; 112 private final String mTestFile; 113 private final EncoderConfigParams mEncCfgParams; 114 private final int mColorFormat; 115 private final boolean mIsOutputToneMapped; 116 private final boolean mUsePersistentSurface; 117 private final String mTestArgs; 118 119 private MediaExtractor mExtractor; 120 private MediaCodec mEncoder; 121 private MediaFormat mEncoderFormat; 122 private final CodecAsyncHandler mAsyncHandleEncoder = new CodecAsyncHandler(); 123 private MediaCodec mDecoder; 124 private MediaFormat mDecoderFormat; 125 private final CodecAsyncHandler mAsyncHandleDecoder = new CodecAsyncHandler(); 126 private boolean mIsCodecInAsyncMode; 127 private boolean mSignalEOSWithLastFrame; 128 private boolean mSawDecInputEOS; 129 private boolean mSawDecOutputEOS; 130 private boolean mSawEncOutputEOS; 131 private int mDecInputCount; 132 private int mDecOutputCount; 133 private int mEncOutputCount; 134 private int mLatency; 135 private boolean mReviseLatency; 136 137 private final StringBuilder mTestConfig = new StringBuilder(); 138 private final StringBuilder mTestEnv = new StringBuilder(); 139 140 private boolean mSaveToMem; 141 private OutputManager mOutputBuff; 142 143 private Surface mSurface; 144 145 private MediaMuxer mMuxer; 146 private int mTrackID = -1; 147 148 private final ArrayList<String> mTmpFiles = new ArrayList<>(); 149 150 static { 151 System.loadLibrary("ctsmediav2codecencsurface_jni"); 152 153 android.os.Bundle args = InstrumentationRegistry.getArguments(); 154 CodecTestBase.mediaTypeSelKeys = args.getString(CodecTestBase.MEDIA_TYPE_SEL_KEY); 155 } 156 CodecEncoderSurfaceTest(String encoder, String mediaType, String decoder, String testFileMediaType, String testFile, EncoderConfigParams encCfgParams, int colorFormat, boolean isOutputToneMapped, boolean usePersistentSurface, @SuppressWarnings("unused") String testLabel, String allTestParams)157 public CodecEncoderSurfaceTest(String encoder, String mediaType, String decoder, 158 String testFileMediaType, String testFile, EncoderConfigParams encCfgParams, 159 int colorFormat, boolean isOutputToneMapped, boolean usePersistentSurface, 160 @SuppressWarnings("unused") String testLabel, String allTestParams) { 161 mEncoderName = encoder; 162 mEncMediaType = mediaType; 163 mDecoderName = decoder; 164 mTestFileMediaType = testFileMediaType; 165 mTestFile = MEDIA_DIR + testFile; 166 mEncCfgParams = encCfgParams; 167 mColorFormat = colorFormat; 168 mIsOutputToneMapped = isOutputToneMapped; 169 mUsePersistentSurface = usePersistentSurface; 170 mTestArgs = allTestParams; 171 mLatency = mEncCfgParams.mMaxBFrames; 172 mReviseLatency = false; 173 } 174 175 @Rule 176 public TestName mTestName = new TestName(); 177 178 @Before setUp()179 public void setUp() throws IOException, CloneNotSupportedException { 180 mTestConfig.setLength(0); 181 mTestConfig.append("\n################## Test Details ####################\n"); 182 mTestConfig.append("Test Name :- ").append(mTestName.getMethodName()).append("\n"); 183 mTestConfig.append("Test Parameters :- ").append(mTestArgs).append("\n"); 184 if (mEncoderName.startsWith(CodecTestBase.INVALID_CODEC) || mDecoderName.startsWith( 185 CodecTestBase.INVALID_CODEC)) { 186 fail("no valid component available for current test. \n" + mTestConfig); 187 } 188 mDecoderFormat = setUpSource(mTestFile); 189 ArrayList<MediaFormat> decoderFormatList = new ArrayList<>(); 190 decoderFormatList.add(mDecoderFormat); 191 Assume.assumeTrue("Decoder: " + mDecoderName + " doesn't support format: " + mDecoderFormat, 192 CodecTestBase.areFormatsSupported(mDecoderName, mTestFileMediaType, 193 decoderFormatList)); 194 if (CodecTestBase.doesAnyFormatHaveHDRProfile(mTestFileMediaType, decoderFormatList) 195 || mTestFile.contains("10bit")) { 196 // Check if encoder is capable of supporting HDR profiles. 197 // Previous check doesn't verify this as profile isn't set in the format 198 Assume.assumeTrue(mEncoderName + " doesn't support HDR encoding", 199 CodecTestBase.doesCodecSupportHDRProfile(mEncoderName, mEncMediaType)); 200 } 201 202 if (mColorFormat == COLOR_FormatSurface) { 203 // TODO(b/253492870) Remove the following assumption check once this is supported 204 Assume.assumeFalse(mDecoderName + "is hardware accelerated and " + mEncoderName 205 + "is software only.", 206 isHardwareAcceleratedCodec(mDecoderName) && isSoftwareCodec(mEncoderName)); 207 } else { 208 // findDecoderForFormat() ignores color-format and decoder returned may not be 209 // supporting the color format set in mDecoderFormat. Following check will 210 // skip the test if decoder doesn't support the color format that is set. 211 boolean decoderSupportsColorFormat = 212 hasSupportForColorFormat(mDecoderName, mTestFileMediaType, mColorFormat); 213 if (mColorFormat == COLOR_FormatYUVP010) { 214 assumeTrue(mDecoderName + " doesn't support P010 output.", 215 decoderSupportsColorFormat); 216 } else { 217 assertTrue(mDecoderName + " doesn't support 420p 888 flexible output.", 218 decoderSupportsColorFormat); 219 } 220 } 221 EncoderConfigParams.Builder foreman = mEncCfgParams.getBuilder() 222 .setWidth(mDecoderFormat.getInteger(MediaFormat.KEY_WIDTH)) 223 .setHeight(mDecoderFormat.getInteger(MediaFormat.KEY_HEIGHT)); 224 mEncoderFormat = foreman.build().getFormat(); 225 } 226 getVideoEncoderCfgParams(String mediaType, int bitRate, int frameRate, int bitDepth, int maxBFrames)227 private static EncoderConfigParams getVideoEncoderCfgParams(String mediaType, int bitRate, 228 int frameRate, int bitDepth, int maxBFrames) { 229 EncoderConfigParams.Builder foreman = new EncoderConfigParams.Builder(mediaType) 230 .setBitRate(bitRate) 231 .setFrameRate(frameRate) 232 .setColorFormat(COLOR_FormatSurface) 233 .setInputBitDepth(bitDepth) 234 .setMaxBFrames(maxBFrames); 235 if (bitDepth == 10) { 236 foreman.setProfile(Objects.requireNonNull(PROFILE_HLG_MAP.get(mediaType))[0]); 237 } 238 return foreman.build(); 239 } 240 241 @After tearDown()242 public void tearDown() { 243 if (mDecoder != null) { 244 mDecoder.release(); 245 mDecoder = null; 246 } 247 if (mSurface != null) { 248 mSurface.release(); 249 mSurface = null; 250 } 251 if (mEncoder != null) { 252 mEncoder.release(); 253 mEncoder = null; 254 } 255 if (mExtractor != null) { 256 mExtractor.release(); 257 mExtractor = null; 258 } 259 if (mMuxer != null) { 260 mMuxer.release(); 261 mMuxer = null; 262 } 263 for (String tmpFile : mTmpFiles) { 264 File tmp = new File(tmpFile); 265 if (tmp.exists()) assertTrue("unable to delete file " + tmpFile, tmp.delete()); 266 } 267 mTmpFiles.clear(); 268 } 269 270 @Parameterized.Parameters(name = "{index}_{0}_{1}_{2}_{3}_{9}") input()271 public static Collection<Object[]> input() throws IOException { 272 final boolean isEncoder = true; 273 final boolean needAudio = false; 274 final boolean needVideo = true; 275 final List<Object[]> exhaustiveArgsList = new ArrayList<>(); 276 final List<Object[]> args = new ArrayList<>(Arrays.asList(new Object[][]{ 277 // mediaType, testFileMediaType, testFile, bitRate, frameRate, toneMap 278 {MediaFormat.MIMETYPE_VIDEO_H263, MediaFormat.MIMETYPE_VIDEO_H263, 279 "bbb_176x144_128kbps_15fps_h263.3gp", 128000, 15, false}, 280 {MediaFormat.MIMETYPE_VIDEO_MPEG4, MediaFormat.MIMETYPE_VIDEO_MPEG4, 281 "bbb_128x96_64kbps_12fps_mpeg4.mp4", 64000, 12, false}, 282 {MediaFormat.MIMETYPE_VIDEO_AVC, MediaFormat.MIMETYPE_VIDEO_AVC, 283 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 284 {MediaFormat.MIMETYPE_VIDEO_HEVC, MediaFormat.MIMETYPE_VIDEO_AVC, 285 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 286 {MediaFormat.MIMETYPE_VIDEO_VP8, MediaFormat.MIMETYPE_VIDEO_AVC, 287 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 288 {MediaFormat.MIMETYPE_VIDEO_VP9, MediaFormat.MIMETYPE_VIDEO_AVC, 289 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 290 {MediaFormat.MIMETYPE_VIDEO_AV1, MediaFormat.MIMETYPE_VIDEO_AVC, 291 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 292 })); 293 294 final List<Object[]> argsHighBitDepth = new ArrayList<>(Arrays.asList(new Object[][]{ 295 {MediaFormat.MIMETYPE_VIDEO_AVC, MediaFormat.MIMETYPE_VIDEO_AVC, 296 "cosmat_520x390_24fps_crf22_avc_10bit.mkv", 512000, 30, false}, 297 {MediaFormat.MIMETYPE_VIDEO_AVC, MediaFormat.MIMETYPE_VIDEO_AVC, 298 "cosmat_520x390_24fps_crf22_avc_10bit.mkv", 512000, 30, true}, 299 {MediaFormat.MIMETYPE_VIDEO_HEVC, MediaFormat.MIMETYPE_VIDEO_HEVC, 300 "cosmat_520x390_24fps_crf22_hevc_10bit.mkv", 512000, 30, false}, 301 {MediaFormat.MIMETYPE_VIDEO_HEVC, MediaFormat.MIMETYPE_VIDEO_HEVC, 302 "cosmat_520x390_24fps_crf22_hevc_10bit.mkv", 512000, 30, true}, 303 {MediaFormat.MIMETYPE_VIDEO_VP9, MediaFormat.MIMETYPE_VIDEO_VP9, 304 "cosmat_520x390_24fps_crf22_vp9_10bit.mkv", 512000, 30, false}, 305 {MediaFormat.MIMETYPE_VIDEO_VP9, MediaFormat.MIMETYPE_VIDEO_VP9, 306 "cosmat_520x390_24fps_crf22_vp9_10bit.mkv", 512000, 30, true}, 307 {MediaFormat.MIMETYPE_VIDEO_AV1, MediaFormat.MIMETYPE_VIDEO_AV1, 308 "cosmat_520x390_24fps_768kbps_av1_10bit.mkv", 512000, 30, false}, 309 {MediaFormat.MIMETYPE_VIDEO_AV1, MediaFormat.MIMETYPE_VIDEO_AV1, 310 "cosmat_520x390_24fps_768kbps_av1_10bit.mkv", 512000, 30, true}, 311 })); 312 313 int[] colorFormats = {COLOR_FormatSurface, COLOR_FormatYUV420Flexible}; 314 int[] maxBFrames = {0, 2}; 315 boolean[] boolStates = {true, false}; 316 for (Object[] arg : args) { 317 final String mediaType = (String) arg[0]; 318 final int br = (int) arg[3]; 319 final int fps = (int) arg[4]; 320 for (int colorFormat : colorFormats) { 321 for (boolean usePersistentSurface : boolStates) { 322 for (int maxBFrame : maxBFrames) { 323 if (!mediaType.equals(MediaFormat.MIMETYPE_VIDEO_AVC) 324 && !mediaType.equals(MediaFormat.MIMETYPE_VIDEO_HEVC) 325 && maxBFrame != 0) { 326 continue; 327 } 328 Object[] testArgs = new Object[8]; 329 testArgs[0] = arg[0]; // encoder mediaType 330 testArgs[1] = arg[1]; // test file mediaType 331 testArgs[2] = arg[2]; // test file 332 testArgs[3] = getVideoEncoderCfgParams(mediaType, br, fps, 8, maxBFrame); 333 testArgs[4] = colorFormat; // color format 334 testArgs[5] = arg[5]; // tone map 335 testArgs[6] = usePersistentSurface; 336 testArgs[7] = String.format("%dkbps_%dfps_%s_%s", br / 1000, fps, 337 colorFormatToString(colorFormat, 8), 338 usePersistentSurface ? "persistentsurface" : "surface"); 339 exhaustiveArgsList.add(testArgs); 340 } 341 } 342 } 343 } 344 // P010 support was added in Android T, hence limit the following tests to Android T and 345 // above 346 if (CodecTestBase.IS_AT_LEAST_T) { 347 int[] colorFormatsHbd = {COLOR_FormatSurface, COLOR_FormatYUVP010}; 348 for (Object[] arg : argsHighBitDepth) { 349 final String mediaType = (String) arg[0]; 350 final int br = (int) arg[3]; 351 final int fps = (int) arg[4]; 352 final boolean toneMap = (boolean) arg[5]; 353 for (int colorFormat : colorFormatsHbd) { 354 for (boolean usePersistentSurface : boolStates) { 355 for (int maxBFrame : maxBFrames) { 356 if (!mediaType.equals(MediaFormat.MIMETYPE_VIDEO_AVC) 357 && !mediaType.equals(MediaFormat.MIMETYPE_VIDEO_HEVC) 358 && maxBFrame != 0) { 359 continue; 360 } 361 Object[] testArgs = new Object[8]; 362 testArgs[0] = arg[0]; // encoder mediaType 363 testArgs[1] = arg[1]; // test file mediaType 364 testArgs[2] = arg[2]; // test file 365 testArgs[3] = 366 getVideoEncoderCfgParams(mediaType, br, fps, toneMap ? 8 : 10, 367 maxBFrame); 368 if (toneMap && (colorFormat == COLOR_FormatYUVP010)) { 369 colorFormat = COLOR_FormatYUV420Flexible; 370 } 371 testArgs[4] = colorFormat; // color format 372 testArgs[5] = arg[5]; // tone map 373 testArgs[6] = usePersistentSurface; 374 testArgs[7] = String.format("%dkbps_%dfps_%s_%s_%s", br / 1000, fps, 375 colorFormatToString(colorFormat, toneMap ? 8 : 10), 376 toneMap ? "tonemapyes" : "tonemapno", 377 usePersistentSurface ? "persistentsurface" : "surface"); 378 exhaustiveArgsList.add(testArgs); 379 } 380 } 381 } 382 } 383 } 384 final List<Object[]> argsList = new ArrayList<>(); 385 for (Object[] arg : exhaustiveArgsList) { 386 ArrayList<String> decoderList = 387 CodecTestBase.selectCodecs((String) arg[1], null, null, false); 388 if (decoderList.size() == 0) { 389 decoderList.add(CodecTestBase.INVALID_CODEC + arg[1]); 390 } 391 for (String decoderName : decoderList) { 392 int argLength = exhaustiveArgsList.get(0).length; 393 Object[] testArg = new Object[argLength + 1]; 394 testArg[0] = arg[0]; // encoder mediaType 395 testArg[1] = decoderName; // decoder name 396 System.arraycopy(arg, 1, testArg, 2, argLength - 1); 397 argsList.add(testArg); 398 } 399 } 400 401 final List<Object[]> expandedArgsList = 402 CodecTestBase.prepareParamList(argsList, isEncoder, needAudio, needVideo, true); 403 404 // Prior to Android U, this test was using the first decoder for a given mediaType. 405 // In Android U, this was updated to test the encoders with all decoders for the 406 // given mediaType. There are some vendor encoders in older versions of Android 407 // which do not work as expected with the surface from s/w decoder. 408 // If the device is has vendor partition older than Android U, limit the tests 409 // to first decoder like it was being done prior to Androd U 410 final List<Object[]> finalArgsList = new ArrayList<>(); 411 for (Object[] arg : expandedArgsList) { 412 String encoderName = (String) arg[0]; 413 String decoderName = (String) arg[2]; 414 String decoderMediaType = (String) arg[3]; 415 if (VNDK_IS_BEFORE_U && isVendorCodec(encoderName)) { 416 if (!isDefaultCodec(decoderName, decoderMediaType, /* isEncoder */false)) { 417 continue; 418 } 419 } 420 finalArgsList.add(arg); 421 } 422 return finalArgsList; 423 } 424 hasSeenError()425 private boolean hasSeenError() { 426 return mAsyncHandleDecoder.hasSeenError() || mAsyncHandleEncoder.hasSeenError(); 427 } 428 setUpSource(String srcFile)429 private MediaFormat setUpSource(String srcFile) throws IOException { 430 Preconditions.assertTestFileExists(srcFile); 431 mExtractor = new MediaExtractor(); 432 mExtractor.setDataSource(srcFile); 433 for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) { 434 MediaFormat format = mExtractor.getTrackFormat(trackID); 435 String mediaType = format.getString(MediaFormat.KEY_MIME); 436 if (mediaType.equals(mTestFileMediaType)) { 437 mExtractor.selectTrack(trackID); 438 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, mColorFormat); 439 if (mIsOutputToneMapped) { 440 format.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, 441 MediaFormat.COLOR_TRANSFER_SDR_VIDEO); 442 } 443 return format; 444 } 445 } 446 mExtractor.release(); 447 fail("No video track found in file: " + srcFile + ". \n" + mTestConfig + mTestEnv); 448 return null; 449 } 450 resetContext(boolean isAsync, boolean signalEOSWithLastFrame)451 private void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) { 452 mAsyncHandleDecoder.resetContext(); 453 mAsyncHandleEncoder.resetContext(); 454 mIsCodecInAsyncMode = isAsync; 455 mSignalEOSWithLastFrame = signalEOSWithLastFrame; 456 mSawDecInputEOS = false; 457 mSawDecOutputEOS = false; 458 mSawEncOutputEOS = false; 459 mDecInputCount = 0; 460 mDecOutputCount = 0; 461 mEncOutputCount = 0; 462 } 463 configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)464 private void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, 465 boolean signalEOSWithLastFrame) { 466 resetContext(isAsync, signalEOSWithLastFrame); 467 mAsyncHandleEncoder.setCallBack(mEncoder, isAsync); 468 mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null); 469 if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) { 470 mReviseLatency = true; 471 mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY); 472 } 473 if (mUsePersistentSurface) { 474 mSurface = MediaCodec.createPersistentInputSurface(); 475 mEncoder.setInputSurface(mSurface); 476 } else { 477 mSurface = mEncoder.createInputSurface(); 478 } 479 assertTrue("Surface is not valid", mSurface.isValid()); 480 mAsyncHandleDecoder.setCallBack(mDecoder, isAsync); 481 mDecoder.configure(decFormat, mSurface, null, 0); 482 mTestEnv.setLength(0); 483 mTestEnv.append("################### Test Environment #####################\n"); 484 mTestEnv.append(String.format("Encoder under test :- %s \n", mEncoderName)); 485 mTestEnv.append(String.format("Format under test :- %s \n", encFormat)); 486 mTestEnv.append(String.format("Encoder is fed with output of :- %s \n", mDecoderName)); 487 mTestEnv.append(String.format("Format of Decoder Input :- %s", decFormat)); 488 mTestEnv.append(String.format("Encoder and Decoder are operating in :- %s mode \n", 489 (isAsync ? "asynchronous" : "synchronous"))); 490 mTestEnv.append(String.format("Components received input eos :- %s \n", 491 (signalEOSWithLastFrame ? "with full buffer" : "with empty buffer"))); 492 if (ENABLE_LOGS) { 493 Log.v(LOG_TAG, "codec configured"); 494 } 495 } 496 enqueueDecoderEOS(int bufferIndex)497 private void enqueueDecoderEOS(int bufferIndex) { 498 if (!mSawDecInputEOS) { 499 mDecoder.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 500 mSawDecInputEOS = true; 501 if (ENABLE_LOGS) { 502 Log.v(LOG_TAG, "Queued End of Stream"); 503 } 504 } 505 } 506 enqueueDecoderInput(int bufferIndex)507 private void enqueueDecoderInput(int bufferIndex) { 508 if (mExtractor.getSampleSize() < 0) { 509 enqueueDecoderEOS(bufferIndex); 510 } else { 511 ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex); 512 mExtractor.readSampleData(inputBuffer, 0); 513 int size = (int) mExtractor.getSampleSize(); 514 long pts = mExtractor.getSampleTime(); 515 int extractorFlags = mExtractor.getSampleFlags(); 516 int codecFlags = 0; 517 if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { 518 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME; 519 } 520 if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) { 521 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME; 522 } 523 if (!mExtractor.advance() && mSignalEOSWithLastFrame) { 524 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM; 525 mSawDecInputEOS = true; 526 } 527 if (ENABLE_LOGS) { 528 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts + 529 " flags: " + codecFlags); 530 } 531 mDecoder.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags); 532 if (size > 0 && (codecFlags & (MediaCodec.BUFFER_FLAG_CODEC_CONFIG | 533 MediaCodec.BUFFER_FLAG_PARTIAL_FRAME)) == 0) { 534 mOutputBuff.saveInPTS(pts); 535 mDecInputCount++; 536 } 537 } 538 } 539 dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info)540 private void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info) { 541 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 542 mSawDecOutputEOS = true; 543 } 544 if (ENABLE_LOGS) { 545 Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " + 546 info.size + " timestamp: " + info.presentationTimeUs); 547 } 548 if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 549 mDecOutputCount++; 550 } 551 mDecoder.releaseOutputBuffer(bufferIndex, mSurface != null); 552 } 553 dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info)554 private void dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info) { 555 if (ENABLE_LOGS) { 556 Log.v(LOG_TAG, "encoder output: id: " + bufferIndex + " flags: " + info.flags + 557 " size: " + info.size + " timestamp: " + info.presentationTimeUs); 558 } 559 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 560 mSawEncOutputEOS = true; 561 } 562 if (info.size > 0) { 563 ByteBuffer buf = mEncoder.getOutputBuffer(bufferIndex); 564 if (mSaveToMem) { 565 mOutputBuff.saveToMemory(buf, info); 566 } 567 if (mMuxer != null) { 568 if (mTrackID == -1) { 569 mTrackID = mMuxer.addTrack(mEncoder.getOutputFormat()); 570 mMuxer.start(); 571 } 572 mMuxer.writeSampleData(mTrackID, buf, info); 573 } 574 if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 575 mOutputBuff.saveOutPTS(info.presentationTimeUs); 576 mEncOutputCount++; 577 } 578 } 579 mEncoder.releaseOutputBuffer(bufferIndex, false); 580 } 581 tryEncoderOutput(long timeOutUs)582 private void tryEncoderOutput(long timeOutUs) throws InterruptedException { 583 if (mIsCodecInAsyncMode) { 584 if (!hasSeenError() && !mSawEncOutputEOS) { 585 int retry = 0; 586 while (mReviseLatency) { 587 if (mAsyncHandleEncoder.hasOutputFormatChanged()) { 588 mReviseLatency = false; 589 int actualLatency = mAsyncHandleEncoder.getOutputFormat() 590 .getInteger(MediaFormat.KEY_LATENCY, mLatency); 591 if (mLatency < actualLatency) { 592 mLatency = actualLatency; 593 return; 594 } 595 } else { 596 if (retry > CodecTestBase.RETRY_LIMIT) throw new InterruptedException( 597 "did not receive output format changed for encoder after " + 598 CodecTestBase.Q_DEQ_TIMEOUT_US * CodecTestBase.RETRY_LIMIT + 599 " us"); 600 Thread.sleep(CodecTestBase.Q_DEQ_TIMEOUT_US / 1000); 601 retry++; 602 } 603 } 604 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleEncoder.getOutput(); 605 if (element != null) { 606 dequeueEncoderOutput(element.first, element.second); 607 } 608 } 609 } else { 610 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 611 if (!mSawEncOutputEOS) { 612 int outputBufferId = mEncoder.dequeueOutputBuffer(outInfo, timeOutUs); 613 if (outputBufferId >= 0) { 614 dequeueEncoderOutput(outputBufferId, outInfo); 615 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 616 mLatency = mEncoder.getOutputFormat() 617 .getInteger(MediaFormat.KEY_LATENCY, mLatency); 618 } 619 } 620 } 621 } 622 waitForAllEncoderOutputs()623 private void waitForAllEncoderOutputs() throws InterruptedException { 624 if (mIsCodecInAsyncMode) { 625 while (!hasSeenError() && !mSawEncOutputEOS) { 626 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US); 627 } 628 } else { 629 while (!mSawEncOutputEOS) { 630 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US); 631 } 632 } 633 } 634 queueEOS()635 private void queueEOS() throws InterruptedException { 636 if (mIsCodecInAsyncMode) { 637 while (!mAsyncHandleDecoder.hasSeenError() && !mSawDecInputEOS) { 638 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork(); 639 if (element != null) { 640 int bufferID = element.first; 641 MediaCodec.BufferInfo info = element.second; 642 if (info != null) { 643 dequeueDecoderOutput(bufferID, info); 644 } else { 645 enqueueDecoderEOS(element.first); 646 } 647 } 648 } 649 } else { 650 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 651 while (!mSawDecInputEOS) { 652 int outputBufferId = 653 mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US); 654 if (outputBufferId >= 0) { 655 dequeueDecoderOutput(outputBufferId, outInfo); 656 } 657 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US); 658 if (inputBufferId != -1) { 659 enqueueDecoderEOS(inputBufferId); 660 } 661 } 662 } 663 if (mIsCodecInAsyncMode) { 664 while (!hasSeenError() && !mSawDecOutputEOS) { 665 Pair<Integer, MediaCodec.BufferInfo> decOp = mAsyncHandleDecoder.getOutput(); 666 if (decOp != null) dequeueDecoderOutput(decOp.first, decOp.second); 667 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream(); 668 if (mDecOutputCount - mEncOutputCount > mLatency) { 669 tryEncoderOutput(-1); 670 } 671 } 672 } else { 673 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 674 while (!mSawDecOutputEOS) { 675 int outputBufferId = 676 mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US); 677 if (outputBufferId >= 0) { 678 dequeueDecoderOutput(outputBufferId, outInfo); 679 } 680 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream(); 681 if (mDecOutputCount - mEncOutputCount > mLatency) { 682 tryEncoderOutput(-1); 683 } 684 } 685 } 686 } 687 doWork(int frameLimit)688 private void doWork(int frameLimit) throws InterruptedException { 689 int frameCnt = 0; 690 if (mIsCodecInAsyncMode) { 691 // dequeue output after inputEOS is expected to be done in waitForAllOutputs() 692 while (!hasSeenError() && !mSawDecInputEOS && frameCnt < frameLimit) { 693 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork(); 694 if (element != null) { 695 int bufferID = element.first; 696 MediaCodec.BufferInfo info = element.second; 697 if (info != null) { 698 // <id, info> corresponds to output callback. Handle it accordingly 699 dequeueDecoderOutput(bufferID, info); 700 } else { 701 // <id, null> corresponds to input callback. Handle it accordingly 702 enqueueDecoderInput(bufferID); 703 frameCnt++; 704 } 705 } 706 // check decoder EOS 707 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream(); 708 // encoder output 709 if (mDecOutputCount - mEncOutputCount > mLatency) { 710 tryEncoderOutput(-1); 711 } 712 } 713 } else { 714 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 715 while (!mSawDecInputEOS && frameCnt < frameLimit) { 716 // decoder input 717 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US); 718 if (inputBufferId != -1) { 719 enqueueDecoderInput(inputBufferId); 720 frameCnt++; 721 } 722 // decoder output 723 int outputBufferId = 724 mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US); 725 if (outputBufferId >= 0) { 726 dequeueDecoderOutput(outputBufferId, outInfo); 727 } 728 // check decoder EOS 729 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream(); 730 // encoder output 731 if (mDecOutputCount - mEncOutputCount > mLatency) { 732 tryEncoderOutput(-1); 733 } 734 } 735 } 736 } 737 validateToneMappedFormat(MediaFormat format, String descriptor)738 private void validateToneMappedFormat(MediaFormat format, String descriptor) { 739 assertEquals("unexpected color transfer in " + descriptor + " after tone mapping", 740 MediaFormat.COLOR_TRANSFER_SDR_VIDEO, 741 format.getInteger(MediaFormat.KEY_COLOR_TRANSFER, 0)); 742 assertNotEquals("unexpected color standard in " + descriptor + " after tone mapping", 743 MediaFormat.COLOR_STANDARD_BT2020, 744 format.getInteger(MediaFormat.KEY_COLOR_STANDARD, 0)); 745 746 int profile = format.getInteger(MediaFormat.KEY_PROFILE, -1); 747 int[] profileArray = CodecTestBase.PROFILE_HDR_MAP.get(mEncMediaType); 748 assertFalse(descriptor + " must not contain HDR profile after tone mapping", 749 IntStream.of(profileArray).anyMatch(x -> x == profile)); 750 } 751 752 /** 753 * Checks if the component under test can encode from surface properly. The test runs 754 * mediacodec in both synchronous and asynchronous mode. The test feeds the encoder input 755 * surface with output of decoder. Assuming no frame drops, the number of output frames from 756 * encoder should be identical to number of input frames to decoder. Also the timestamps 757 * should be identical. As encoder output is deterministic, the test expects consistent 758 * output in all runs. The output is written to a file using muxer. This file is validated 759 * for PSNR to check if the encoding happened successfully with out any obvious artifacts. 760 */ 761 @CddTest(requirements = {"2.2.2", "2.3.2", "2.5.2"}) 762 @ApiTest(apis = {"MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface"}) 763 @LargeTest 764 @Test(timeout = CodecTestBase.PER_TEST_TIMEOUT_LARGE_TEST_MS) testSimpleEncodeFromSurface()765 public void testSimpleEncodeFromSurface() throws IOException, InterruptedException { 766 mDecoder = MediaCodec.createByCodecName(mDecoderName); 767 String tmpPath = null; 768 boolean muxOutput = true; 769 if (mEncMediaType.equals(MediaFormat.MIMETYPE_VIDEO_AV1) && CodecTestBase.IS_BEFORE_U) { 770 muxOutput = false; 771 } 772 { 773 mEncoder = MediaCodec.createByCodecName(mEncoderName); 774 /* TODO(b/149027258) */ 775 mSaveToMem = false; 776 OutputManager ref = new OutputManager(); 777 OutputManager test = new OutputManager(ref.getSharedErrorLogs()); 778 int loopCounter = 0; 779 boolean[] boolStates = {true, false}; 780 for (boolean isAsync : boolStates) { 781 mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC); 782 mOutputBuff = loopCounter == 0 ? ref : test; 783 mOutputBuff.reset(); 784 if (muxOutput && loopCounter == 0) { 785 int muxerFormat = getMuxerFormatForMediaType(mEncMediaType); 786 tmpPath = getTempFilePath(mEncCfgParams.mInputBitDepth > 8 ? "10bit" : ""); 787 mTmpFiles.add(tmpPath); 788 mMuxer = new MediaMuxer(tmpPath, muxerFormat); 789 } 790 configureCodec(mDecoderFormat, mEncoderFormat, isAsync, false); 791 if (mIsOutputToneMapped) { 792 int transferRequest = mDecoder.getInputFormat().getInteger( 793 MediaFormat.KEY_COLOR_TRANSFER_REQUEST, 0); 794 assumeTrue(mDecoderName + " does not support HDR to SDR tone mapping", 795 0 != transferRequest); 796 } 797 mEncoder.start(); 798 mDecoder.start(); 799 doWork(Integer.MAX_VALUE); 800 queueEOS(); 801 waitForAllEncoderOutputs(); 802 MediaFormat encoderOutputFormat = mEncoder.getOutputFormat(); 803 MediaFormat decoderOutputFormat = mDecoder.getOutputFormat(); 804 if (muxOutput) { 805 if (mTrackID != -1) { 806 mMuxer.stop(); 807 mTrackID = -1; 808 } 809 if (mMuxer != null) { 810 mMuxer.release(); 811 mMuxer = null; 812 } 813 } 814 mDecoder.stop(); 815 /* TODO(b/147348711) */ 816 if (false) mEncoder.stop(); 817 else mEncoder.reset(); 818 819 assertFalse("Decoder has encountered error in async mode. \n" 820 + mTestConfig + mTestEnv + mAsyncHandleDecoder.getErrMsg(), 821 mAsyncHandleDecoder.hasSeenError()); 822 assertFalse("Encoder has encountered error in async mode. \n" 823 + mTestConfig + mTestEnv + mAsyncHandleEncoder.getErrMsg(), 824 mAsyncHandleEncoder.hasSeenError()); 825 assertTrue("Decoder has not received any input \n" + mTestConfig + mTestEnv, 826 0 != mDecInputCount); 827 assertTrue("Decoder has not sent any output \n" + mTestConfig + mTestEnv, 828 0 != mDecOutputCount); 829 assertTrue("Encoder has not sent any output \n" + mTestConfig + mTestEnv, 830 0 != mEncOutputCount); 831 assertEquals("Decoder output count is not equal to decoder input count \n" 832 + mTestConfig + mTestEnv, mDecInputCount, mDecOutputCount); 833 834 /* TODO(b/153127506) 835 * Currently disabling all encoder output checks. Added checks only for encoder 836 * timeStamp is in increasing order or not. 837 * Once issue is fixed remove increasing timestamp check and enable encoder checks. 838 */ 839 /*assertEquals("Encoder output count is not equal to Decoder input count \n" 840 + mTestConfig + mTestEnv, mDecInputCount, mEncOutputCount); 841 if (loopCounter != 0 && !ref.equals(test)) { 842 fail("Encoder output is not consistent across runs \n" + mTestConfig + mTestEnv 843 + test.getErrMsg()); 844 } 845 if (loopCounter == 0 && 846 !ref.isOutPtsListIdenticalToInpPtsList((mEncCfgParams.mMaxBFrames != 0))) { 847 fail("Input pts list and Output pts list are not identical \n" + mTestConfig 848 + mTestEnv + ref.getErrMsg()); 849 }*/ 850 if (mEncCfgParams.mMaxBFrames == 0 && !mOutputBuff.isPtsStrictlyIncreasing( 851 Long.MIN_VALUE)) { 852 fail("Output timestamps are not strictly increasing \n" + mTestConfig + mTestEnv 853 + mOutputBuff.getErrMsg()); 854 } 855 if (mIsOutputToneMapped) { 856 validateToneMappedFormat(decoderOutputFormat, "decoder output format"); 857 validateToneMappedFormat(encoderOutputFormat, "encoder output format"); 858 859 if (tmpPath != null) { 860 MediaExtractor extractor = new MediaExtractor(); 861 extractor.setDataSource(tmpPath); 862 MediaFormat extractorFormat = extractor.getTrackFormat(0); 863 extractor.release(); 864 validateToneMappedFormat(extractorFormat, "extractor format"); 865 } 866 } 867 loopCounter++; 868 mSurface.release(); 869 mSurface = null; 870 } 871 mEncoder.release(); 872 } 873 mDecoder.release(); 874 mExtractor.release(); 875 // Skip stream validation as there is no reference for tone mapped input 876 if (muxOutput && !mIsOutputToneMapped) { 877 if (mEncCfgParams.mInputBitDepth > 8 && !VNDK_IS_AT_LEAST_T) return; 878 CodecEncoderTestBase.validateEncodedPSNR(mTestFileMediaType, mTestFile, mEncMediaType, 879 tmpPath, false, false, ACCEPTABLE_WIRELESS_TX_QUALITY); 880 } 881 } 882 nativeTestSimpleEncode(String encoder, String decoder, String mediaType, String testFile, String muxFile, int colorFormat, boolean usePersistentSurface, String cfgParams, String separator, StringBuilder retMsg)883 private native boolean nativeTestSimpleEncode(String encoder, String decoder, String mediaType, 884 String testFile, String muxFile, int colorFormat, boolean usePersistentSurface, 885 String cfgParams, String separator, StringBuilder retMsg); 886 887 /** 888 * Test is similar to {@link #testSimpleEncodeFromSurface()} but uses ndk api 889 */ 890 @CddTest(requirements = {"2.2.2", "2.3.2", "2.5.2"}) 891 @ApiTest(apis = {"MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface"}) 892 @LargeTest 893 @Test(timeout = CodecTestBase.PER_TEST_TIMEOUT_LARGE_TEST_MS) testSimpleEncodeFromSurfaceNative()894 public void testSimpleEncodeFromSurfaceNative() throws IOException, InterruptedException { 895 // TODO(b/281661171) Update native tests to encode for tone mapped output 896 assumeFalse("tone mapping tests are skipped in native mode", mIsOutputToneMapped); 897 String tmpPath = null; 898 if (!mEncMediaType.equals(MediaFormat.MIMETYPE_VIDEO_AV1) || CodecTestBase.IS_AT_LEAST_U) { 899 tmpPath = getTempFilePath(mEncCfgParams.mInputBitDepth > 8 ? "10bit" : ""); 900 mTmpFiles.add(tmpPath); 901 } 902 int colorFormat = mDecoderFormat.getInteger(MediaFormat.KEY_COLOR_FORMAT, -1); 903 boolean isPass = nativeTestSimpleEncode(mEncoderName, mDecoderName, mEncMediaType, 904 mTestFile, tmpPath, colorFormat, mUsePersistentSurface, 905 EncoderConfigParams.serializeMediaFormat(mEncoderFormat), 906 EncoderConfigParams.TOKEN_SEPARATOR, mTestConfig); 907 assertTrue(mTestConfig.toString(), isPass); 908 if (tmpPath != null) { 909 if (mEncCfgParams.mInputBitDepth > 8 && !VNDK_IS_AT_LEAST_T) return; 910 CodecEncoderTestBase.validateEncodedPSNR(mTestFileMediaType, mTestFile, mEncMediaType, 911 tmpPath, false, false, ACCEPTABLE_WIRELESS_TX_QUALITY); 912 } 913 } 914 } 915