1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.common.cts; 18 19 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 20 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; 21 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010; 22 23 import static org.junit.Assert.assertEquals; 24 import static org.junit.Assert.assertNotNull; 25 import static org.junit.Assert.assertTrue; 26 import static org.junit.Assert.fail; 27 import static org.junit.Assume.assumeTrue; 28 29 import android.graphics.ImageFormat; 30 import android.media.Image; 31 import android.media.MediaCodec; 32 import android.media.MediaCodecInfo; 33 import android.media.MediaExtractor; 34 import android.media.MediaFormat; 35 import android.os.PersistableBundle; 36 import android.util.Log; 37 import android.util.Pair; 38 39 import com.android.compatibility.common.util.Preconditions; 40 41 import org.junit.After; 42 import org.junit.Before; 43 44 import java.io.IOException; 45 import java.nio.ByteBuffer; 46 import java.util.ArrayList; 47 48 /** 49 * Wrapper class for trying and testing mediacodec decoder components. 50 */ 51 public class CodecDecoderTestBase extends CodecTestBase { 52 private static final String LOG_TAG = CodecDecoderTestBase.class.getSimpleName(); 53 54 protected final String mTestFile; 55 protected boolean mIsInterlaced; 56 protected boolean mSkipChecksumVerification; 57 58 protected final ArrayList<ByteBuffer> mCsdBuffers; 59 private int mCurrCsdIdx; 60 61 private final ByteBuffer mFlatBuffer = ByteBuffer.allocate(4 * Integer.BYTES); 62 63 protected MediaExtractor mExtractor; 64 CodecDecoderTestBase(String codecName, String mediaType, String testFile, String allTestParams)65 public CodecDecoderTestBase(String codecName, String mediaType, String testFile, 66 String allTestParams) { 67 super(codecName, mediaType, allTestParams); 68 mTestFile = testFile; 69 mCsdBuffers = new ArrayList<>(); 70 } 71 72 @Before setUpCodecDecoderTestBase()73 public void setUpCodecDecoderTestBase() { 74 assertTrue("Testing a mediaType that is neither audio nor video is not supported \n" 75 + mTestConfig, mIsAudio || mIsVideo); 76 } 77 78 @After tearDownCodecDecoderTestBase()79 public void tearDownCodecDecoderTestBase() { 80 if (mExtractor != null) { 81 mExtractor.release(); 82 mExtractor = null; 83 } 84 } 85 setUpSource(String srcFile)86 protected MediaFormat setUpSource(String srcFile) throws IOException { 87 Preconditions.assertTestFileExists(srcFile); 88 mExtractor = new MediaExtractor(); 89 mExtractor.setDataSource(srcFile); 90 for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) { 91 MediaFormat format = mExtractor.getTrackFormat(trackID); 92 if (mMediaType.equalsIgnoreCase(format.getString(MediaFormat.KEY_MIME))) { 93 mExtractor.selectTrack(trackID); 94 if (mIsVideo) { 95 ArrayList<MediaFormat> formatList = new ArrayList<>(); 96 formatList.add(format); 97 boolean selectHBD = doesAnyFormatHaveHDRProfile(mMediaType, formatList); 98 if (!selectHBD && srcFile.contains("10bit")) { 99 selectHBD = true; 100 if (mMediaType.equals(MediaFormat.MIMETYPE_VIDEO_VP9)) { 101 // In some cases, webm extractor may not signal profile for 10-bit VP9 102 // clips. In such cases, set profile to a 10-bit compatible profile. 103 // TODO (b/295804596) Remove the following once webm extractor signals 104 // profile correctly for all 10-bit clips 105 int[] profileArray = CodecTestBase.PROFILE_HDR_MAP.get(mMediaType); 106 format.setInteger(MediaFormat.KEY_PROFILE, profileArray[0]); 107 } 108 } 109 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 110 getColorFormat(mCodecName, mMediaType, mSurface != null, selectHBD)); 111 if (selectHBD && (format.getInteger(MediaFormat.KEY_COLOR_FORMAT) 112 != COLOR_FormatYUVP010)) { 113 mSkipChecksumVerification = true; 114 } 115 116 if ((format.getInteger(MediaFormat.KEY_COLOR_FORMAT) != COLOR_FormatYUVP010) 117 && selectHBD && mSurface == null) { 118 // Codecs that do not advertise P010 on devices with VNDK version < T, do 119 // not support decoding high bit depth clips when color format is set to 120 // COLOR_FormatYUV420Flexible in byte buffer mode. Since byte buffer mode 121 // for high bit depth decoding wasn't tested prior to Android T, skip this 122 // when device is older 123 assumeTrue("Skipping High Bit Depth tests on VNDK < T", VNDK_IS_AT_LEAST_T); 124 } 125 } 126 // TODO: determine this from the extractor format when it becomes exposed. 127 mIsInterlaced = srcFile.contains("_interlaced_"); 128 return format; 129 } 130 } 131 fail("No track with mediaType: " + mMediaType + " found in file: " + srcFile + "\n" 132 + mTestConfig + mTestEnv); 133 return null; 134 } 135 getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode)136 int getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode) 137 throws IOException { 138 if (surfaceMode) return COLOR_FormatSurface; 139 if (hbdMode) { 140 MediaCodec codec = MediaCodec.createByCodecName(name); 141 MediaCodecInfo.CodecCapabilities cap = 142 codec.getCodecInfo().getCapabilitiesForType(mediaType); 143 codec.release(); 144 for (int c : cap.colorFormats) { 145 if (c == COLOR_FormatYUVP010) { 146 return c; 147 } 148 } 149 } 150 return COLOR_FormatYUV420Flexible; 151 } 152 hasCSD(MediaFormat format)153 public static boolean hasCSD(MediaFormat format) { 154 return format.containsKey("csd-0"); 155 } 156 flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio)157 void flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio) { 158 if (isAudio) { 159 mFlatBuffer.putInt(info.size); 160 } 161 mFlatBuffer.putInt(info.flags & ~MediaCodec.BUFFER_FLAG_END_OF_STREAM) 162 .putLong(info.presentationTimeUs); 163 mFlatBuffer.flip(); 164 } 165 enqueueCodecConfig(int bufferIndex)166 void enqueueCodecConfig(int bufferIndex) { 167 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 168 ByteBuffer csdBuffer = mCsdBuffers.get(mCurrCsdIdx); 169 inputBuffer.put((ByteBuffer) csdBuffer.rewind()); 170 mCodec.queueInputBuffer(bufferIndex, 0, csdBuffer.limit(), 0, 171 MediaCodec.BUFFER_FLAG_CODEC_CONFIG); 172 if (ENABLE_LOGS) { 173 Log.v(LOG_TAG, "queued csd: id: " + bufferIndex + " size: " + csdBuffer.limit()); 174 } 175 } 176 enqueueInput(int bufferIndex)177 protected void enqueueInput(int bufferIndex) { 178 if (mExtractor.getSampleSize() < 0) { 179 enqueueEOS(bufferIndex); 180 } else { 181 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 182 mExtractor.readSampleData(inputBuffer, 0); 183 int size = (int) mExtractor.getSampleSize(); 184 long pts = mExtractor.getSampleTime(); 185 int extractorFlags = mExtractor.getSampleFlags(); 186 int codecFlags = 0; 187 if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { 188 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME; 189 } 190 if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) { 191 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME; 192 } 193 if (!mExtractor.advance() && mSignalEOSWithLastFrame) { 194 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM; 195 mSawInputEOS = true; 196 } 197 if (ENABLE_LOGS) { 198 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts 199 + " flags: " + codecFlags); 200 } 201 mCodec.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags); 202 if (size > 0 && (codecFlags & (MediaCodec.BUFFER_FLAG_CODEC_CONFIG 203 | MediaCodec.BUFFER_FLAG_PARTIAL_FRAME)) == 0) { 204 mOutputBuff.saveInPTS(pts); 205 mInputCount++; 206 } 207 } 208 } 209 enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info)210 protected void enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info) { 211 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 212 buffer.position(info.offset); 213 for (int i = 0; i < info.size; i++) { 214 inputBuffer.put(buffer.get()); 215 } 216 if (ENABLE_LOGS) { 217 Log.v(LOG_TAG, "input: id: " + bufferIndex + " flags: " + info.flags + " size: " 218 + info.size + " timestamp: " + info.presentationTimeUs); 219 } 220 mCodec.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs, 221 info.flags); 222 if (info.size > 0 && ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) 223 && ((info.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) == 0)) { 224 mOutputBuff.saveInPTS(info.presentationTimeUs); 225 mInputCount++; 226 } 227 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 228 mSawInputEOS = true; 229 } 230 } 231 dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)232 protected void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) { 233 if (info.size > 0 && mSaveToMem) { 234 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex); 235 flattenBufferInfo(info, mIsAudio); 236 mOutputBuff.checksum(mFlatBuffer, mFlatBuffer.limit()); 237 if (mIsAudio) { 238 mOutputBuff.checksum(buf, info.size); 239 mOutputBuff.saveToMemory(buf, info); 240 } else { 241 // tests both getOutputImage and getOutputBuffer. Can do time division 242 // multiplexing but lets allow it for now 243 Image img = mCodec.getOutputImage(bufferIndex); 244 assertNotNull("CPU-read via ImageReader API is not available", img); 245 mOutputBuff.checksum(img); 246 int imgFormat = img.getFormat(); 247 int bytesPerSample = (ImageFormat.getBitsPerPixel(imgFormat) * 2) / (8 * 3); 248 249 MediaFormat format = mCodec.getOutputFormat(); 250 buf = mCodec.getOutputBuffer(bufferIndex); 251 int width = format.getInteger(MediaFormat.KEY_WIDTH); 252 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 253 int stride = format.getInteger(MediaFormat.KEY_STRIDE); 254 mOutputBuff.checksum(buf, info.size, width, height, stride, bytesPerSample); 255 } 256 } 257 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 258 mSawOutputEOS = true; 259 } 260 if (ENABLE_LOGS) { 261 Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " 262 + info.size + " timestamp: " + info.presentationTimeUs); 263 } 264 if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 265 mOutputBuff.saveOutPTS(info.presentationTimeUs); 266 mOutputCount++; 267 } 268 mCodec.releaseOutputBuffer(bufferIndex, false); 269 } 270 doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)271 protected void doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list) 272 throws InterruptedException { 273 int frameCount = 0; 274 if (mIsCodecInAsyncMode) { 275 // output processing after queuing EOS is done in waitForAllOutputs() 276 while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < list.size()) { 277 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork(); 278 if (element != null) { 279 int bufferID = element.first; 280 MediaCodec.BufferInfo info = element.second; 281 if (info != null) { 282 dequeueOutput(bufferID, info); 283 } else { 284 enqueueInput(bufferID, buffer, list.get(frameCount)); 285 frameCount++; 286 } 287 } 288 } 289 } else { 290 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 291 // output processing after queuing EOS is done in waitForAllOutputs() 292 while (!mSawInputEOS && frameCount < list.size()) { 293 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US); 294 if (outputBufferId >= 0) { 295 dequeueOutput(outputBufferId, outInfo); 296 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 297 mOutFormat = mCodec.getOutputFormat(); 298 mSignalledOutFormatChanged = true; 299 } 300 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US); 301 if (inputBufferId != -1) { 302 enqueueInput(inputBufferId, buffer, list.get(frameCount)); 303 frameCount++; 304 } 305 } 306 } 307 } 308 queueCodecConfig()309 protected void queueCodecConfig() throws InterruptedException { 310 if (mIsCodecInAsyncMode) { 311 for (mCurrCsdIdx = 0; !mAsyncHandle.hasSeenError() && mCurrCsdIdx < mCsdBuffers.size(); 312 mCurrCsdIdx++) { 313 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getInput(); 314 if (element != null) { 315 enqueueCodecConfig(element.first); 316 } 317 } 318 } else { 319 for (mCurrCsdIdx = 0; mCurrCsdIdx < mCsdBuffers.size(); mCurrCsdIdx++) { 320 enqueueCodecConfig(mCodec.dequeueInputBuffer(-1)); 321 } 322 } 323 } 324 validateTestState()325 void validateTestState() { 326 super.validateTestState(); 327 if (!mOutputBuff.isPtsStrictlyIncreasing(mPrevOutputPts)) { 328 fail("Output timestamps are not strictly increasing \n" + mTestConfig + mTestEnv 329 + mOutputBuff.getErrMsg()); 330 } 331 if (mIsVideo) { 332 // TODO: Timestamps for deinterlaced content are under review. (E.g. can decoders 333 // produce multiple progressive frames?) For now, do not verify timestamps. 334 if (!mIsInterlaced && !mOutputBuff.isOutPtsListIdenticalToInpPtsList(false)) { 335 fail("Input pts list and Output pts list are not identical ]\n" + mTestConfig 336 + mTestEnv + mOutputBuff.getErrMsg()); 337 } 338 } 339 } 340 decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)341 public void decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit) 342 throws IOException, InterruptedException { 343 mSaveToMem = true; 344 mOutputBuff = new OutputManager(); 345 mCodec = MediaCodec.createByCodecName(decoder); 346 MediaFormat format = setUpSource(file); 347 configureCodec(format, false, true, false); 348 mCodec.start(); 349 mExtractor.seekTo(pts, mode); 350 doWork(frameLimit); 351 queueEOS(); 352 waitForAllOutputs(); 353 mCodec.stop(); 354 mCodec.release(); 355 mExtractor.release(); 356 mSaveToMem = false; 357 } 358 decodeToMemory(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list, MediaFormat format, String decoder)359 public void decodeToMemory(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list, 360 MediaFormat format, String decoder) throws IOException, InterruptedException { 361 mSaveToMem = true; 362 mOutputBuff = new OutputManager(); 363 mCodec = MediaCodec.createByCodecName(decoder); 364 configureCodec(format, false, true, false); 365 mCodec.start(); 366 doWork(buffer, list); 367 queueEOS(); 368 waitForAllOutputs(); 369 mCodec.stop(); 370 mCodec.release(); 371 mSaveToMem = false; 372 } 373 374 @Override validateMetrics(String decoder, MediaFormat format)375 protected PersistableBundle validateMetrics(String decoder, MediaFormat format) { 376 PersistableBundle metrics = super.validateMetrics(decoder, format); 377 assertEquals("error! metrics#MetricsConstants.MIME_TYPE is not as expected \n" + mTestConfig 378 + mTestEnv, metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE), mMediaType); 379 assertEquals("error! metrics#MetricsConstants.ENCODER is not as expected \n" + mTestConfig 380 + mTestEnv, 0, metrics.getInt(MediaCodec.MetricsConstants.ENCODER)); 381 return metrics; 382 } 383 validateColorAspects(int range, int standard, int transfer, boolean ignoreColorBox)384 public void validateColorAspects(int range, int standard, int transfer, boolean ignoreColorBox) 385 throws IOException, InterruptedException { 386 Preconditions.assertTestFileExists(mTestFile); 387 mOutputBuff = new OutputManager(); 388 MediaFormat format = setUpSource(mTestFile); 389 if (ignoreColorBox) { 390 format.removeKey(MediaFormat.KEY_COLOR_RANGE); 391 format.removeKey(MediaFormat.KEY_COLOR_STANDARD); 392 format.removeKey(MediaFormat.KEY_COLOR_TRANSFER); 393 } 394 mCodec = MediaCodec.createByCodecName(mCodecName); 395 configureCodec(format, true, true, false); 396 mCodec.start(); 397 doWork(1); 398 queueEOS(); 399 waitForAllOutputs(); 400 validateColorAspects(mCodec.getOutputFormat(), range, standard, transfer); 401 mCodec.stop(); 402 mCodec.release(); 403 mExtractor.release(); 404 } 405 } 406