1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.cts; 18 19 import android.content.Context; 20 import android.content.pm.PackageManager; 21 import android.graphics.ImageFormat; 22 import android.graphics.Rect; 23 import android.hardware.display.DisplayManager; 24 import android.media.AudioFormat; 25 import android.media.Image; 26 import android.media.MediaCodec; 27 import android.media.MediaCodecInfo; 28 import android.media.MediaCodecInfo.CodecCapabilities; 29 import android.media.MediaCodecInfo.CodecProfileLevel; 30 import android.media.MediaCodecList; 31 import android.media.MediaExtractor; 32 import android.media.MediaFormat; 33 import android.os.Build; 34 import android.os.PersistableBundle; 35 import android.os.SystemProperties; 36 import android.util.Log; 37 import android.util.Pair; 38 import android.view.Display; 39 import android.view.Surface; 40 41 import androidx.annotation.NonNull; 42 import androidx.test.platform.app.InstrumentationRegistry; 43 44 import org.junit.After; 45 import org.junit.Assert; 46 import org.junit.Assume; 47 import org.junit.Before; 48 49 import java.io.File; 50 import java.io.FileInputStream; 51 import java.io.IOException; 52 import java.nio.ByteBuffer; 53 import java.nio.ByteOrder; 54 import java.util.ArrayList; 55 import java.util.Arrays; 56 import java.util.Collections; 57 import java.util.HashMap; 58 import java.util.HashSet; 59 import java.util.LinkedList; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Set; 63 import java.util.concurrent.locks.Condition; 64 import java.util.concurrent.locks.Lock; 65 import java.util.concurrent.locks.ReentrantLock; 66 import java.util.regex.Matcher; 67 import java.util.regex.Pattern; 68 import java.util.stream.IntStream; 69 import java.util.zip.CRC32; 70 71 import com.android.compatibility.common.util.ApiLevelUtil; 72 import com.android.compatibility.common.util.MediaUtils; 73 74 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 75 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; 76 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010; 77 import static android.media.MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing; 78 import static android.media.MediaCodecInfo.CodecProfileLevel.*; 79 import static org.junit.Assert.assertEquals; 80 import static org.junit.Assert.assertNotNull; 81 import static org.junit.Assert.assertTrue; 82 import static org.junit.Assert.fail; 83 import static org.junit.Assume.assumeTrue; 84 85 class CodecAsyncHandler extends MediaCodec.Callback { 86 private static final String LOG_TAG = CodecAsyncHandler.class.getSimpleName(); 87 private final Lock mLock = new ReentrantLock(); 88 private final Condition mCondition = mLock.newCondition(); 89 private final LinkedList<Pair<Integer, MediaCodec.BufferInfo>> mCbInputQueue; 90 private final LinkedList<Pair<Integer, MediaCodec.BufferInfo>> mCbOutputQueue; 91 private MediaFormat mOutFormat; 92 private boolean mSignalledOutFormatChanged; 93 private volatile boolean mSignalledError; 94 CodecAsyncHandler()95 CodecAsyncHandler() { 96 mCbInputQueue = new LinkedList<>(); 97 mCbOutputQueue = new LinkedList<>(); 98 mSignalledError = false; 99 mSignalledOutFormatChanged = false; 100 } 101 clearQueues()102 void clearQueues() { 103 mLock.lock(); 104 mCbInputQueue.clear(); 105 mCbOutputQueue.clear(); 106 mLock.unlock(); 107 } 108 resetContext()109 void resetContext() { 110 clearQueues(); 111 mOutFormat = null; 112 mSignalledOutFormatChanged = false; 113 mSignalledError = false; 114 } 115 116 @Override onInputBufferAvailable(@onNull MediaCodec codec, int bufferIndex)117 public void onInputBufferAvailable(@NonNull MediaCodec codec, int bufferIndex) { 118 assertTrue(bufferIndex >= 0); 119 mLock.lock(); 120 mCbInputQueue.add(new Pair<>(bufferIndex, (MediaCodec.BufferInfo) null)); 121 mCondition.signalAll(); 122 mLock.unlock(); 123 } 124 125 @Override onOutputBufferAvailable(@onNull MediaCodec codec, int bufferIndex, @NonNull MediaCodec.BufferInfo info)126 public void onOutputBufferAvailable(@NonNull MediaCodec codec, int bufferIndex, 127 @NonNull MediaCodec.BufferInfo info) { 128 assertTrue(bufferIndex >= 0); 129 mLock.lock(); 130 mCbOutputQueue.add(new Pair<>(bufferIndex, info)); 131 mCondition.signalAll(); 132 mLock.unlock(); 133 } 134 135 @Override onError(@onNull MediaCodec codec, MediaCodec.CodecException e)136 public void onError(@NonNull MediaCodec codec, MediaCodec.CodecException e) { 137 mLock.lock(); 138 mSignalledError = true; 139 mCondition.signalAll(); 140 mLock.unlock(); 141 Log.e(LOG_TAG, "received media codec error : " + e.getMessage()); 142 } 143 144 @Override onOutputFormatChanged(@onNull MediaCodec codec, @NonNull MediaFormat format)145 public void onOutputFormatChanged(@NonNull MediaCodec codec, @NonNull MediaFormat format) { 146 mOutFormat = format; 147 mSignalledOutFormatChanged = true; 148 Log.i(LOG_TAG, "Output format changed: " + format.toString()); 149 } 150 setCallBack(MediaCodec codec, boolean isCodecInAsyncMode)151 void setCallBack(MediaCodec codec, boolean isCodecInAsyncMode) { 152 if (isCodecInAsyncMode) { 153 codec.setCallback(this); 154 } else { 155 codec.setCallback(null); 156 } 157 } 158 getInput()159 Pair<Integer, MediaCodec.BufferInfo> getInput() throws InterruptedException { 160 Pair<Integer, MediaCodec.BufferInfo> element = null; 161 mLock.lock(); 162 while (!mSignalledError) { 163 if (mCbInputQueue.isEmpty()) { 164 mCondition.await(); 165 } else { 166 element = mCbInputQueue.remove(0); 167 break; 168 } 169 } 170 mLock.unlock(); 171 return element; 172 } 173 getOutput()174 Pair<Integer, MediaCodec.BufferInfo> getOutput() throws InterruptedException { 175 Pair<Integer, MediaCodec.BufferInfo> element = null; 176 mLock.lock(); 177 while (!mSignalledError) { 178 if (mCbOutputQueue.isEmpty()) { 179 mCondition.await(); 180 } else { 181 element = mCbOutputQueue.remove(0); 182 break; 183 } 184 } 185 mLock.unlock(); 186 return element; 187 } 188 getWork()189 Pair<Integer, MediaCodec.BufferInfo> getWork() throws InterruptedException { 190 Pair<Integer, MediaCodec.BufferInfo> element = null; 191 mLock.lock(); 192 while (!mSignalledError) { 193 if (mCbInputQueue.isEmpty() && mCbOutputQueue.isEmpty()) { 194 mCondition.await(); 195 } else { 196 if (!mCbOutputQueue.isEmpty()) { 197 element = mCbOutputQueue.remove(0); 198 break; 199 } 200 if (!mCbInputQueue.isEmpty()) { 201 element = mCbInputQueue.remove(0); 202 break; 203 } 204 } 205 } 206 mLock.unlock(); 207 return element; 208 } 209 isInputQueueEmpty()210 boolean isInputQueueEmpty() { 211 mLock.lock(); 212 boolean isEmpty = mCbInputQueue.isEmpty(); 213 mLock.unlock(); 214 return isEmpty; 215 } 216 hasSeenError()217 boolean hasSeenError() { 218 return mSignalledError; 219 } 220 hasOutputFormatChanged()221 boolean hasOutputFormatChanged() { 222 return mSignalledOutFormatChanged; 223 } 224 getOutputFormat()225 MediaFormat getOutputFormat() { 226 return mOutFormat; 227 } 228 } 229 230 class OutputManager { 231 private static final String LOG_TAG = OutputManager.class.getSimpleName(); 232 private byte[] memory; 233 private int memIndex; 234 private CRC32 mCrc32UsingImage; 235 private CRC32 mCrc32UsingBuffer; 236 private ArrayList<Long> inpPtsList; 237 private ArrayList<Long> outPtsList; 238 OutputManager()239 OutputManager() { 240 memory = new byte[1024]; 241 memIndex = 0; 242 mCrc32UsingImage = new CRC32(); 243 mCrc32UsingBuffer = new CRC32(); 244 inpPtsList = new ArrayList<>(); 245 outPtsList = new ArrayList<>(); 246 } 247 saveInPTS(long pts)248 void saveInPTS(long pts) { 249 // Add only Unique timeStamp, discarding any duplicate frame / non-display frame 250 if (!inpPtsList.contains(pts)) { 251 inpPtsList.add(pts); 252 } 253 } 254 saveOutPTS(long pts)255 void saveOutPTS(long pts) { 256 outPtsList.add(pts); 257 } 258 isPtsStrictlyIncreasing(long lastPts)259 boolean isPtsStrictlyIncreasing(long lastPts) { 260 boolean res = true; 261 for (int i = 0; i < outPtsList.size(); i++) { 262 if (lastPts < outPtsList.get(i)) { 263 lastPts = outPtsList.get(i); 264 } else { 265 Log.e(LOG_TAG, "Timestamp ordering check failed: last timestamp: " + lastPts + 266 " current timestamp:" + outPtsList.get(i)); 267 res = false; 268 break; 269 } 270 } 271 return res; 272 } 273 isOutPtsListIdenticalToInpPtsList(boolean requireSorting)274 boolean isOutPtsListIdenticalToInpPtsList(boolean requireSorting) { 275 boolean res; 276 Collections.sort(inpPtsList); 277 if (requireSorting) { 278 Collections.sort(outPtsList); 279 } 280 if (outPtsList.size() != inpPtsList.size()) { 281 Log.e(LOG_TAG, "input and output presentation timestamp list sizes are not identical" + 282 "exp/rec" + inpPtsList.size() + '/' + outPtsList.size()); 283 return false; 284 } else { 285 int count = 0; 286 for (int i = 0; i < outPtsList.size(); i++) { 287 if (!outPtsList.get(i).equals(inpPtsList.get(i))) { 288 count ++; 289 Log.e(LOG_TAG, "input output pts mismatch, exp/rec " + outPtsList.get(i) + '/' + 290 inpPtsList.get(i)); 291 if (count == 20) { 292 Log.e(LOG_TAG, "stopping after 20 mismatches, ..."); 293 break; 294 } 295 } 296 } 297 res = (count == 0); 298 } 299 return res; 300 } 301 getOutStreamSize()302 int getOutStreamSize() { 303 return memIndex; 304 } 305 checksum(ByteBuffer buf, int size)306 void checksum(ByteBuffer buf, int size) { 307 checksum(buf, size, 0, 0, 0, 0); 308 } 309 checksum(ByteBuffer buf, int size, int width, int height, int stride, int bytesPerSample)310 void checksum(ByteBuffer buf, int size, int width, int height, int stride, int bytesPerSample) { 311 int cap = buf.capacity(); 312 assertTrue("checksum() params are invalid: size = " + size + " cap = " + cap, 313 size > 0 && size <= cap); 314 if (buf.hasArray()) { 315 if (width > 0 && height > 0 && stride > 0 && bytesPerSample > 0) { 316 int offset = buf.position() + buf.arrayOffset(); 317 byte[] bb = new byte[width * height * bytesPerSample]; 318 for (int i = 0; i < height; ++i) { 319 System.arraycopy(buf.array(), offset, bb, i * width * bytesPerSample, 320 width * bytesPerSample); 321 offset += stride; 322 } 323 mCrc32UsingBuffer.update(bb, 0, width * height * bytesPerSample); 324 } else { 325 mCrc32UsingBuffer.update(buf.array(), buf.position() + buf.arrayOffset(), size); 326 } 327 } else if (width > 0 && height > 0 && stride > 0 && bytesPerSample > 0) { 328 // Checksum only the Y plane 329 int pos = buf.position(); 330 int offset = pos; 331 byte[] bb = new byte[width * height * bytesPerSample]; 332 for (int i = 0; i < height; ++i) { 333 buf.position(offset); 334 buf.get(bb, i * width * bytesPerSample, width * bytesPerSample); 335 offset += stride; 336 } 337 mCrc32UsingBuffer.update(bb, 0, width * height * bytesPerSample); 338 buf.position(pos); 339 } else { 340 int pos = buf.position(); 341 final int rdsize = Math.min(4096, size); 342 byte[] bb = new byte[rdsize]; 343 int chk; 344 for (int i = 0; i < size; i += chk) { 345 chk = Math.min(rdsize, size - i); 346 buf.get(bb, 0, chk); 347 mCrc32UsingBuffer.update(bb, 0, chk); 348 } 349 buf.position(pos); 350 } 351 } 352 checksum(Image image)353 void checksum(Image image) { 354 int format = image.getFormat(); 355 assertTrue("unexpected image format", 356 format == ImageFormat.YUV_420_888 || format == ImageFormat.YCBCR_P010); 357 int bytesPerSample = (ImageFormat.getBitsPerPixel(format) * 2) / (8 * 3); // YUV420 358 359 Rect cropRect = image.getCropRect(); 360 int imageWidth = cropRect.width(); 361 int imageHeight = cropRect.height(); 362 assertTrue("unexpected image dimensions", imageWidth > 0 && imageHeight > 0); 363 364 int imageLeft = cropRect.left; 365 int imageTop = cropRect.top; 366 Image.Plane[] planes = image.getPlanes(); 367 for (int i = 0; i < planes.length; ++i) { 368 ByteBuffer buf = planes[i].getBuffer(); 369 int width, height, rowStride, pixelStride, x, y, left, top; 370 rowStride = planes[i].getRowStride(); 371 pixelStride = planes[i].getPixelStride(); 372 if (i == 0) { 373 assertEquals(bytesPerSample, pixelStride); 374 width = imageWidth; 375 height = imageHeight; 376 left = imageLeft; 377 top = imageTop; 378 } else { 379 width = imageWidth / 2; 380 height = imageHeight / 2; 381 left = imageLeft / 2; 382 top = imageTop / 2; 383 } 384 int cropOffset = (left * pixelStride) + top * rowStride; 385 // local contiguous pixel buffer 386 byte[] bb = new byte[width * height * bytesPerSample]; 387 388 if (buf.hasArray()) { 389 byte[] b = buf.array(); 390 int offs = buf.arrayOffset() + cropOffset; 391 if (pixelStride == bytesPerSample) { 392 for (y = 0; y < height; ++y) { 393 System.arraycopy(b, offs + y * rowStride, bb, y * width * bytesPerSample, 394 width * bytesPerSample); 395 } 396 } else { 397 // do it pixel-by-pixel 398 for (y = 0; y < height; ++y) { 399 int lineOffset = offs + y * rowStride; 400 for (x = 0; x < width; ++x) { 401 for (int bytePos = 0; bytePos < bytesPerSample; ++bytePos) { 402 bb[y * width * bytesPerSample + x * bytesPerSample + bytePos] = 403 b[lineOffset + x * pixelStride + bytePos]; 404 } 405 } 406 } 407 } 408 } else { // almost always ends up here due to direct buffers 409 int base = buf.position(); 410 int pos = base + cropOffset; 411 if (pixelStride == bytesPerSample) { 412 for (y = 0; y < height; ++y) { 413 buf.position(pos + y * rowStride); 414 buf.get(bb, y * width * bytesPerSample, width * bytesPerSample); 415 } 416 } else { 417 // local line buffer 418 byte[] lb = new byte[rowStride]; 419 // do it pixel-by-pixel 420 for (y = 0; y < height; ++y) { 421 buf.position(pos + y * rowStride); 422 // we're only guaranteed to have pixelStride * (width - 1) + 423 // bytesPerSample bytes 424 buf.get(lb, 0, pixelStride * (width - 1) + bytesPerSample); 425 for (x = 0; x < width; ++x) { 426 for (int bytePos = 0; bytePos < bytesPerSample; ++bytePos) { 427 bb[y * width * bytesPerSample + x * bytesPerSample + bytePos] = 428 lb[x * pixelStride + bytePos]; 429 } 430 } 431 } 432 } 433 buf.position(base); 434 } 435 mCrc32UsingImage.update(bb, 0, width * height * bytesPerSample); 436 } 437 } 438 saveToMemory(ByteBuffer buf, MediaCodec.BufferInfo info)439 void saveToMemory(ByteBuffer buf, MediaCodec.BufferInfo info) { 440 if (memIndex + info.size >= memory.length) { 441 memory = Arrays.copyOf(memory, memIndex + info.size); 442 } 443 buf.position(info.offset); 444 buf.get(memory, memIndex, info.size); 445 memIndex += info.size; 446 } 447 position(int index)448 void position(int index) { 449 if (index < 0 || index >= memory.length) index = 0; 450 memIndex = index; 451 } 452 getBuffer()453 ByteBuffer getBuffer() { 454 return ByteBuffer.wrap(memory); 455 } 456 reset()457 void reset() { 458 position(0); 459 mCrc32UsingImage.reset(); 460 mCrc32UsingBuffer.reset(); 461 inpPtsList.clear(); 462 outPtsList.clear(); 463 } 464 getRmsError(Object refObject, int audioFormat)465 float getRmsError(Object refObject, int audioFormat) { 466 double totalErrorSquared = 0; 467 double avgErrorSquared; 468 int bytesPerSample = AudioFormat.getBytesPerSample(audioFormat); 469 if (refObject instanceof float[]) { 470 if (audioFormat != AudioFormat.ENCODING_PCM_FLOAT) return Float.MAX_VALUE; 471 float[] refData = (float[]) refObject; 472 if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE; 473 float[] floatData = new float[refData.length]; 474 ByteBuffer.wrap(memory, 0, memIndex).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer() 475 .get(floatData); 476 for (int i = 0; i < refData.length; i++) { 477 float d = floatData[i] - refData[i]; 478 totalErrorSquared += d * d; 479 } 480 avgErrorSquared = (totalErrorSquared / refData.length); 481 } else if (refObject instanceof int[]) { 482 int[] refData = (int[]) refObject; 483 int[] intData; 484 if (audioFormat == AudioFormat.ENCODING_PCM_24BIT_PACKED) { 485 if (refData.length != (memIndex / bytesPerSample)) return Float.MAX_VALUE; 486 intData = new int[refData.length]; 487 for (int i = 0, j = 0; i < memIndex; i += 3, j++) { 488 intData[j] = memory[j] | (memory[j + 1] << 8) | (memory[j + 2] << 16); 489 } 490 } else if (audioFormat == AudioFormat.ENCODING_PCM_32BIT) { 491 if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE; 492 intData = new int[refData.length]; 493 ByteBuffer.wrap(memory, 0, memIndex).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer() 494 .get(intData); 495 } else { 496 return Float.MAX_VALUE; 497 } 498 for (int i = 0; i < intData.length; i++) { 499 float d = intData[i] - refData[i]; 500 totalErrorSquared += d * d; 501 } 502 avgErrorSquared = (totalErrorSquared / refData.length); 503 } else if (refObject instanceof short[]) { 504 short[] refData = (short[]) refObject; 505 if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE; 506 if (audioFormat != AudioFormat.ENCODING_PCM_16BIT) return Float.MAX_VALUE; 507 short[] shortData = new short[refData.length]; 508 ByteBuffer.wrap(memory, 0, memIndex).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer() 509 .get(shortData); 510 for (int i = 0; i < shortData.length; i++) { 511 float d = shortData[i] - refData[i]; 512 totalErrorSquared += d * d; 513 } 514 avgErrorSquared = (totalErrorSquared / refData.length); 515 } else if (refObject instanceof byte[]) { 516 byte[] refData = (byte[]) refObject; 517 if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE; 518 if (audioFormat != AudioFormat.ENCODING_PCM_8BIT) return Float.MAX_VALUE; 519 byte[] byteData = new byte[refData.length]; 520 ByteBuffer.wrap(memory, 0, memIndex).get(byteData); 521 for (int i = 0; i < byteData.length; i++) { 522 float d = byteData[i] - refData[i]; 523 totalErrorSquared += d * d; 524 } 525 avgErrorSquared = (totalErrorSquared / refData.length); 526 } else { 527 return Float.MAX_VALUE; 528 } 529 return (float) Math.sqrt(avgErrorSquared); 530 } 531 getCheckSumImage()532 long getCheckSumImage() { 533 return mCrc32UsingImage.getValue(); 534 } 535 getCheckSumBuffer()536 long getCheckSumBuffer() { 537 return mCrc32UsingBuffer.getValue(); 538 } 539 540 @Override equals(Object o)541 public boolean equals(Object o) { 542 if (this == o) return true; 543 if (o == null || getClass() != o.getClass()) return false; 544 OutputManager that = (OutputManager) o; 545 // TODO: Timestamps for deinterlaced content are under review. (E.g. can decoders 546 // produce multiple progressive frames?) For now, do not verify timestamps. 547 boolean isEqual = this.equalsInterlaced(o); 548 if (!outPtsList.equals(that.outPtsList)) { 549 isEqual = false; 550 Log.e(LOG_TAG, "ref and test presentation timestamp mismatch"); 551 } 552 return isEqual; 553 } 554 equalsInterlaced(Object o)555 public boolean equalsInterlaced(Object o) { 556 if (this == o) return true; 557 if (o == null || getClass() != o.getClass()) return false; 558 OutputManager that = (OutputManager) o; 559 boolean isEqual = true; 560 if (mCrc32UsingImage.getValue() != that.mCrc32UsingImage.getValue()) { 561 isEqual = false; 562 Log.e(LOG_TAG, "ref and test crc32 checksums calculated using image mismatch " + 563 mCrc32UsingImage.getValue() + '/' + that.mCrc32UsingImage.getValue()); 564 } 565 if (mCrc32UsingBuffer.getValue() != that.mCrc32UsingBuffer.getValue()) { 566 isEqual = false; 567 Log.e(LOG_TAG, "ref and test crc32 checksums calculated using buffer mismatch " + 568 mCrc32UsingBuffer.getValue() + '/' + that.mCrc32UsingBuffer.getValue()); 569 if (memIndex == that.memIndex) { 570 int count = 0; 571 for (int i = 0; i < memIndex; i++) { 572 if (memory[i] != that.memory[i]) { 573 count++; 574 if (count < 20) { 575 Log.d(LOG_TAG, "sample at " + i + " exp/got:: " + memory[i] + '/' + 576 that.memory[i]); 577 } 578 } 579 } 580 if (count != 0) { 581 Log.e(LOG_TAG, "ref and test o/p samples mismatch " + count); 582 } 583 } else { 584 Log.e(LOG_TAG, "ref and test o/p sizes mismatch " + memIndex + '/' + that.memIndex); 585 } 586 } 587 return isEqual; 588 } 589 } 590 591 abstract class CodecTestBase { 592 public static final boolean IS_Q = ApiLevelUtil.getApiLevel() == Build.VERSION_CODES.Q; 593 public static final boolean IS_AT_LEAST_R = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.R); 594 // Checking for CODENAME helps in cases when build version on the development branch isn't 595 // updated yet but CODENAME is updated. 596 public static final boolean IS_AT_LEAST_T = 597 ApiLevelUtil.isAtLeast(Build.VERSION_CODES.TIRAMISU) || 598 ApiLevelUtil.codenameEquals("Tiramisu"); 599 // TODO (b/223868241) Update the following to check for Build.VERSION_CODES.TIRAMISU once 600 // TIRAMISU is set correctly 601 public static final boolean FIRST_SDK_IS_AT_LEAST_T = 602 ApiLevelUtil.isFirstApiAfter(Build.VERSION_CODES.S_V2); 603 public static final boolean VNDK_IS_AT_LEAST_T = 604 SystemProperties.getInt("ro.vndk.version", 0) > Build.VERSION_CODES.S_V2; 605 public static final boolean BOARD_SDK_IS_AT_LEAST_T = 606 SystemProperties.getInt("ro.board.api_level", 0) > Build.VERSION_CODES.S_V2; 607 public static final boolean IS_HDR_EDITING_SUPPORTED = isHDREditingSupported(); 608 private static final String LOG_TAG = CodecTestBase.class.getSimpleName(); 609 enum SupportClass { 610 CODEC_ALL, // All codecs must support 611 CODEC_ANY, // At least one codec must support 612 CODEC_DEFAULT, // Default codec must support 613 CODEC_OPTIONAL // Codec support is optional 614 } 615 static final String HDR_STATIC_INFO = 616 "00 d0 84 80 3e c2 33 c4 86 4c 1d b8 0b 13 3d 42 40 e8 03 64 00 e8 03 2c 01"; 617 static final String[] HDR_DYNAMIC_INFO = new String[]{ 618 "b5 00 3c 00 01 04 00 40 00 0c 80 4e 20 27 10 00" + 619 "0a 00 00 24 08 00 00 28 00 00 50 00 28 c8 00 c9" + 620 "90 02 aa 58 05 ca d0 0c 0a f8 16 83 18 9c 18 00" + 621 "40 78 13 64 d5 7c 2e 2c c3 59 de 79 6e c3 c2 00", 622 623 "b5 00 3c 00 01 04 00 40 00 0c 80 4e 20 27 10 00" + 624 "0a 00 00 24 08 00 00 28 00 00 50 00 28 c8 00 c9" + 625 "90 02 aa 58 05 ca d0 0c 0a f8 16 83 18 9c 18 00" + 626 "40 78 13 64 d5 7c 2e 2c c3 59 de 79 6e c3 c2 00", 627 628 "b5 00 3c 00 01 04 00 40 00 0c 80 4e 20 27 10 00" + 629 "0e 80 00 24 08 00 00 28 00 00 50 00 28 c8 00 c9" + 630 "90 02 aa 58 05 ca d0 0c 0a f8 16 83 18 9c 18 00" + 631 "40 78 13 64 d5 7c 2e 2c c3 59 de 79 6e c3 c2 00", 632 633 "b5 00 3c 00 01 04 00 40 00 0c 80 4e 20 27 10 00" + 634 "0e 80 00 24 08 00 00 28 00 00 50 00 28 c8 00 c9" + 635 "90 02 aa 58 05 ca d0 0c 0a f8 16 83 18 9c 18 00" + 636 "40 78 13 64 d5 7c 2e 2c c3 59 de 79 6e c3 c2 00", 637 }; 638 boolean mTestDynamicMetadata = false; 639 static final String CODEC_PREFIX_KEY = "codec-prefix"; 640 static final String MEDIA_TYPE_PREFIX_KEY = "media-type-prefix"; 641 static final String MIME_SEL_KEY = "mime-sel"; 642 static final Map<String, String> codecSelKeyMimeMap = new HashMap<>(); 643 static final Map<String, String> mDefaultEncoders = new HashMap<>(); 644 static final Map<String, String> mDefaultDecoders = new HashMap<>(); 645 static final HashMap<String, int[]> mProfileMap = new HashMap<>(); 646 static final HashMap<String, int[]> mProfileSdrMap = new HashMap<>(); 647 static final HashMap<String, int[]> mProfileHlgMap = new HashMap<>(); 648 static final HashMap<String, int[]> mProfileHdr10Map = new HashMap<>(); 649 static final HashMap<String, int[]> mProfileHdr10PlusMap = new HashMap<>(); 650 static final HashMap<String, int[]> mProfileHdrMap = new HashMap<>(); 651 static final boolean ENABLE_LOGS = false; 652 static final int PER_TEST_TIMEOUT_LARGE_TEST_MS = 300000; 653 static final int PER_TEST_TIMEOUT_SMALL_TEST_MS = 60000; 654 static final int UNSPECIFIED = 0; 655 // Maintain Timeouts in sync with their counterpart in NativeMediaCommon.h 656 static final long Q_DEQ_TIMEOUT_US = 5000; // block at most 5ms while looking for io buffers 657 static final int RETRY_LIMIT = 100; // max poll counter before test aborts and returns error 658 static final String INVALID_CODEC = "unknown.codec_"; 659 static final int[] MPEG2_PROFILES = new int[]{MPEG2ProfileSimple, MPEG2ProfileMain, 660 MPEG2Profile422, MPEG2ProfileSNR, MPEG2ProfileSpatial, MPEG2ProfileHigh}; 661 static final int[] MPEG4_PROFILES = new int[]{MPEG4ProfileSimple, MPEG4ProfileSimpleScalable, 662 MPEG4ProfileCore, MPEG4ProfileMain, MPEG4ProfileNbit, MPEG4ProfileScalableTexture, 663 MPEG4ProfileSimpleFace, MPEG4ProfileSimpleFBA, MPEG4ProfileBasicAnimated, 664 MPEG4ProfileHybrid, MPEG4ProfileAdvancedRealTime, MPEG4ProfileCoreScalable, 665 MPEG4ProfileAdvancedCoding, MPEG4ProfileAdvancedCore, MPEG4ProfileAdvancedScalable, 666 MPEG4ProfileAdvancedSimple}; 667 static final int[] H263_PROFILES = new int[]{H263ProfileBaseline, H263ProfileH320Coding, 668 H263ProfileBackwardCompatible, H263ProfileISWV2, H263ProfileISWV3, 669 H263ProfileHighCompression, H263ProfileInternet, H263ProfileInterlace, 670 H263ProfileHighLatency}; 671 static final int[] VP8_PROFILES = new int[] {VP8ProfileMain}; 672 static final int[] AVC_SDR_PROFILES = new int[]{AVCProfileBaseline, AVCProfileMain, 673 AVCProfileExtended, AVCProfileHigh, AVCProfileConstrainedBaseline, 674 AVCProfileConstrainedHigh}; 675 static final int[] AVC_HLG_PROFILES = new int[]{AVCProfileHigh10}; 676 static final int[] AVC_HDR_PROFILES = AVC_HLG_PROFILES; 677 static final int[] AVC_PROFILES = combine(AVC_SDR_PROFILES, AVC_HDR_PROFILES); 678 static final int[] VP9_SDR_PROFILES = new int[]{VP9Profile0}; 679 static final int[] VP9_HLG_PROFILES = new int[]{VP9Profile2}; 680 static final int[] VP9_HDR10_PROFILES = new int[]{VP9Profile2HDR}; 681 static final int[] VP9_HDR10Plus_PROFILES = new int[]{VP9Profile2HDR10Plus}; 682 static final int[] VP9_HDR_PROFILES = 683 combine(VP9_HLG_PROFILES, combine(VP9_HDR10_PROFILES, VP9_HDR10Plus_PROFILES)); 684 static final int[] VP9_PROFILES = combine(VP9_SDR_PROFILES, VP9_HDR_PROFILES); 685 static final int[] HEVC_SDR_PROFILES = new int[]{HEVCProfileMain, HEVCProfileMainStill}; 686 static final int[] HEVC_HLG_PROFILES = new int[]{HEVCProfileMain10}; 687 static final int[] HEVC_HDR10_PROFILES = new int[]{HEVCProfileMain10HDR10}; 688 static final int[] HEVC_HDR10Plus_PROFILES = new int[]{HEVCProfileMain10HDR10Plus}; 689 static final int[] HEVC_HDR_PROFILES = 690 combine(HEVC_HLG_PROFILES, combine(HEVC_HDR10_PROFILES, HEVC_HDR10Plus_PROFILES)); 691 static final int[] HEVC_PROFILES = combine(HEVC_SDR_PROFILES, HEVC_HDR_PROFILES); 692 static final int[] AV1_SDR_PROFILES = new int[]{AV1ProfileMain8}; 693 static final int[] AV1_HLG_PROFILES = new int[]{AV1ProfileMain10}; 694 static final int[] AV1_HDR10_PROFILES = new int[]{AV1ProfileMain10HDR10}; 695 static final int[] AV1_HDR10Plus_PROFILES = new int[]{AV1ProfileMain10HDR10Plus}; 696 static final int[] AV1_HDR_PROFILES = 697 combine(AV1_HLG_PROFILES, combine(AV1_HDR10_PROFILES, AV1_HDR10Plus_PROFILES)); 698 static final int[] AV1_PROFILES = combine(AV1_SDR_PROFILES, AV1_HDR_PROFILES); 699 static final int[] AAC_PROFILES = new int[]{AACObjectMain, AACObjectLC, AACObjectSSR, 700 AACObjectLTP, AACObjectHE, AACObjectScalable, AACObjectERLC, AACObjectERScalable, 701 AACObjectLD, AACObjectELD, AACObjectXHE}; 702 static final String mInpPrefix = WorkDir.getMediaDirString(); 703 static final Context mContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); 704 static final PackageManager pm = mContext.getPackageManager(); 705 static String mimeSelKeys; 706 static String codecPrefix; 707 static String mediaTypePrefix; 708 709 CodecAsyncHandler mAsyncHandle; 710 boolean mIsCodecInAsyncMode; 711 boolean mSawInputEOS; 712 boolean mSawOutputEOS; 713 boolean mSignalEOSWithLastFrame; 714 int mInputCount; 715 int mOutputCount; 716 long mPrevOutputPts; 717 boolean mSignalledOutFormatChanged; 718 MediaFormat mOutFormat; 719 boolean mIsAudio; 720 721 boolean mSaveToMem; 722 OutputManager mOutputBuff; 723 724 String mCodecName; 725 MediaCodec mCodec; 726 Surface mSurface; 727 728 static { 729 System.loadLibrary("ctsmediav2codec_jni"); 730 731 codecSelKeyMimeMap.put("vp8", MediaFormat.MIMETYPE_VIDEO_VP8); 732 codecSelKeyMimeMap.put("vp9", MediaFormat.MIMETYPE_VIDEO_VP9); 733 codecSelKeyMimeMap.put("av1", MediaFormat.MIMETYPE_VIDEO_AV1); 734 codecSelKeyMimeMap.put("avc", MediaFormat.MIMETYPE_VIDEO_AVC); 735 codecSelKeyMimeMap.put("hevc", MediaFormat.MIMETYPE_VIDEO_HEVC); 736 codecSelKeyMimeMap.put("mpeg4", MediaFormat.MIMETYPE_VIDEO_MPEG4); 737 codecSelKeyMimeMap.put("h263", MediaFormat.MIMETYPE_VIDEO_H263); 738 codecSelKeyMimeMap.put("mpeg2", MediaFormat.MIMETYPE_VIDEO_MPEG2); 739 codecSelKeyMimeMap.put("vraw", MediaFormat.MIMETYPE_VIDEO_RAW); 740 codecSelKeyMimeMap.put("amrnb", MediaFormat.MIMETYPE_AUDIO_AMR_NB); 741 codecSelKeyMimeMap.put("amrwb", MediaFormat.MIMETYPE_AUDIO_AMR_WB); 742 codecSelKeyMimeMap.put("mp3", MediaFormat.MIMETYPE_AUDIO_MPEG); 743 codecSelKeyMimeMap.put("aac", MediaFormat.MIMETYPE_AUDIO_AAC); 744 codecSelKeyMimeMap.put("vorbis", MediaFormat.MIMETYPE_AUDIO_VORBIS); 745 codecSelKeyMimeMap.put("opus", MediaFormat.MIMETYPE_AUDIO_OPUS); 746 codecSelKeyMimeMap.put("g711alaw", MediaFormat.MIMETYPE_AUDIO_G711_ALAW); 747 codecSelKeyMimeMap.put("g711mlaw", MediaFormat.MIMETYPE_AUDIO_G711_MLAW); 748 codecSelKeyMimeMap.put("araw", MediaFormat.MIMETYPE_AUDIO_RAW); 749 codecSelKeyMimeMap.put("flac", MediaFormat.MIMETYPE_AUDIO_FLAC); 750 codecSelKeyMimeMap.put("gsm", MediaFormat.MIMETYPE_AUDIO_MSGSM); 751 752 android.os.Bundle args = InstrumentationRegistry.getArguments(); 753 mimeSelKeys = args.getString(MIME_SEL_KEY); 754 codecPrefix = args.getString(CODEC_PREFIX_KEY); 755 mediaTypePrefix = args.getString(MEDIA_TYPE_PREFIX_KEY); 756 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_SDR_PROFILES)757 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_SDR_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_SDR_PROFILES)758 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_SDR_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES)759 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES)760 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES)761 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES)762 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_SDR_PROFILES)763 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_SDR_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_SDR_PROFILES)764 mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_SDR_PROFILES); mProfileSdrMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES)765 mProfileSdrMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES); 766 mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HLG_PROFILES)767 mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HLG_PROFILES); mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HLG_PROFILES)768 mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HLG_PROFILES); mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HLG_PROFILES)769 mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HLG_PROFILES); mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HLG_PROFILES)770 mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HLG_PROFILES); 771 mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10_PROFILES)772 mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10_PROFILES); mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10_PROFILES)773 mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10_PROFILES); mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10_PROFILES)774 mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10_PROFILES); 775 mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10Plus_PROFILES)776 mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10Plus_PROFILES); mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10Plus_PROFILES)777 mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10Plus_PROFILES); mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10Plus_PROFILES)778 mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10Plus_PROFILES); 779 mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HDR_PROFILES)780 mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HDR_PROFILES); mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR_PROFILES)781 mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR_PROFILES); mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR_PROFILES)782 mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR_PROFILES); mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR_PROFILES)783 mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR_PROFILES); 784 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_PROFILES)785 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_PROFILES)786 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES)787 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES)788 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES)789 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES)790 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_PROFILES)791 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_PROFILES)792 mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_PROFILES); mProfileMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES)793 mProfileMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES); 794 } 795 combine(int[] first, int[] second)796 static int[] combine(int[] first, int[] second) { 797 int[] result = Arrays.copyOf(first, first.length + second.length); 798 System.arraycopy(second, 0, result, first.length, second.length); 799 return result; 800 } 801 isCodecLossless(String mime)802 static boolean isCodecLossless(String mime) { 803 return mime.equals(MediaFormat.MIMETYPE_AUDIO_FLAC) || 804 mime.equals(MediaFormat.MIMETYPE_AUDIO_RAW); 805 } 806 hasDecoder(String mime)807 static boolean hasDecoder(String mime) { 808 return CodecTestBase.selectCodecs(mime, null, null, false).size() != 0; 809 } 810 hasEncoder(String mime)811 static boolean hasEncoder(String mime) { 812 return CodecTestBase.selectCodecs(mime, null, null, true).size() != 0; 813 } 814 checkFormatSupport(String codecName, String mime, boolean isEncoder, ArrayList<MediaFormat> formats, String[] features, SupportClass supportRequirements)815 public static void checkFormatSupport(String codecName, String mime, boolean isEncoder, 816 ArrayList<MediaFormat> formats, String[] features, SupportClass supportRequirements) 817 throws IOException { 818 if (!areFormatsSupported(codecName, mime, formats)) { 819 switch (supportRequirements) { 820 case CODEC_ALL: 821 fail("format(s) not supported by codec: " + codecName 822 + " for mime : " + mime + " formats: " + formats); 823 break; 824 case CODEC_ANY: 825 if (selectCodecs(mime, formats, features, isEncoder).isEmpty()) 826 fail("format(s) not supported by any component for mime : " + mime 827 + " formats: " + formats); 828 break; 829 case CODEC_DEFAULT: 830 if (isDefaultCodec(codecName, mime, isEncoder)) 831 fail("format(s) not supported by default codec : " + codecName + 832 "for mime : " + mime + " formats: " + formats); 833 break; 834 case CODEC_OPTIONAL: 835 default: 836 // the later assumeTrue() ensures we skip the test for unsupported codecs 837 break; 838 } 839 Assume.assumeTrue("format(s) not supported by codec: " + codecName + " for mime : " + 840 mime, false); 841 } 842 } 843 isFeatureSupported(String name, String mime, String feature)844 static boolean isFeatureSupported(String name, String mime, String feature) throws IOException { 845 MediaCodec codec = MediaCodec.createByCodecName(name); 846 MediaCodecInfo.CodecCapabilities codecCapabilities = 847 codec.getCodecInfo().getCapabilitiesForType(mime); 848 boolean isSupported = codecCapabilities.isFeatureSupported(feature); 849 codec.release(); 850 return isSupported; 851 } 852 isHDREditingSupported()853 static boolean isHDREditingSupported() { 854 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 855 for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) { 856 if (!codecInfo.isEncoder()) { 857 continue; 858 } 859 for (String mediaType : codecInfo.getSupportedTypes()) { 860 CodecCapabilities caps = codecInfo.getCapabilitiesForType(mediaType); 861 if (caps != null && caps.isFeatureSupported(FEATURE_HdrEditing)) { 862 return true; 863 } 864 } 865 } 866 return false; 867 } 868 doesAnyFormatHaveHDRProfile(String mime, ArrayList<MediaFormat> formats)869 static boolean doesAnyFormatHaveHDRProfile(String mime, ArrayList<MediaFormat> formats) { 870 int[] profileArray = mProfileHdrMap.get(mime); 871 if (profileArray != null) { 872 for (MediaFormat format : formats) { 873 assertEquals(mime, format.getString(MediaFormat.KEY_MIME)); 874 int profile = format.getInteger(MediaFormat.KEY_PROFILE, -1); 875 if (IntStream.of(profileArray).anyMatch(x -> x == profile)) return true; 876 } 877 } 878 return false; 879 } 880 doesCodecSupportHDRProfile(String codecName, String mediaType)881 static boolean doesCodecSupportHDRProfile(String codecName, String mediaType) 882 throws IOException { 883 int[] hdrProfiles = mProfileHdrMap.get(mediaType); 884 if (hdrProfiles == null) { 885 return false; 886 } 887 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 888 for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) { 889 if (!codecName.equals(codecInfo.getName())) { 890 continue; 891 } 892 CodecCapabilities caps = codecInfo.getCapabilitiesForType(mediaType); 893 if (caps == null) { 894 return false; 895 } 896 for (CodecProfileLevel pl : caps.profileLevels) { 897 if (IntStream.of(hdrProfiles).anyMatch(x -> x == pl.profile)) { 898 return true; 899 } 900 } 901 } 902 return false; 903 } 904 canDisplaySupportHDRContent()905 static boolean canDisplaySupportHDRContent() { 906 DisplayManager displayManager = mContext.getSystemService(DisplayManager.class); 907 return displayManager.getDisplay(Display.DEFAULT_DISPLAY).getHdrCapabilities() 908 .getSupportedHdrTypes().length != 0; 909 } 910 areFormatsSupported(String name, String mime, ArrayList<MediaFormat> formats)911 static boolean areFormatsSupported(String name, String mime, ArrayList<MediaFormat> formats) 912 throws IOException { 913 MediaCodec codec = MediaCodec.createByCodecName(name); 914 MediaCodecInfo.CodecCapabilities codecCapabilities = 915 codec.getCodecInfo().getCapabilitiesForType(mime); 916 boolean isSupported = true; 917 if (formats != null) { 918 for (int i = 0; i < formats.size() && isSupported; i++) { 919 isSupported = codecCapabilities.isFormatSupported(formats.get(i)); 920 } 921 } 922 codec.release(); 923 return isSupported; 924 } 925 hasSupportForColorFormat(String name, String mime, int colorFormat)926 static boolean hasSupportForColorFormat(String name, String mime, int colorFormat) 927 throws IOException { 928 MediaCodec codec = MediaCodec.createByCodecName(name); 929 MediaCodecInfo.CodecCapabilities cap = 930 codec.getCodecInfo().getCapabilitiesForType(mime); 931 boolean hasSupport = false; 932 for (int c : cap.colorFormats) { 933 if (c == colorFormat) { 934 hasSupport = true; 935 break; 936 } 937 } 938 codec.release(); 939 return hasSupport; 940 } 941 isDefaultCodec(String codecName, String mime, boolean isEncoder)942 static boolean isDefaultCodec(String codecName, String mime, boolean isEncoder) 943 throws IOException { 944 Map<String,String> mDefaultCodecs = isEncoder ? mDefaultEncoders: mDefaultDecoders; 945 if (mDefaultCodecs.containsKey(mime)) { 946 return mDefaultCodecs.get(mime).equalsIgnoreCase(codecName); 947 } 948 MediaCodec codec = isEncoder ? MediaCodec.createEncoderByType(mime) 949 : MediaCodec.createDecoderByType(mime); 950 boolean isDefault = codec.getName().equalsIgnoreCase(codecName); 951 mDefaultCodecs.put(mime, codec.getName()); 952 codec.release(); 953 return isDefault; 954 } 955 isVendorCodec(String codecName)956 static boolean isVendorCodec(String codecName) { 957 MediaCodecList mcl = new MediaCodecList(MediaCodecList.ALL_CODECS); 958 for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) { 959 if (codecName.equals(codecInfo.getName())) { 960 return codecInfo.isVendor(); 961 } 962 } 963 return false; 964 } 965 compileRequiredMimeList(boolean isEncoder, boolean needAudio, boolean needVideo)966 static ArrayList<String> compileRequiredMimeList(boolean isEncoder, boolean needAudio, 967 boolean needVideo) { 968 Set<String> list = new HashSet<>(); 969 if (!isEncoder) { 970 if (MediaUtils.hasAudioOutput() && needAudio) { 971 // sec 5.1.2 972 list.add(MediaFormat.MIMETYPE_AUDIO_AAC); 973 list.add(MediaFormat.MIMETYPE_AUDIO_FLAC); 974 list.add(MediaFormat.MIMETYPE_AUDIO_MPEG); 975 list.add(MediaFormat.MIMETYPE_AUDIO_VORBIS); 976 list.add(MediaFormat.MIMETYPE_AUDIO_RAW); 977 list.add(MediaFormat.MIMETYPE_AUDIO_OPUS); 978 } 979 if (MediaUtils.isHandheld() || MediaUtils.isTablet() || MediaUtils.isTv() || 980 MediaUtils.isAutomotive()) { 981 // sec 2.2.2, 2.3.2, 2.5.2 982 if (needAudio) { 983 list.add(MediaFormat.MIMETYPE_AUDIO_AAC); 984 } 985 if (needVideo) { 986 list.add(MediaFormat.MIMETYPE_VIDEO_AVC); 987 list.add(MediaFormat.MIMETYPE_VIDEO_MPEG4); 988 list.add(MediaFormat.MIMETYPE_VIDEO_H263); 989 list.add(MediaFormat.MIMETYPE_VIDEO_VP8); 990 list.add(MediaFormat.MIMETYPE_VIDEO_VP9); 991 } 992 } 993 if (MediaUtils.isHandheld() || MediaUtils.isTablet()) { 994 // sec 2.2.2 995 if (needAudio) { 996 list.add(MediaFormat.MIMETYPE_AUDIO_AMR_NB); 997 list.add(MediaFormat.MIMETYPE_AUDIO_AMR_WB); 998 } 999 if (needVideo) { 1000 list.add(MediaFormat.MIMETYPE_VIDEO_HEVC); 1001 } 1002 } 1003 if (MediaUtils.isTv() && needVideo) { 1004 // sec 2.3.2 1005 list.add(MediaFormat.MIMETYPE_VIDEO_HEVC); 1006 list.add(MediaFormat.MIMETYPE_VIDEO_MPEG2); 1007 } 1008 } else { 1009 if (MediaUtils.hasMicrophone() && needAudio) { 1010 // sec 5.1.1 1011 // TODO(b/154423550) 1012 // list.add(MediaFormat.MIMETYPE_AUDIO_RAW); 1013 list.add(MediaFormat.MIMETYPE_AUDIO_FLAC); 1014 list.add(MediaFormat.MIMETYPE_AUDIO_OPUS); 1015 } 1016 if (MediaUtils.isHandheld() || MediaUtils.isTablet() || MediaUtils.isTv() || 1017 MediaUtils.isAutomotive()) { 1018 // sec 2.2.2, 2.3.2, 2.5.2 1019 if (needAudio) { 1020 list.add(MediaFormat.MIMETYPE_AUDIO_AAC); 1021 } 1022 if (needVideo) { 1023 list.add(MediaFormat.MIMETYPE_VIDEO_AVC); 1024 list.add(MediaFormat.MIMETYPE_VIDEO_VP8); 1025 } 1026 } 1027 if ((MediaUtils.isHandheld() || MediaUtils.isTablet()) && needAudio) { 1028 // sec 2.2.2 1029 list.add(MediaFormat.MIMETYPE_AUDIO_AMR_NB); 1030 list.add(MediaFormat.MIMETYPE_AUDIO_AMR_WB); 1031 } 1032 } 1033 return new ArrayList<>(list); 1034 } 1035 compileCompleteTestMimeList(boolean isEncoder, boolean needAudio, boolean needVideo)1036 static ArrayList<String> compileCompleteTestMimeList(boolean isEncoder, boolean needAudio, 1037 boolean needVideo) { 1038 ArrayList<String> mimes = new ArrayList<>(); 1039 if (mimeSelKeys == null) { 1040 ArrayList<String> cddRequiredMimeList = 1041 compileRequiredMimeList(isEncoder, needAudio, needVideo); 1042 MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 1043 MediaCodecInfo[] codecInfos = codecList.getCodecInfos(); 1044 for (MediaCodecInfo codecInfo : codecInfos) { 1045 if (codecInfo.isEncoder() != isEncoder) continue; 1046 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue; 1047 String[] types = codecInfo.getSupportedTypes(); 1048 for (String type : types) { 1049 if (mediaTypePrefix != null && !type.startsWith(mediaTypePrefix)) { 1050 continue; 1051 } 1052 if (!needAudio && type.startsWith("audio/")) continue; 1053 if (!needVideo && type.startsWith("video/")) continue; 1054 if (!mimes.contains(type)) { 1055 mimes.add(type); 1056 } 1057 } 1058 } 1059 if (mediaTypePrefix != null) { 1060 return mimes; 1061 } 1062 // feature_video_output is not exposed to package manager. Testing for video output 1063 // ports, such as VGA, HDMI, DisplayPort, or a wireless port for display is also not 1064 // direct. 1065 /* sec 5.2: device implementations include an embedded screen display with the 1066 diagonal length of at least 2.5 inches or include a video output port or declare the 1067 support of a camera */ 1068 if (isEncoder && needVideo && 1069 (MediaUtils.hasCamera() || MediaUtils.getScreenSizeInInches() >= 2.5) && 1070 !mimes.contains(MediaFormat.MIMETYPE_VIDEO_AVC) && 1071 !mimes.contains(MediaFormat.MIMETYPE_VIDEO_VP8)) { 1072 // Add required cdd mimes here so that respective codec tests fail. 1073 mimes.add(MediaFormat.MIMETYPE_VIDEO_AVC); 1074 mimes.add(MediaFormat.MIMETYPE_VIDEO_VP8); 1075 Log.e(LOG_TAG,"device must support at least one of VP8 or AVC video encoders"); 1076 } 1077 for (String mime : cddRequiredMimeList) { 1078 if (!mimes.contains(mime)) { 1079 // Add required cdd mimes here so that respective codec tests fail. 1080 mimes.add(mime); 1081 Log.e(LOG_TAG, "no codec found for mime " + mime + " as required by cdd"); 1082 } 1083 } 1084 } else { 1085 for (Map.Entry<String, String> entry : codecSelKeyMimeMap.entrySet()) { 1086 String key = entry.getKey(); 1087 String value = entry.getValue(); 1088 if (mimeSelKeys.contains(key) && !mimes.contains(value)) mimes.add(value); 1089 } 1090 } 1091 return mimes; 1092 } 1093 prepareParamList(List<Object[]> exhaustiveArgsList, boolean isEncoder, boolean needAudio, boolean needVideo, boolean mustTestAllCodecs)1094 static List<Object[]> prepareParamList(List<Object[]> exhaustiveArgsList, boolean isEncoder, 1095 boolean needAudio, boolean needVideo, boolean mustTestAllCodecs) { 1096 ArrayList<String> mimes = compileCompleteTestMimeList(isEncoder, needAudio, needVideo); 1097 ArrayList<String> cddRequiredMimeList = 1098 compileRequiredMimeList(isEncoder, needAudio, needVideo); 1099 final List<Object[]> argsList = new ArrayList<>(); 1100 int argLength = exhaustiveArgsList.get(0).length; 1101 for (String mime : mimes) { 1102 ArrayList<String> totalListOfCodecs = selectCodecs(mime, null, null, isEncoder); 1103 ArrayList<String> listOfCodecs = new ArrayList<>(); 1104 if (codecPrefix != null) { 1105 for (String codec : totalListOfCodecs) { 1106 if (codec.startsWith(codecPrefix)) { 1107 listOfCodecs.add(codec); 1108 } 1109 } 1110 } else { 1111 listOfCodecs = totalListOfCodecs; 1112 } 1113 if (mustTestAllCodecs && listOfCodecs.size() == 0 && codecPrefix == null) { 1114 listOfCodecs.add(INVALID_CODEC + mime); 1115 } 1116 boolean miss = true; 1117 for (Object[] arg : exhaustiveArgsList) { 1118 if (mime.equals(arg[0])) { 1119 for (String codec : listOfCodecs) { 1120 Object[] arg_ = new Object[argLength + 1]; 1121 arg_[0] = codec; 1122 System.arraycopy(arg, 0, arg_, 1, argLength); 1123 argsList.add(arg_); 1124 } 1125 miss = false; 1126 } 1127 } 1128 if (miss && mustTestAllCodecs) { 1129 if (!cddRequiredMimeList.contains(mime)) { 1130 Log.w(LOG_TAG, "no test vectors available for optional mime type " + mime); 1131 continue; 1132 } 1133 for (String codec : listOfCodecs) { 1134 Object[] arg_ = new Object[argLength + 1]; 1135 arg_[0] = codec; 1136 arg_[1] = mime; 1137 System.arraycopy(exhaustiveArgsList.get(0), 1, arg_, 2, argLength - 1); 1138 argsList.add(arg_); 1139 } 1140 } 1141 } 1142 return argsList; 1143 } 1144 enqueueInput(int bufferIndex)1145 abstract void enqueueInput(int bufferIndex) throws IOException; 1146 dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)1147 abstract void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info); 1148 configureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, boolean isEncoder)1149 void configureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, 1150 boolean isEncoder) { 1151 resetContext(isAsync, signalEOSWithLastFrame); 1152 mAsyncHandle.setCallBack(mCodec, isAsync); 1153 // signalEOS flag has nothing to do with configure. We are using this flag to try all 1154 // available configure apis 1155 if (signalEOSWithLastFrame) { 1156 mCodec.configure(format, mSurface, null, 1157 isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0); 1158 } else { 1159 mCodec.configure(format, mSurface, isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0, 1160 null); 1161 } 1162 if (ENABLE_LOGS) { 1163 Log.v(LOG_TAG, "codec configured"); 1164 } 1165 } 1166 flushCodec()1167 void flushCodec() { 1168 mCodec.flush(); 1169 // TODO(b/147576107): is it ok to clearQueues right away or wait for some signal 1170 mAsyncHandle.clearQueues(); 1171 mSawInputEOS = false; 1172 mSawOutputEOS = false; 1173 mInputCount = 0; 1174 mOutputCount = 0; 1175 mPrevOutputPts = Long.MIN_VALUE; 1176 if (ENABLE_LOGS) { 1177 Log.v(LOG_TAG, "codec flushed"); 1178 } 1179 } 1180 reConfigureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, boolean isEncoder)1181 void reConfigureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, 1182 boolean isEncoder) { 1183 /* TODO(b/147348711) */ 1184 if (false) mCodec.stop(); 1185 else mCodec.reset(); 1186 configureCodec(format, isAsync, signalEOSWithLastFrame, isEncoder); 1187 } 1188 resetContext(boolean isAsync, boolean signalEOSWithLastFrame)1189 void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) { 1190 mAsyncHandle.resetContext(); 1191 mIsCodecInAsyncMode = isAsync; 1192 mSawInputEOS = false; 1193 mSawOutputEOS = false; 1194 mSignalEOSWithLastFrame = signalEOSWithLastFrame; 1195 mInputCount = 0; 1196 mOutputCount = 0; 1197 mPrevOutputPts = Long.MIN_VALUE; 1198 mSignalledOutFormatChanged = false; 1199 } 1200 enqueueEOS(int bufferIndex)1201 void enqueueEOS(int bufferIndex) { 1202 if (!mSawInputEOS) { 1203 mCodec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 1204 mSawInputEOS = true; 1205 if (ENABLE_LOGS) { 1206 Log.v(LOG_TAG, "Queued End of Stream"); 1207 } 1208 } 1209 } 1210 doWork(int frameLimit)1211 void doWork(int frameLimit) throws InterruptedException, IOException { 1212 int frameCount = 0; 1213 if (mIsCodecInAsyncMode) { 1214 // dequeue output after inputEOS is expected to be done in waitForAllOutputs() 1215 while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < frameLimit) { 1216 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork(); 1217 if (element != null) { 1218 int bufferID = element.first; 1219 MediaCodec.BufferInfo info = element.second; 1220 if (info != null) { 1221 // <id, info> corresponds to output callback. Handle it accordingly 1222 dequeueOutput(bufferID, info); 1223 } else { 1224 // <id, null> corresponds to input callback. Handle it accordingly 1225 enqueueInput(bufferID); 1226 frameCount++; 1227 } 1228 } 1229 } 1230 } else { 1231 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 1232 // dequeue output after inputEOS is expected to be done in waitForAllOutputs() 1233 while (!mSawInputEOS && frameCount < frameLimit) { 1234 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US); 1235 if (outputBufferId >= 0) { 1236 dequeueOutput(outputBufferId, outInfo); 1237 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1238 mOutFormat = mCodec.getOutputFormat(); 1239 mSignalledOutFormatChanged = true; 1240 } 1241 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US); 1242 if (inputBufferId != -1) { 1243 enqueueInput(inputBufferId); 1244 frameCount++; 1245 } 1246 } 1247 } 1248 } 1249 queueEOS()1250 void queueEOS() throws InterruptedException { 1251 if (mIsCodecInAsyncMode) { 1252 while (!mAsyncHandle.hasSeenError() && !mSawInputEOS) { 1253 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork(); 1254 if (element != null) { 1255 int bufferID = element.first; 1256 MediaCodec.BufferInfo info = element.second; 1257 if (info != null) { 1258 dequeueOutput(bufferID, info); 1259 } else { 1260 enqueueEOS(element.first); 1261 } 1262 } 1263 } 1264 } else { 1265 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 1266 while (!mSawInputEOS) { 1267 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US); 1268 if (outputBufferId >= 0) { 1269 dequeueOutput(outputBufferId, outInfo); 1270 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1271 mOutFormat = mCodec.getOutputFormat(); 1272 mSignalledOutFormatChanged = true; 1273 } 1274 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US); 1275 if (inputBufferId != -1) { 1276 enqueueEOS(inputBufferId); 1277 } 1278 } 1279 } 1280 } 1281 waitForAllOutputs()1282 void waitForAllOutputs() throws InterruptedException { 1283 if (mIsCodecInAsyncMode) { 1284 while (!mAsyncHandle.hasSeenError() && !mSawOutputEOS) { 1285 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getOutput(); 1286 if (element != null) { 1287 dequeueOutput(element.first, element.second); 1288 } 1289 } 1290 } else { 1291 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 1292 while (!mSawOutputEOS) { 1293 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US); 1294 if (outputBufferId >= 0) { 1295 dequeueOutput(outputBufferId, outInfo); 1296 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1297 mOutFormat = mCodec.getOutputFormat(); 1298 mSignalledOutFormatChanged = true; 1299 } 1300 } 1301 } 1302 } 1303 selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)1304 static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats, 1305 String[] features, boolean isEncoder) { 1306 MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 1307 MediaCodecInfo[] codecInfos = codecList.getCodecInfos(); 1308 ArrayList<String> listOfCodecs = new ArrayList<>(); 1309 for (MediaCodecInfo codecInfo : codecInfos) { 1310 if (codecInfo.isEncoder() != isEncoder) continue; 1311 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue; 1312 String[] types = codecInfo.getSupportedTypes(); 1313 for (String type : types) { 1314 if (type.equalsIgnoreCase(mime)) { 1315 boolean isOk = true; 1316 MediaCodecInfo.CodecCapabilities codecCapabilities = 1317 codecInfo.getCapabilitiesForType(type); 1318 if (formats != null) { 1319 for (MediaFormat format : formats) { 1320 if (!codecCapabilities.isFormatSupported(format)) { 1321 isOk = false; 1322 break; 1323 } 1324 } 1325 } 1326 if (features != null) { 1327 for (String feature : features) { 1328 if (!codecCapabilities.isFeatureSupported(feature)) { 1329 isOk = false; 1330 break; 1331 } 1332 } 1333 } 1334 if (isOk) listOfCodecs.add(codecInfo.getName()); 1335 } 1336 } 1337 } 1338 return listOfCodecs; 1339 } 1340 getWidth(MediaFormat format)1341 static int getWidth(MediaFormat format) { 1342 int width = format.getInteger(MediaFormat.KEY_WIDTH, -1); 1343 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 1344 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 1345 } 1346 return width; 1347 } 1348 getHeight(MediaFormat format)1349 static int getHeight(MediaFormat format) { 1350 int height = format.getInteger(MediaFormat.KEY_HEIGHT, -1); 1351 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 1352 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 1353 } 1354 return height; 1355 } 1356 loadByteArrayFromString(final String str)1357 byte[] loadByteArrayFromString(final String str) { 1358 if (str == null) { 1359 return null; 1360 } 1361 Pattern pattern = Pattern.compile("[0-9a-fA-F]{2}"); 1362 Matcher matcher = pattern.matcher(str); 1363 // allocate a large enough byte array first 1364 byte[] tempArray = new byte[str.length() / 2]; 1365 int i = 0; 1366 while (matcher.find()) { 1367 tempArray[i++] = (byte) Integer.parseInt(matcher.group(), 16); 1368 } 1369 return Arrays.copyOfRange(tempArray, 0, i); 1370 } 1371 isFormatSimilar(MediaFormat inpFormat, MediaFormat outFormat)1372 boolean isFormatSimilar(MediaFormat inpFormat, MediaFormat outFormat) { 1373 if (inpFormat == null || outFormat == null) return false; 1374 String inpMime = inpFormat.getString(MediaFormat.KEY_MIME); 1375 String outMime = outFormat.getString(MediaFormat.KEY_MIME); 1376 // not comparing input and output mimes because for a codec, mime is raw on one side and 1377 // encoded type on the other 1378 if (outMime.startsWith("audio/")) { 1379 return inpFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT, -1) == 1380 outFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT, -2) && 1381 inpFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE, -1) == 1382 outFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE, -2) && 1383 inpMime.startsWith("audio/"); 1384 } else if (outMime.startsWith("video/")) { 1385 return getWidth(inpFormat) == getWidth(outFormat) && 1386 getHeight(inpFormat) == getHeight(outFormat) && inpMime.startsWith("video/"); 1387 } 1388 return true; 1389 } 1390 validateMetrics(String codec)1391 PersistableBundle validateMetrics(String codec) { 1392 PersistableBundle metrics = mCodec.getMetrics(); 1393 assertTrue("metrics is null", metrics != null); 1394 assertTrue(metrics.getString(MediaCodec.MetricsConstants.CODEC).equals(codec)); 1395 if (mIsAudio) { 1396 assertTrue(metrics.getString(MediaCodec.MetricsConstants.MODE) 1397 .equals(MediaCodec.MetricsConstants.MODE_AUDIO)); 1398 } else { 1399 assertTrue(metrics.getString(MediaCodec.MetricsConstants.MODE) 1400 .equals(MediaCodec.MetricsConstants.MODE_VIDEO)); 1401 } 1402 return metrics; 1403 } 1404 validateMetrics(String codec, MediaFormat format)1405 PersistableBundle validateMetrics(String codec, MediaFormat format) { 1406 PersistableBundle metrics = validateMetrics(codec); 1407 if (!mIsAudio) { 1408 assertTrue(metrics.getInt(MediaCodec.MetricsConstants.WIDTH) == getWidth(format)); 1409 assertTrue(metrics.getInt(MediaCodec.MetricsConstants.HEIGHT) == getHeight(format)); 1410 } 1411 assertTrue(metrics.getInt(MediaCodec.MetricsConstants.SECURE) == 0); 1412 return metrics; 1413 } 1414 validateColorAspects(MediaFormat fmt, int range, int standard, int transfer)1415 void validateColorAspects(MediaFormat fmt, int range, int standard, int transfer) { 1416 int colorRange = fmt.getInteger(MediaFormat.KEY_COLOR_RANGE, UNSPECIFIED); 1417 int colorStandard = fmt.getInteger(MediaFormat.KEY_COLOR_STANDARD, UNSPECIFIED); 1418 int colorTransfer = fmt.getInteger(MediaFormat.KEY_COLOR_TRANSFER, UNSPECIFIED); 1419 if (range > UNSPECIFIED) { 1420 assertEquals("color range mismatch ", range, colorRange); 1421 } 1422 if (standard > UNSPECIFIED) { 1423 assertEquals("color standard mismatch ", standard, colorStandard); 1424 } 1425 if (transfer > UNSPECIFIED) { 1426 assertEquals("color transfer mismatch ", transfer, colorTransfer); 1427 } 1428 } 1429 validateHDRStaticMetaData(MediaFormat fmt, ByteBuffer hdrStaticRef)1430 void validateHDRStaticMetaData(MediaFormat fmt, ByteBuffer hdrStaticRef) { 1431 ByteBuffer hdrStaticInfo = fmt.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO, null); 1432 assertNotNull("No HDR static metadata present in format : " + fmt, hdrStaticInfo); 1433 if (!hdrStaticRef.equals(hdrStaticInfo)) { 1434 StringBuilder refString = new StringBuilder(""); 1435 StringBuilder testString = new StringBuilder(""); 1436 byte[] ref = new byte[hdrStaticRef.capacity()]; 1437 hdrStaticRef.get(ref); 1438 byte[] test = new byte[hdrStaticInfo.capacity()]; 1439 hdrStaticInfo.get(test); 1440 for (int i = 0; i < Math.min(ref.length, test.length); i++) { 1441 refString.append(String.format("%2x ", ref[i])); 1442 testString.append(String.format("%2x ", test[i])); 1443 } 1444 fail("hdr static info mismatch" + "\n" + "ref static info : " + refString + "\n" + 1445 "test static info : " + testString); 1446 } 1447 } 1448 validateHDRDynamicMetaData(MediaFormat fmt, ByteBuffer hdrDynamicRef)1449 void validateHDRDynamicMetaData(MediaFormat fmt, ByteBuffer hdrDynamicRef) { 1450 ByteBuffer hdrDynamicInfo = fmt.getByteBuffer(MediaFormat.KEY_HDR10_PLUS_INFO, null); 1451 assertNotNull("No HDR dynamic metadata present in format : " + fmt, hdrDynamicInfo); 1452 if (!hdrDynamicRef.equals(hdrDynamicInfo)) { 1453 StringBuilder refString = new StringBuilder(""); 1454 StringBuilder testString = new StringBuilder(""); 1455 byte[] ref = new byte[hdrDynamicRef.capacity()]; 1456 hdrDynamicRef.get(ref); 1457 byte[] test = new byte[hdrDynamicInfo.capacity()]; 1458 hdrDynamicInfo.get(test); 1459 for (int i = 0; i < Math.min(ref.length, test.length); i++) { 1460 refString.append(String.format("%2x ", ref[i])); 1461 testString.append(String.format("%2x ", test[i])); 1462 } 1463 fail("hdr dynamic info mismatch" + "\n" + "ref dynamic info : " + refString + "\n" + 1464 "test dynamic info : " + testString); 1465 } 1466 } 1467 setUpSurface(CodecTestActivity activity)1468 public void setUpSurface(CodecTestActivity activity) throws InterruptedException { 1469 activity.waitTillSurfaceIsCreated(); 1470 mSurface = activity.getSurface(); 1471 assertTrue("Surface created is null.", mSurface != null); 1472 assertTrue("Surface created is invalid.", mSurface.isValid()); 1473 } 1474 tearDownSurface()1475 public void tearDownSurface() { 1476 if (mSurface != null) { 1477 mSurface.release(); 1478 mSurface = null; 1479 } 1480 } 1481 1482 @Before isCodecNameValid()1483 public void isCodecNameValid() { 1484 if (mCodecName != null && mCodecName.startsWith(INVALID_CODEC)) { 1485 fail("no valid component available for current test "); 1486 } 1487 } 1488 1489 @After tearDown()1490 public void tearDown() { 1491 if (mCodec != null) { 1492 mCodec.release(); 1493 mCodec = null; 1494 } 1495 } 1496 } 1497 1498 class CodecDecoderTestBase extends CodecTestBase { 1499 private static final String LOG_TAG = CodecDecoderTestBase.class.getSimpleName(); 1500 1501 String mMime; 1502 String mTestFile; 1503 boolean mIsInterlaced; 1504 boolean mSkipChecksumVerification; 1505 1506 ArrayList<ByteBuffer> mCsdBuffers; 1507 private int mCurrCsdIdx; 1508 1509 private ByteBuffer flatBuffer = ByteBuffer.allocate(4 * Integer.BYTES); 1510 1511 MediaExtractor mExtractor; 1512 CodecTestActivity mActivity; 1513 CodecDecoderTestBase(String codecName, String mime, String testFile)1514 CodecDecoderTestBase(String codecName, String mime, String testFile) { 1515 mCodecName = codecName; 1516 mMime = mime; 1517 mTestFile = testFile; 1518 mAsyncHandle = new CodecAsyncHandler(); 1519 mCsdBuffers = new ArrayList<>(); 1520 mIsAudio = mMime.startsWith("audio/"); 1521 } 1522 setUpSource(String srcFile)1523 MediaFormat setUpSource(String srcFile) throws IOException { 1524 return setUpSource(mInpPrefix, srcFile); 1525 } 1526 setUpSource(String prefix, String srcFile)1527 MediaFormat setUpSource(String prefix, String srcFile) throws IOException { 1528 Preconditions.assertTestFileExists(prefix + srcFile); 1529 mExtractor = new MediaExtractor(); 1530 Preconditions.assertTestFileExists(prefix + srcFile); 1531 mExtractor.setDataSource(prefix + srcFile); 1532 for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) { 1533 MediaFormat format = mExtractor.getTrackFormat(trackID); 1534 if (mMime.equalsIgnoreCase(format.getString(MediaFormat.KEY_MIME))) { 1535 mExtractor.selectTrack(trackID); 1536 if (mIsAudio) { 1537 // as per cdd, pcm/wave must support PCM_{8, 16, 24, 32, float} and flac must 1538 // support PCM_{16, float}. For raw media type let extractor manage the 1539 // encoding type directly. For flac, basing on bits-per-sample select the type 1540 if (mMime.equals(MediaFormat.MIMETYPE_AUDIO_FLAC)) { 1541 if (format.getInteger("bits-per-sample", 16) > 16) { 1542 format.setInteger(MediaFormat.KEY_PCM_ENCODING, 1543 AudioFormat.ENCODING_PCM_FLOAT); 1544 } 1545 } 1546 } else { 1547 ArrayList<MediaFormat> formatList = new ArrayList<>(); 1548 formatList.add(format); 1549 boolean selectHBD = doesAnyFormatHaveHDRProfile(mMime, formatList) || 1550 srcFile.contains("10bit"); 1551 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 1552 getColorFormat(mCodecName, mMime, mSurface != null, selectHBD)); 1553 if (selectHBD && (format.getInteger(MediaFormat.KEY_COLOR_FORMAT) != 1554 COLOR_FormatYUVP010)) { 1555 mSkipChecksumVerification = true; 1556 } 1557 1558 if ((format.getInteger(MediaFormat.KEY_COLOR_FORMAT) != COLOR_FormatYUVP010) 1559 && selectHBD && mSurface == null) { 1560 // Codecs that do not advertise P010 on devices with VNDK version < T, do 1561 // not support decoding high bit depth clips when color format is set to 1562 // COLOR_FormatYUV420Flexible in byte buffer mode. Since byte buffer mode 1563 // for high bit depth decoding wasn't tested prior to Android T, skip this 1564 // when device is older 1565 assumeTrue("Skipping High Bit Depth tests on VNDK < T", VNDK_IS_AT_LEAST_T); 1566 } 1567 } 1568 // TODO: determine this from the extractor format when it becomes exposed. 1569 mIsInterlaced = srcFile.contains("_interlaced_"); 1570 return format; 1571 } 1572 } 1573 fail("No track with mime: " + mMime + " found in file: " + srcFile); 1574 return null; 1575 } 1576 getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode)1577 int getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode) 1578 throws IOException { 1579 if (surfaceMode) return COLOR_FormatSurface; 1580 if (hbdMode) { 1581 MediaCodec codec = MediaCodec.createByCodecName(name); 1582 MediaCodecInfo.CodecCapabilities cap = 1583 codec.getCodecInfo().getCapabilitiesForType(mediaType); 1584 codec.release(); 1585 for (int c : cap.colorFormats) { 1586 if (c == COLOR_FormatYUVP010) { 1587 return c; 1588 } 1589 } 1590 } 1591 return COLOR_FormatYUV420Flexible; 1592 } 1593 hasCSD(MediaFormat format)1594 boolean hasCSD(MediaFormat format) { 1595 return format.containsKey("csd-0"); 1596 } 1597 flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio)1598 void flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio) { 1599 if (isAudio) { 1600 flatBuffer.putInt(info.size); 1601 } 1602 flatBuffer.putInt(info.flags & ~MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1603 .putLong(info.presentationTimeUs); 1604 flatBuffer.flip(); 1605 } 1606 enqueueCodecConfig(int bufferIndex)1607 void enqueueCodecConfig(int bufferIndex) { 1608 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 1609 ByteBuffer csdBuffer = mCsdBuffers.get(mCurrCsdIdx); 1610 inputBuffer.put((ByteBuffer) csdBuffer.rewind()); 1611 mCodec.queueInputBuffer(bufferIndex, 0, csdBuffer.limit(), 0, 1612 MediaCodec.BUFFER_FLAG_CODEC_CONFIG); 1613 if (ENABLE_LOGS) { 1614 Log.v(LOG_TAG, "queued csd: id: " + bufferIndex + " size: " + csdBuffer.limit()); 1615 } 1616 } 1617 enqueueInput(int bufferIndex)1618 void enqueueInput(int bufferIndex) { 1619 if (mExtractor.getSampleSize() < 0) { 1620 enqueueEOS(bufferIndex); 1621 } else { 1622 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 1623 mExtractor.readSampleData(inputBuffer, 0); 1624 int size = (int) mExtractor.getSampleSize(); 1625 long pts = mExtractor.getSampleTime(); 1626 int extractorFlags = mExtractor.getSampleFlags(); 1627 int codecFlags = 0; 1628 if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { 1629 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME; 1630 } 1631 if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) { 1632 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME; 1633 } 1634 if (!mExtractor.advance() && mSignalEOSWithLastFrame) { 1635 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM; 1636 mSawInputEOS = true; 1637 } 1638 if (ENABLE_LOGS) { 1639 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts + 1640 " flags: " + codecFlags); 1641 } 1642 mCodec.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags); 1643 if (size > 0 && (codecFlags & (MediaCodec.BUFFER_FLAG_CODEC_CONFIG | 1644 MediaCodec.BUFFER_FLAG_PARTIAL_FRAME)) == 0) { 1645 mOutputBuff.saveInPTS(pts); 1646 mInputCount++; 1647 } 1648 } 1649 } 1650 enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info)1651 void enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info) { 1652 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 1653 buffer.position(info.offset); 1654 for (int i = 0; i < info.size; i++) { 1655 inputBuffer.put(buffer.get()); 1656 } 1657 if (ENABLE_LOGS) { 1658 Log.v(LOG_TAG, "input: id: " + bufferIndex + " flags: " + info.flags + " size: " + 1659 info.size + " timestamp: " + info.presentationTimeUs); 1660 } 1661 mCodec.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs, 1662 info.flags); 1663 if (info.size > 0 && ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) && 1664 ((info.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) == 0)) { 1665 mOutputBuff.saveInPTS(info.presentationTimeUs); 1666 mInputCount++; 1667 } 1668 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1669 mSawInputEOS = true; 1670 } 1671 } 1672 dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)1673 void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) { 1674 if (info.size > 0 && mSaveToMem) { 1675 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex); 1676 flattenBufferInfo(info, mIsAudio); 1677 mOutputBuff.checksum(flatBuffer, flatBuffer.limit()); 1678 if (mIsAudio) { 1679 mOutputBuff.checksum(buf, info.size); 1680 mOutputBuff.saveToMemory(buf, info); 1681 } else { 1682 // tests both getOutputImage and getOutputBuffer. Can do time division 1683 // multiplexing but lets allow it for now 1684 Image img = mCodec.getOutputImage(bufferIndex); 1685 assertTrue(img != null); 1686 mOutputBuff.checksum(img); 1687 int imgFormat = img.getFormat(); 1688 int bytesPerSample = (ImageFormat.getBitsPerPixel(imgFormat) * 2) / (8 * 3); 1689 1690 MediaFormat format = mCodec.getOutputFormat(); 1691 buf = mCodec.getOutputBuffer(bufferIndex); 1692 int width = format.getInteger(MediaFormat.KEY_WIDTH); 1693 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 1694 int stride = format.getInteger(MediaFormat.KEY_STRIDE); 1695 mOutputBuff.checksum(buf, info.size, width, height, stride, bytesPerSample); 1696 1697 if (mTestDynamicMetadata) { 1698 validateHDRDynamicMetaData(mCodec.getOutputFormat(), ByteBuffer 1699 .wrap(loadByteArrayFromString(HDR_DYNAMIC_INFO[mOutputCount]))); 1700 1701 } 1702 } 1703 } 1704 1705 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1706 mSawOutputEOS = true; 1707 } 1708 if (ENABLE_LOGS) { 1709 Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " + 1710 info.size + " timestamp: " + info.presentationTimeUs); 1711 } 1712 if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 1713 mOutputBuff.saveOutPTS(info.presentationTimeUs); 1714 mOutputCount++; 1715 } 1716 mCodec.releaseOutputBuffer(bufferIndex, false); 1717 } 1718 doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)1719 void doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list) 1720 throws InterruptedException { 1721 int frameCount = 0; 1722 if (mIsCodecInAsyncMode) { 1723 // output processing after queuing EOS is done in waitForAllOutputs() 1724 while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < list.size()) { 1725 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork(); 1726 if (element != null) { 1727 int bufferID = element.first; 1728 MediaCodec.BufferInfo info = element.second; 1729 if (info != null) { 1730 dequeueOutput(bufferID, info); 1731 } else { 1732 enqueueInput(bufferID, buffer, list.get(frameCount)); 1733 frameCount++; 1734 } 1735 } 1736 } 1737 } else { 1738 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo(); 1739 // output processing after queuing EOS is done in waitForAllOutputs() 1740 while (!mSawInputEOS && frameCount < list.size()) { 1741 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US); 1742 if (outputBufferId >= 0) { 1743 dequeueOutput(outputBufferId, outInfo); 1744 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1745 mOutFormat = mCodec.getOutputFormat(); 1746 mSignalledOutFormatChanged = true; 1747 } 1748 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US); 1749 if (inputBufferId != -1) { 1750 enqueueInput(inputBufferId, buffer, list.get(frameCount)); 1751 frameCount++; 1752 } 1753 } 1754 } 1755 } 1756 queueCodecConfig()1757 void queueCodecConfig() throws InterruptedException { 1758 if (mIsCodecInAsyncMode) { 1759 for (mCurrCsdIdx = 0; !mAsyncHandle.hasSeenError() && mCurrCsdIdx < mCsdBuffers.size(); 1760 mCurrCsdIdx++) { 1761 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getInput(); 1762 if (element != null) { 1763 enqueueCodecConfig(element.first); 1764 } 1765 } 1766 } else { 1767 for (mCurrCsdIdx = 0; mCurrCsdIdx < mCsdBuffers.size(); mCurrCsdIdx++) { 1768 enqueueCodecConfig(mCodec.dequeueInputBuffer(-1)); 1769 } 1770 } 1771 } 1772 decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)1773 void decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit) 1774 throws IOException, InterruptedException { 1775 mSaveToMem = true; 1776 mOutputBuff = new OutputManager(); 1777 mCodec = MediaCodec.createByCodecName(decoder); 1778 MediaFormat format = setUpSource(file); 1779 configureCodec(format, false, true, false); 1780 mCodec.start(); 1781 mExtractor.seekTo(pts, mode); 1782 doWork(frameLimit); 1783 queueEOS(); 1784 waitForAllOutputs(); 1785 mCodec.stop(); 1786 mCodec.release(); 1787 mExtractor.release(); 1788 mSaveToMem = false; 1789 } 1790 1791 @Override validateMetrics(String decoder, MediaFormat format)1792 PersistableBundle validateMetrics(String decoder, MediaFormat format) { 1793 PersistableBundle metrics = super.validateMetrics(decoder, format); 1794 assertTrue(metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE).equals(mMime)); 1795 assertTrue(metrics.getInt(MediaCodec.MetricsConstants.ENCODER) == 0); 1796 return metrics; 1797 } 1798 validateColorAspects(String decoder, String parent, String name, int range, int standard, int transfer, boolean ignoreColorBox)1799 void validateColorAspects(String decoder, String parent, String name, int range, int standard, 1800 int transfer, boolean ignoreColorBox) 1801 throws IOException, InterruptedException { 1802 mOutputBuff = new OutputManager(); 1803 MediaFormat format = setUpSource(parent, name); 1804 if (ignoreColorBox) { 1805 format.removeKey(MediaFormat.KEY_COLOR_RANGE); 1806 format.removeKey(MediaFormat.KEY_COLOR_STANDARD); 1807 format.removeKey(MediaFormat.KEY_COLOR_TRANSFER); 1808 } 1809 if (decoder == null) { 1810 MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 1811 decoder = codecList.findDecoderForFormat(format); 1812 } 1813 mCodec = MediaCodec.createByCodecName(decoder); 1814 configureCodec(format, true, true, false); 1815 mCodec.start(); 1816 doWork(1); 1817 queueEOS(); 1818 waitForAllOutputs(); 1819 validateColorAspects(mCodec.getOutputFormat(), range, standard, transfer); 1820 mCodec.stop(); 1821 mCodec.release(); 1822 mExtractor.release(); 1823 } 1824 validateHDRStaticMetaData(String parent, String name, ByteBuffer HDRStatic, boolean ignoreContainerStaticInfo)1825 void validateHDRStaticMetaData(String parent, String name, ByteBuffer HDRStatic, 1826 boolean ignoreContainerStaticInfo) 1827 throws IOException, InterruptedException { 1828 mOutputBuff = new OutputManager(); 1829 MediaFormat format = setUpSource(parent, name); 1830 if (ignoreContainerStaticInfo) { 1831 format.removeKey(MediaFormat.KEY_HDR_STATIC_INFO); 1832 } 1833 mCodec = MediaCodec.createByCodecName(mCodecName); 1834 configureCodec(format, true, true, false); 1835 mCodec.start(); 1836 doWork(10); 1837 queueEOS(); 1838 waitForAllOutputs(); 1839 validateHDRStaticMetaData(mCodec.getOutputFormat(), HDRStatic); 1840 mCodec.stop(); 1841 mCodec.release(); 1842 mExtractor.release(); 1843 } 1844 validateHDRDynamicMetaData(String parent, String name, boolean ignoreContainerDynamicInfo)1845 void validateHDRDynamicMetaData(String parent, String name, boolean ignoreContainerDynamicInfo) 1846 throws IOException, InterruptedException { 1847 mOutputBuff = new OutputManager(); 1848 MediaFormat format = setUpSource(parent, name); 1849 if (ignoreContainerDynamicInfo) { 1850 format.removeKey(MediaFormat.KEY_HDR10_PLUS_INFO); 1851 } 1852 mCodec = MediaCodec.createByCodecName(mCodecName); 1853 configureCodec(format, true, true, false); 1854 mCodec.start(); 1855 doWork(10); 1856 queueEOS(); 1857 waitForAllOutputs(); 1858 mCodec.stop(); 1859 mCodec.release(); 1860 mExtractor.release(); 1861 } 1862 } 1863 1864 class CodecEncoderTestBase extends CodecTestBase { 1865 private static final String LOG_TAG = CodecEncoderTestBase.class.getSimpleName(); 1866 1867 // files are in WorkDir.getMediaDirString(); 1868 private static final String INPUT_AUDIO_FILE = "bbb_2ch_44kHz_s16le.raw"; 1869 private static final String INPUT_VIDEO_FILE = "bbb_cif_yuv420p_30fps.yuv"; 1870 protected static final String INPUT_AUDIO_FILE_HBD = "audio/sd_2ch_48kHz_f32le.raw"; 1871 protected static final String INPUT_VIDEO_FILE_HBD = "cosmat_cif_24fps_yuv420p16le.yuv"; 1872 1873 private final int INP_FRM_WIDTH = 352; 1874 private final int INP_FRM_HEIGHT = 288; 1875 1876 final String mMime; 1877 final int[] mBitrates; 1878 final int[] mEncParamList1; 1879 final int[] mEncParamList2; 1880 1881 final String mInputFile; 1882 byte[] mInputData; 1883 int mNumBytesSubmitted; 1884 long mInputOffsetPts; 1885 1886 ArrayList<MediaFormat> mFormats; 1887 ArrayList<MediaCodec.BufferInfo> mInfoList; 1888 1889 int mWidth, mHeight; 1890 int mFrameRate; 1891 int mMaxBFrames; 1892 int mChannels; 1893 int mSampleRate; 1894 int mBytesPerSample; 1895 CodecEncoderTestBase(String encoder, String mime, int[] bitrates, int[] encoderInfo1, int[] encoderInfo2)1896 CodecEncoderTestBase(String encoder, String mime, int[] bitrates, int[] encoderInfo1, 1897 int[] encoderInfo2) { 1898 mMime = mime; 1899 mCodecName = encoder; 1900 mBitrates = bitrates; 1901 mEncParamList1 = encoderInfo1; 1902 mEncParamList2 = encoderInfo2; 1903 mFormats = new ArrayList<>(); 1904 mInfoList = new ArrayList<>(); 1905 mWidth = INP_FRM_WIDTH; 1906 mHeight = INP_FRM_HEIGHT; 1907 if (mime.equals(MediaFormat.MIMETYPE_VIDEO_MPEG4)) mFrameRate = 12; 1908 else if (mime.equals(MediaFormat.MIMETYPE_VIDEO_H263)) mFrameRate = 12; 1909 else mFrameRate = 30; 1910 mMaxBFrames = 0; 1911 mChannels = 1; 1912 mSampleRate = 8000; 1913 mAsyncHandle = new CodecAsyncHandler(); 1914 mIsAudio = mMime.startsWith("audio/"); 1915 mBytesPerSample = mIsAudio ? 2 : 1; 1916 mInputFile = mIsAudio ? INPUT_AUDIO_FILE : INPUT_VIDEO_FILE; 1917 } 1918 1919 /** 1920 * Selects encoder input color format in byte buffer mode. As of now ndk tests support only 1921 * 420p, 420sp. COLOR_FormatYUV420Flexible although can represent any form of yuv, it doesn't 1922 * work in ndk due to lack of AMediaCodec_GetInputImage() 1923 */ findByteBufferColorFormat(String encoder, String mime)1924 static int findByteBufferColorFormat(String encoder, String mime) throws IOException { 1925 MediaCodec codec = MediaCodec.createByCodecName(encoder); 1926 MediaCodecInfo.CodecCapabilities cap = codec.getCodecInfo().getCapabilitiesForType(mime); 1927 int colorFormat = -1; 1928 for (int c : cap.colorFormats) { 1929 if (c == MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar || 1930 c == MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar) { 1931 Log.v(LOG_TAG, "selecting color format: " + c); 1932 colorFormat = c; 1933 break; 1934 } 1935 } 1936 codec.release(); 1937 return colorFormat; 1938 } 1939 1940 @Override resetContext(boolean isAsync, boolean signalEOSWithLastFrame)1941 void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) { 1942 super.resetContext(isAsync, signalEOSWithLastFrame); 1943 mNumBytesSubmitted = 0; 1944 mInputOffsetPts = 0; 1945 } 1946 1947 @Override flushCodec()1948 void flushCodec() { 1949 super.flushCodec(); 1950 if (mIsAudio) { 1951 mInputOffsetPts = (mNumBytesSubmitted + 1024) * 1000000L / 1952 (mBytesPerSample * mChannels * mSampleRate); 1953 } else { 1954 mInputOffsetPts = (mInputCount + 5) * 1000000L / mFrameRate; 1955 } 1956 mPrevOutputPts = mInputOffsetPts - 1; 1957 mNumBytesSubmitted = 0; 1958 } 1959 setUpSource(String srcFile)1960 void setUpSource(String srcFile) throws IOException { 1961 String inpPath = mInpPrefix + srcFile; 1962 try (FileInputStream fInp = new FileInputStream(inpPath)) { 1963 int size = (int) new File(inpPath).length(); 1964 mInputData = new byte[size]; 1965 fInp.read(mInputData, 0, size); 1966 } 1967 } 1968 fillImage(Image image)1969 void fillImage(Image image) { 1970 int format = image.getFormat(); 1971 assertTrue("unexpected image format", 1972 format == ImageFormat.YUV_420_888 || format == ImageFormat.YCBCR_P010); 1973 int bytesPerSample = (ImageFormat.getBitsPerPixel(format) * 2) / (8 * 3); // YUV420 1974 assertEquals("Invalid bytes per sample", bytesPerSample, mBytesPerSample); 1975 1976 int imageWidth = image.getWidth(); 1977 int imageHeight = image.getHeight(); 1978 Image.Plane[] planes = image.getPlanes(); 1979 int offset = mNumBytesSubmitted; 1980 for (int i = 0; i < planes.length; ++i) { 1981 ByteBuffer buf = planes[i].getBuffer(); 1982 int width = imageWidth; 1983 int height = imageHeight; 1984 int tileWidth = INP_FRM_WIDTH; 1985 int tileHeight = INP_FRM_HEIGHT; 1986 int rowStride = planes[i].getRowStride(); 1987 int pixelStride = planes[i].getPixelStride(); 1988 if (i != 0) { 1989 width = imageWidth / 2; 1990 height = imageHeight / 2; 1991 tileWidth = INP_FRM_WIDTH / 2; 1992 tileHeight = INP_FRM_HEIGHT / 2; 1993 } 1994 if (pixelStride == bytesPerSample) { 1995 if (width == rowStride && width == tileWidth && height == tileHeight) { 1996 buf.put(mInputData, offset, width * height * bytesPerSample); 1997 } else { 1998 for (int z = 0; z < height; z += tileHeight) { 1999 int rowsToCopy = Math.min(height - z, tileHeight); 2000 for (int y = 0; y < rowsToCopy; y++) { 2001 for (int x = 0; x < width; x += tileWidth) { 2002 int colsToCopy = Math.min(width - x, tileWidth); 2003 buf.position((z + y) * rowStride + x * bytesPerSample); 2004 buf.put(mInputData, offset + y * tileWidth * bytesPerSample, 2005 colsToCopy * bytesPerSample); 2006 } 2007 } 2008 } 2009 } 2010 } else { 2011 // do it pixel-by-pixel 2012 for (int z = 0; z < height; z += tileHeight) { 2013 int rowsToCopy = Math.min(height - z, tileHeight); 2014 for (int y = 0; y < rowsToCopy; y++) { 2015 int lineOffset = (z + y) * rowStride; 2016 for (int x = 0; x < width; x += tileWidth) { 2017 int colsToCopy = Math.min(width - x, tileWidth); 2018 for (int w = 0; w < colsToCopy; w++) { 2019 for (int bytePos = 0; bytePos < bytesPerSample; bytePos++) { 2020 buf.position(lineOffset + (x + w) * pixelStride + bytePos); 2021 buf.put(mInputData[offset + y * tileWidth * bytesPerSample + 2022 w * bytesPerSample + bytePos]); 2023 } 2024 } 2025 } 2026 } 2027 } 2028 } 2029 offset += tileWidth * tileHeight * bytesPerSample; 2030 } 2031 } 2032 fillByteBuffer(ByteBuffer inputBuffer)2033 void fillByteBuffer(ByteBuffer inputBuffer) { 2034 int offset = 0, frmOffset = mNumBytesSubmitted; 2035 for (int plane = 0; plane < 3; plane++) { 2036 int width = mWidth; 2037 int height = mHeight; 2038 int tileWidth = INP_FRM_WIDTH; 2039 int tileHeight = INP_FRM_HEIGHT; 2040 if (plane != 0) { 2041 width = mWidth / 2; 2042 height = mHeight / 2; 2043 tileWidth = INP_FRM_WIDTH / 2; 2044 tileHeight = INP_FRM_HEIGHT / 2; 2045 } 2046 for (int k = 0; k < height; k += tileHeight) { 2047 int rowsToCopy = Math.min(height - k, tileHeight); 2048 for (int j = 0; j < rowsToCopy; j++) { 2049 for (int i = 0; i < width; i += tileWidth) { 2050 int colsToCopy = Math.min(width - i, tileWidth); 2051 inputBuffer.position( 2052 offset + (k + j) * width * mBytesPerSample + i * mBytesPerSample); 2053 inputBuffer.put(mInputData, frmOffset + j * tileWidth * mBytesPerSample, 2054 colsToCopy * mBytesPerSample); 2055 } 2056 } 2057 } 2058 offset += width * height * mBytesPerSample; 2059 frmOffset += tileWidth * tileHeight * mBytesPerSample; 2060 } 2061 } 2062 enqueueInput(int bufferIndex)2063 void enqueueInput(int bufferIndex) { 2064 ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex); 2065 if (mNumBytesSubmitted >= mInputData.length) { 2066 enqueueEOS(bufferIndex); 2067 } else { 2068 int size; 2069 int flags = 0; 2070 long pts = mInputOffsetPts; 2071 if (mIsAudio) { 2072 pts += mNumBytesSubmitted * 1000000L / (mBytesPerSample * mChannels * mSampleRate); 2073 size = Math.min(inputBuffer.capacity(), mInputData.length - mNumBytesSubmitted); 2074 assertTrue(size % (mBytesPerSample * mChannels) == 0); 2075 inputBuffer.put(mInputData, mNumBytesSubmitted, size); 2076 if (mNumBytesSubmitted + size >= mInputData.length && mSignalEOSWithLastFrame) { 2077 flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM; 2078 mSawInputEOS = true; 2079 } 2080 mNumBytesSubmitted += size; 2081 } else { 2082 pts += mInputCount * 1000000L / mFrameRate; 2083 size = mBytesPerSample * mWidth * mHeight * 3 / 2; 2084 int frmSize = mBytesPerSample * INP_FRM_WIDTH * INP_FRM_HEIGHT * 3 / 2; 2085 if (mNumBytesSubmitted + frmSize > mInputData.length) { 2086 fail("received partial frame to encode"); 2087 } else { 2088 Image img = mCodec.getInputImage(bufferIndex); 2089 if (img != null) { 2090 fillImage(img); 2091 } else { 2092 if (mWidth == INP_FRM_WIDTH && mHeight == INP_FRM_HEIGHT) { 2093 inputBuffer.put(mInputData, mNumBytesSubmitted, size); 2094 } else { 2095 fillByteBuffer(inputBuffer); 2096 } 2097 } 2098 } 2099 if (mNumBytesSubmitted + frmSize >= mInputData.length && mSignalEOSWithLastFrame) { 2100 flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM; 2101 mSawInputEOS = true; 2102 } 2103 mNumBytesSubmitted += frmSize; 2104 } 2105 if (ENABLE_LOGS) { 2106 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts + 2107 " flags: " + flags); 2108 } 2109 mCodec.queueInputBuffer(bufferIndex, 0, size, pts, flags); 2110 mOutputBuff.saveInPTS(pts); 2111 mInputCount++; 2112 } 2113 } 2114 dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)2115 void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) { 2116 if (ENABLE_LOGS) { 2117 Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " + 2118 info.size + " timestamp: " + info.presentationTimeUs); 2119 } 2120 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 2121 mSawOutputEOS = true; 2122 } 2123 if (info.size > 0) { 2124 if (mSaveToMem) { 2125 MediaCodec.BufferInfo copy = new MediaCodec.BufferInfo(); 2126 copy.set(mOutputBuff.getOutStreamSize(), info.size, info.presentationTimeUs, 2127 info.flags); 2128 mInfoList.add(copy); 2129 2130 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex); 2131 mOutputBuff.saveToMemory(buf, info); 2132 } 2133 if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 2134 mOutputBuff.saveOutPTS(info.presentationTimeUs); 2135 mOutputCount++; 2136 } 2137 } 2138 mCodec.releaseOutputBuffer(bufferIndex, false); 2139 } 2140 2141 @Override validateMetrics(String codec, MediaFormat format)2142 PersistableBundle validateMetrics(String codec, MediaFormat format) { 2143 PersistableBundle metrics = super.validateMetrics(codec, format); 2144 assertTrue(metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE).equals(mMime)); 2145 assertTrue(metrics.getInt(MediaCodec.MetricsConstants.ENCODER) == 1); 2146 return metrics; 2147 } 2148 setUpParams(int limit)2149 void setUpParams(int limit) { 2150 int count = 0; 2151 for (int bitrate : mBitrates) { 2152 if (mIsAudio) { 2153 for (int rate : mEncParamList1) { 2154 for (int channels : mEncParamList2) { 2155 MediaFormat format = new MediaFormat(); 2156 format.setString(MediaFormat.KEY_MIME, mMime); 2157 if (mMime.equals(MediaFormat.MIMETYPE_AUDIO_FLAC)) { 2158 format.setInteger(MediaFormat.KEY_FLAC_COMPRESSION_LEVEL, bitrate); 2159 } else { 2160 format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); 2161 } 2162 format.setInteger(MediaFormat.KEY_SAMPLE_RATE, rate); 2163 format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, channels); 2164 mFormats.add(format); 2165 count++; 2166 if (count >= limit) return; 2167 } 2168 } 2169 } else { 2170 assertTrue("Wrong number of height, width parameters", 2171 mEncParamList1.length == mEncParamList2.length); 2172 for (int i = 0; i < mEncParamList1.length; i++) { 2173 MediaFormat format = new MediaFormat(); 2174 format.setString(MediaFormat.KEY_MIME, mMime); 2175 format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); 2176 format.setInteger(MediaFormat.KEY_WIDTH, mEncParamList1[i]); 2177 format.setInteger(MediaFormat.KEY_HEIGHT, mEncParamList2[i]); 2178 format.setInteger(MediaFormat.KEY_FRAME_RATE, mFrameRate); 2179 format.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames); 2180 format.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f); 2181 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 2182 MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); 2183 mFormats.add(format); 2184 count++; 2185 if (count >= limit) return; 2186 } 2187 } 2188 } 2189 } 2190 encodeToMemory(String file, String encoder, int frameLimit, MediaFormat format, boolean saveToMem)2191 void encodeToMemory(String file, String encoder, int frameLimit, MediaFormat format, 2192 boolean saveToMem) throws IOException, InterruptedException { 2193 mSaveToMem = saveToMem; 2194 mOutputBuff = new OutputManager(); 2195 mInfoList.clear(); 2196 mCodec = MediaCodec.createByCodecName(encoder); 2197 setUpSource(file); 2198 configureCodec(format, false, true, true); 2199 if (mIsAudio) { 2200 mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE); 2201 mChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 2202 } else { 2203 mWidth = format.getInteger(MediaFormat.KEY_WIDTH); 2204 mHeight = format.getInteger(MediaFormat.KEY_HEIGHT); 2205 } 2206 mCodec.start(); 2207 doWork(frameLimit); 2208 queueEOS(); 2209 waitForAllOutputs(); 2210 mCodec.stop(); 2211 mCodec.release(); 2212 mSaveToMem = false; 2213 } 2214 decodeElementaryStream(String decoder, MediaFormat format, ByteBuffer elementaryStream, ArrayList<MediaCodec.BufferInfo> infos)2215 ByteBuffer decodeElementaryStream(String decoder, MediaFormat format, 2216 ByteBuffer elementaryStream, ArrayList<MediaCodec.BufferInfo> infos) 2217 throws IOException, InterruptedException { 2218 String mime = format.getString(MediaFormat.KEY_MIME); 2219 CodecDecoderTestBase cdtb = new CodecDecoderTestBase(decoder, mime, null); 2220 cdtb.mOutputBuff = new OutputManager(); 2221 cdtb.mSaveToMem = true; 2222 cdtb.mCodec = MediaCodec.createByCodecName(decoder); 2223 cdtb.mCodec.configure(format, null, null, 0); 2224 cdtb.mCodec.start(); 2225 cdtb.doWork(elementaryStream, infos); 2226 cdtb.queueEOS(); 2227 cdtb.waitForAllOutputs(); 2228 cdtb.mCodec.stop(); 2229 cdtb.mCodec.release(); 2230 return cdtb.mOutputBuff.getBuffer(); 2231 } 2232 } 2233