• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediav2.cts;
18 
19 import android.content.Context;
20 import android.content.pm.PackageManager;
21 import android.graphics.ImageFormat;
22 import android.graphics.Rect;
23 import android.hardware.display.DisplayManager;
24 import android.media.AudioFormat;
25 import android.media.Image;
26 import android.media.MediaCodec;
27 import android.media.MediaCodecInfo;
28 import android.media.MediaCodecInfo.CodecCapabilities;
29 import android.media.MediaCodecInfo.CodecProfileLevel;
30 import android.media.MediaCodecList;
31 import android.media.MediaExtractor;
32 import android.media.MediaFormat;
33 import android.os.Build;
34 import android.os.PersistableBundle;
35 import android.os.SystemProperties;
36 import android.util.Log;
37 import android.util.Pair;
38 import android.view.Display;
39 import android.view.Surface;
40 
41 import androidx.annotation.NonNull;
42 import androidx.test.platform.app.InstrumentationRegistry;
43 
44 import org.junit.After;
45 import org.junit.Assert;
46 import org.junit.Assume;
47 import org.junit.Before;
48 
49 import java.io.File;
50 import java.io.FileInputStream;
51 import java.io.IOException;
52 import java.nio.ByteBuffer;
53 import java.nio.ByteOrder;
54 import java.util.ArrayList;
55 import java.util.Arrays;
56 import java.util.Collections;
57 import java.util.HashMap;
58 import java.util.HashSet;
59 import java.util.LinkedList;
60 import java.util.List;
61 import java.util.Map;
62 import java.util.Set;
63 import java.util.concurrent.locks.Condition;
64 import java.util.concurrent.locks.Lock;
65 import java.util.concurrent.locks.ReentrantLock;
66 import java.util.regex.Matcher;
67 import java.util.regex.Pattern;
68 import java.util.stream.IntStream;
69 import java.util.zip.CRC32;
70 
71 import com.android.compatibility.common.util.ApiLevelUtil;
72 import com.android.compatibility.common.util.MediaUtils;
73 
74 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface;
75 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible;
76 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010;
77 import static android.media.MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing;
78 import static android.media.MediaCodecInfo.CodecProfileLevel.*;
79 import static org.junit.Assert.assertEquals;
80 import static org.junit.Assert.assertNotNull;
81 import static org.junit.Assert.assertTrue;
82 import static org.junit.Assert.fail;
83 import static org.junit.Assume.assumeTrue;
84 
85 class CodecAsyncHandler extends MediaCodec.Callback {
86     private static final String LOG_TAG = CodecAsyncHandler.class.getSimpleName();
87     private final Lock mLock = new ReentrantLock();
88     private final Condition mCondition = mLock.newCondition();
89     private final LinkedList<Pair<Integer, MediaCodec.BufferInfo>> mCbInputQueue;
90     private final LinkedList<Pair<Integer, MediaCodec.BufferInfo>> mCbOutputQueue;
91     private MediaFormat mOutFormat;
92     private boolean mSignalledOutFormatChanged;
93     private volatile boolean mSignalledError;
94 
CodecAsyncHandler()95     CodecAsyncHandler() {
96         mCbInputQueue = new LinkedList<>();
97         mCbOutputQueue = new LinkedList<>();
98         mSignalledError = false;
99         mSignalledOutFormatChanged = false;
100     }
101 
clearQueues()102     void clearQueues() {
103         mLock.lock();
104         mCbInputQueue.clear();
105         mCbOutputQueue.clear();
106         mLock.unlock();
107     }
108 
resetContext()109     void resetContext() {
110         clearQueues();
111         mOutFormat = null;
112         mSignalledOutFormatChanged = false;
113         mSignalledError = false;
114     }
115 
116     @Override
onInputBufferAvailable(@onNull MediaCodec codec, int bufferIndex)117     public void onInputBufferAvailable(@NonNull MediaCodec codec, int bufferIndex) {
118         assertTrue(bufferIndex >= 0);
119         mLock.lock();
120         mCbInputQueue.add(new Pair<>(bufferIndex, (MediaCodec.BufferInfo) null));
121         mCondition.signalAll();
122         mLock.unlock();
123     }
124 
125     @Override
onOutputBufferAvailable(@onNull MediaCodec codec, int bufferIndex, @NonNull MediaCodec.BufferInfo info)126     public void onOutputBufferAvailable(@NonNull MediaCodec codec, int bufferIndex,
127             @NonNull MediaCodec.BufferInfo info) {
128         assertTrue(bufferIndex >= 0);
129         mLock.lock();
130         mCbOutputQueue.add(new Pair<>(bufferIndex, info));
131         mCondition.signalAll();
132         mLock.unlock();
133     }
134 
135     @Override
onError(@onNull MediaCodec codec, MediaCodec.CodecException e)136     public void onError(@NonNull MediaCodec codec, MediaCodec.CodecException e) {
137         mLock.lock();
138         mSignalledError = true;
139         mCondition.signalAll();
140         mLock.unlock();
141         Log.e(LOG_TAG, "received media codec error : " + e.getMessage());
142     }
143 
144     @Override
onOutputFormatChanged(@onNull MediaCodec codec, @NonNull MediaFormat format)145     public void onOutputFormatChanged(@NonNull MediaCodec codec, @NonNull MediaFormat format) {
146         mOutFormat = format;
147         mSignalledOutFormatChanged = true;
148         Log.i(LOG_TAG, "Output format changed: " + format.toString());
149     }
150 
setCallBack(MediaCodec codec, boolean isCodecInAsyncMode)151     void setCallBack(MediaCodec codec, boolean isCodecInAsyncMode) {
152         if (isCodecInAsyncMode) {
153             codec.setCallback(this);
154         } else {
155             codec.setCallback(null);
156         }
157     }
158 
getInput()159     Pair<Integer, MediaCodec.BufferInfo> getInput() throws InterruptedException {
160         Pair<Integer, MediaCodec.BufferInfo> element = null;
161         mLock.lock();
162         while (!mSignalledError) {
163             if (mCbInputQueue.isEmpty()) {
164                 mCondition.await();
165             } else {
166                 element = mCbInputQueue.remove(0);
167                 break;
168             }
169         }
170         mLock.unlock();
171         return element;
172     }
173 
getOutput()174     Pair<Integer, MediaCodec.BufferInfo> getOutput() throws InterruptedException {
175         Pair<Integer, MediaCodec.BufferInfo> element = null;
176         mLock.lock();
177         while (!mSignalledError) {
178             if (mCbOutputQueue.isEmpty()) {
179                 mCondition.await();
180             } else {
181                 element = mCbOutputQueue.remove(0);
182                 break;
183             }
184         }
185         mLock.unlock();
186         return element;
187     }
188 
getWork()189     Pair<Integer, MediaCodec.BufferInfo> getWork() throws InterruptedException {
190         Pair<Integer, MediaCodec.BufferInfo> element = null;
191         mLock.lock();
192         while (!mSignalledError) {
193             if (mCbInputQueue.isEmpty() && mCbOutputQueue.isEmpty()) {
194                 mCondition.await();
195             } else {
196                 if (!mCbOutputQueue.isEmpty()) {
197                     element = mCbOutputQueue.remove(0);
198                     break;
199                 }
200                 if (!mCbInputQueue.isEmpty()) {
201                     element = mCbInputQueue.remove(0);
202                     break;
203                 }
204             }
205         }
206         mLock.unlock();
207         return element;
208     }
209 
isInputQueueEmpty()210     boolean isInputQueueEmpty() {
211         mLock.lock();
212         boolean isEmpty = mCbInputQueue.isEmpty();
213         mLock.unlock();
214         return isEmpty;
215     }
216 
hasSeenError()217     boolean hasSeenError() {
218         return mSignalledError;
219     }
220 
hasOutputFormatChanged()221     boolean hasOutputFormatChanged() {
222         return mSignalledOutFormatChanged;
223     }
224 
getOutputFormat()225     MediaFormat getOutputFormat() {
226         return mOutFormat;
227     }
228 }
229 
230 class OutputManager {
231     private static final String LOG_TAG = OutputManager.class.getSimpleName();
232     private byte[] memory;
233     private int memIndex;
234     private CRC32 mCrc32UsingImage;
235     private CRC32 mCrc32UsingBuffer;
236     private ArrayList<Long> inpPtsList;
237     private ArrayList<Long> outPtsList;
238 
OutputManager()239     OutputManager() {
240         memory = new byte[1024];
241         memIndex = 0;
242         mCrc32UsingImage = new CRC32();
243         mCrc32UsingBuffer = new CRC32();
244         inpPtsList = new ArrayList<>();
245         outPtsList = new ArrayList<>();
246     }
247 
saveInPTS(long pts)248     void saveInPTS(long pts) {
249         // Add only Unique timeStamp, discarding any duplicate frame / non-display frame
250         if (!inpPtsList.contains(pts)) {
251             inpPtsList.add(pts);
252         }
253     }
254 
saveOutPTS(long pts)255     void saveOutPTS(long pts) {
256         outPtsList.add(pts);
257     }
258 
isPtsStrictlyIncreasing(long lastPts)259     boolean isPtsStrictlyIncreasing(long lastPts) {
260         boolean res = true;
261         for (int i = 0; i < outPtsList.size(); i++) {
262             if (lastPts < outPtsList.get(i)) {
263                 lastPts = outPtsList.get(i);
264             } else {
265                 Log.e(LOG_TAG, "Timestamp ordering check failed: last timestamp: " + lastPts +
266                         " current timestamp:" + outPtsList.get(i));
267                 res = false;
268                 break;
269             }
270         }
271         return res;
272     }
273 
isOutPtsListIdenticalToInpPtsList(boolean requireSorting)274     boolean isOutPtsListIdenticalToInpPtsList(boolean requireSorting) {
275         boolean res;
276         Collections.sort(inpPtsList);
277         if (requireSorting) {
278             Collections.sort(outPtsList);
279         }
280         if (outPtsList.size() != inpPtsList.size()) {
281             Log.e(LOG_TAG, "input and output presentation timestamp list sizes are not identical" +
282                     "exp/rec" + inpPtsList.size() + '/' + outPtsList.size());
283             return false;
284         } else {
285             int count = 0;
286             for (int i = 0; i < outPtsList.size(); i++) {
287                 if (!outPtsList.get(i).equals(inpPtsList.get(i))) {
288                     count ++;
289                     Log.e(LOG_TAG, "input output pts mismatch, exp/rec " + outPtsList.get(i) + '/' +
290                             inpPtsList.get(i));
291                     if (count == 20) {
292                         Log.e(LOG_TAG, "stopping after 20 mismatches, ...");
293                         break;
294                     }
295                 }
296             }
297             res = (count == 0);
298         }
299         return res;
300     }
301 
getOutStreamSize()302     int getOutStreamSize() {
303         return memIndex;
304     }
305 
checksum(ByteBuffer buf, int size)306     void checksum(ByteBuffer buf, int size) {
307         checksum(buf, size, 0, 0, 0, 0);
308     }
309 
checksum(ByteBuffer buf, int size, int width, int height, int stride, int bytesPerSample)310     void checksum(ByteBuffer buf, int size, int width, int height, int stride, int bytesPerSample) {
311         int cap = buf.capacity();
312         assertTrue("checksum() params are invalid: size = " + size + " cap = " + cap,
313                 size > 0 && size <= cap);
314         if (buf.hasArray()) {
315             if (width > 0 && height > 0 && stride > 0 && bytesPerSample > 0) {
316                 int offset = buf.position() + buf.arrayOffset();
317                 byte[] bb = new byte[width * height * bytesPerSample];
318                 for (int i = 0; i < height; ++i) {
319                     System.arraycopy(buf.array(), offset, bb, i * width * bytesPerSample,
320                             width * bytesPerSample);
321                     offset += stride;
322                 }
323                 mCrc32UsingBuffer.update(bb, 0, width * height * bytesPerSample);
324             } else {
325                 mCrc32UsingBuffer.update(buf.array(), buf.position() + buf.arrayOffset(), size);
326             }
327         } else if (width > 0 && height > 0 && stride > 0 && bytesPerSample > 0) {
328             // Checksum only the Y plane
329             int pos = buf.position();
330             int offset = pos;
331             byte[] bb = new byte[width * height * bytesPerSample];
332             for (int i = 0; i < height; ++i) {
333                 buf.position(offset);
334                 buf.get(bb, i * width * bytesPerSample, width * bytesPerSample);
335                 offset += stride;
336             }
337             mCrc32UsingBuffer.update(bb, 0, width * height * bytesPerSample);
338             buf.position(pos);
339         } else {
340             int pos = buf.position();
341             final int rdsize = Math.min(4096, size);
342             byte[] bb = new byte[rdsize];
343             int chk;
344             for (int i = 0; i < size; i += chk) {
345                 chk = Math.min(rdsize, size - i);
346                 buf.get(bb, 0, chk);
347                 mCrc32UsingBuffer.update(bb, 0, chk);
348             }
349             buf.position(pos);
350         }
351     }
352 
checksum(Image image)353     void checksum(Image image) {
354         int format = image.getFormat();
355         assertTrue("unexpected image format",
356                 format == ImageFormat.YUV_420_888 || format == ImageFormat.YCBCR_P010);
357         int bytesPerSample = (ImageFormat.getBitsPerPixel(format) * 2) / (8 * 3);  // YUV420
358 
359         Rect cropRect = image.getCropRect();
360         int imageWidth = cropRect.width();
361         int imageHeight = cropRect.height();
362         assertTrue("unexpected image dimensions", imageWidth > 0 && imageHeight > 0);
363 
364         int imageLeft = cropRect.left;
365         int imageTop = cropRect.top;
366         Image.Plane[] planes = image.getPlanes();
367         for (int i = 0; i < planes.length; ++i) {
368             ByteBuffer buf = planes[i].getBuffer();
369             int width, height, rowStride, pixelStride, x, y, left, top;
370             rowStride = planes[i].getRowStride();
371             pixelStride = planes[i].getPixelStride();
372             if (i == 0) {
373                 assertEquals(bytesPerSample, pixelStride);
374                 width = imageWidth;
375                 height = imageHeight;
376                 left = imageLeft;
377                 top = imageTop;
378             } else {
379                 width = imageWidth / 2;
380                 height = imageHeight / 2;
381                 left = imageLeft / 2;
382                 top = imageTop / 2;
383             }
384             int cropOffset = (left * pixelStride) + top * rowStride;
385             // local contiguous pixel buffer
386             byte[] bb = new byte[width * height * bytesPerSample];
387 
388             if (buf.hasArray()) {
389                 byte[] b = buf.array();
390                 int offs = buf.arrayOffset() + cropOffset;
391                 if (pixelStride == bytesPerSample) {
392                     for (y = 0; y < height; ++y) {
393                         System.arraycopy(b, offs + y * rowStride, bb, y * width * bytesPerSample,
394                                 width * bytesPerSample);
395                     }
396                 } else {
397                     // do it pixel-by-pixel
398                     for (y = 0; y < height; ++y) {
399                         int lineOffset = offs + y * rowStride;
400                         for (x = 0; x < width; ++x) {
401                             for (int bytePos = 0; bytePos < bytesPerSample; ++bytePos) {
402                                 bb[y * width * bytesPerSample + x * bytesPerSample + bytePos] =
403                                         b[lineOffset + x * pixelStride + bytePos];
404                             }
405                         }
406                     }
407                 }
408             } else { // almost always ends up here due to direct buffers
409                 int base = buf.position();
410                 int pos = base + cropOffset;
411                 if (pixelStride == bytesPerSample) {
412                     for (y = 0; y < height; ++y) {
413                         buf.position(pos + y * rowStride);
414                         buf.get(bb, y * width * bytesPerSample, width * bytesPerSample);
415                     }
416                 } else {
417                     // local line buffer
418                     byte[] lb = new byte[rowStride];
419                     // do it pixel-by-pixel
420                     for (y = 0; y < height; ++y) {
421                         buf.position(pos + y * rowStride);
422                         // we're only guaranteed to have pixelStride * (width - 1) +
423                         // bytesPerSample bytes
424                         buf.get(lb, 0, pixelStride * (width - 1) + bytesPerSample);
425                         for (x = 0; x < width; ++x) {
426                             for (int bytePos = 0; bytePos < bytesPerSample; ++bytePos) {
427                                 bb[y * width * bytesPerSample + x * bytesPerSample + bytePos] =
428                                         lb[x * pixelStride + bytePos];
429                             }
430                         }
431                     }
432                 }
433                 buf.position(base);
434             }
435             mCrc32UsingImage.update(bb, 0, width * height * bytesPerSample);
436         }
437     }
438 
saveToMemory(ByteBuffer buf, MediaCodec.BufferInfo info)439     void saveToMemory(ByteBuffer buf, MediaCodec.BufferInfo info) {
440         if (memIndex + info.size >= memory.length) {
441             memory = Arrays.copyOf(memory, memIndex + info.size);
442         }
443         buf.position(info.offset);
444         buf.get(memory, memIndex, info.size);
445         memIndex += info.size;
446     }
447 
position(int index)448     void position(int index) {
449         if (index < 0 || index >= memory.length) index = 0;
450         memIndex = index;
451     }
452 
getBuffer()453     ByteBuffer getBuffer() {
454         return ByteBuffer.wrap(memory);
455     }
456 
reset()457     void reset() {
458         position(0);
459         mCrc32UsingImage.reset();
460         mCrc32UsingBuffer.reset();
461         inpPtsList.clear();
462         outPtsList.clear();
463     }
464 
getRmsError(Object refObject, int audioFormat)465     float getRmsError(Object refObject, int audioFormat) {
466         double totalErrorSquared = 0;
467         double avgErrorSquared;
468         int bytesPerSample = AudioFormat.getBytesPerSample(audioFormat);
469         if (refObject instanceof float[]) {
470             if (audioFormat != AudioFormat.ENCODING_PCM_FLOAT) return Float.MAX_VALUE;
471             float[] refData = (float[]) refObject;
472             if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE;
473             float[] floatData = new float[refData.length];
474             ByteBuffer.wrap(memory, 0, memIndex).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer()
475                     .get(floatData);
476             for (int i = 0; i < refData.length; i++) {
477                 float d = floatData[i] - refData[i];
478                 totalErrorSquared += d * d;
479             }
480             avgErrorSquared = (totalErrorSquared / refData.length);
481         } else if (refObject instanceof int[]) {
482             int[] refData = (int[]) refObject;
483             int[] intData;
484             if (audioFormat == AudioFormat.ENCODING_PCM_24BIT_PACKED) {
485                 if (refData.length != (memIndex / bytesPerSample)) return Float.MAX_VALUE;
486                 intData = new int[refData.length];
487                 for (int i = 0, j = 0; i < memIndex; i += 3, j++) {
488                     intData[j] =  memory[j] | (memory[j + 1] << 8) | (memory[j + 2] << 16);
489                 }
490             } else if (audioFormat == AudioFormat.ENCODING_PCM_32BIT) {
491                 if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE;
492                 intData = new int[refData.length];
493                 ByteBuffer.wrap(memory, 0, memIndex).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()
494                         .get(intData);
495             } else {
496                 return Float.MAX_VALUE;
497             }
498             for (int i = 0; i < intData.length; i++) {
499                 float d = intData[i] - refData[i];
500                 totalErrorSquared += d * d;
501             }
502             avgErrorSquared = (totalErrorSquared / refData.length);
503         } else if (refObject instanceof short[]) {
504             short[] refData = (short[]) refObject;
505             if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE;
506             if (audioFormat != AudioFormat.ENCODING_PCM_16BIT) return Float.MAX_VALUE;
507             short[] shortData = new short[refData.length];
508             ByteBuffer.wrap(memory, 0, memIndex).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer()
509                     .get(shortData);
510             for (int i = 0; i < shortData.length; i++) {
511                 float d = shortData[i] - refData[i];
512                 totalErrorSquared += d * d;
513             }
514             avgErrorSquared = (totalErrorSquared / refData.length);
515         } else if (refObject instanceof byte[]) {
516             byte[] refData = (byte[]) refObject;
517             if (refData.length != memIndex / bytesPerSample) return Float.MAX_VALUE;
518             if (audioFormat != AudioFormat.ENCODING_PCM_8BIT) return Float.MAX_VALUE;
519             byte[] byteData = new byte[refData.length];
520             ByteBuffer.wrap(memory, 0, memIndex).get(byteData);
521             for (int i = 0; i < byteData.length; i++) {
522                 float d = byteData[i] - refData[i];
523                 totalErrorSquared += d * d;
524             }
525             avgErrorSquared = (totalErrorSquared / refData.length);
526         } else {
527             return Float.MAX_VALUE;
528         }
529         return (float) Math.sqrt(avgErrorSquared);
530     }
531 
getCheckSumImage()532     long getCheckSumImage() {
533         return mCrc32UsingImage.getValue();
534     }
535 
getCheckSumBuffer()536     long getCheckSumBuffer() {
537         return mCrc32UsingBuffer.getValue();
538     }
539 
540     @Override
equals(Object o)541     public boolean equals(Object o) {
542         if (this == o) return true;
543         if (o == null || getClass() != o.getClass()) return false;
544         OutputManager that = (OutputManager) o;
545         // TODO: Timestamps for deinterlaced content are under review. (E.g. can decoders
546         // produce multiple progressive frames?) For now, do not verify timestamps.
547         boolean isEqual = this.equalsInterlaced(o);
548         if (!outPtsList.equals(that.outPtsList)) {
549             isEqual = false;
550             Log.e(LOG_TAG, "ref and test presentation timestamp mismatch");
551         }
552         return isEqual;
553     }
554 
equalsInterlaced(Object o)555     public boolean equalsInterlaced(Object o) {
556         if (this == o) return true;
557         if (o == null || getClass() != o.getClass()) return false;
558         OutputManager that = (OutputManager) o;
559         boolean isEqual = true;
560         if (mCrc32UsingImage.getValue() != that.mCrc32UsingImage.getValue()) {
561             isEqual = false;
562             Log.e(LOG_TAG, "ref and test crc32 checksums calculated using image mismatch " +
563                           mCrc32UsingImage.getValue() + '/' + that.mCrc32UsingImage.getValue());
564         }
565         if (mCrc32UsingBuffer.getValue() != that.mCrc32UsingBuffer.getValue()) {
566             isEqual = false;
567             Log.e(LOG_TAG, "ref and test crc32 checksums calculated using buffer mismatch " +
568                           mCrc32UsingBuffer.getValue() + '/' + that.mCrc32UsingBuffer.getValue());
569             if (memIndex == that.memIndex) {
570                 int count = 0;
571                 for (int i = 0; i < memIndex; i++) {
572                     if (memory[i] != that.memory[i]) {
573                         count++;
574                         if (count < 20) {
575                             Log.d(LOG_TAG, "sample at " + i + " exp/got:: " + memory[i] + '/' +
576                                     that.memory[i]);
577                         }
578                     }
579                 }
580                 if (count != 0) {
581                     Log.e(LOG_TAG, "ref and test o/p samples mismatch " + count);
582                 }
583             } else {
584                 Log.e(LOG_TAG, "ref and test o/p sizes mismatch " + memIndex + '/' + that.memIndex);
585             }
586         }
587         return isEqual;
588     }
589 }
590 
591 abstract class CodecTestBase {
592     public static final boolean IS_Q = ApiLevelUtil.getApiLevel() == Build.VERSION_CODES.Q;
593     public static final boolean IS_AT_LEAST_R = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.R);
594     // Checking for CODENAME helps in cases when build version on the development branch isn't
595     // updated yet but CODENAME is updated.
596     public static final boolean IS_AT_LEAST_T =
597             ApiLevelUtil.isAtLeast(Build.VERSION_CODES.TIRAMISU) ||
598                     ApiLevelUtil.codenameEquals("Tiramisu");
599     // TODO (b/223868241) Update the following to check for Build.VERSION_CODES.TIRAMISU once
600     // TIRAMISU is set correctly
601     public static final boolean FIRST_SDK_IS_AT_LEAST_T =
602             ApiLevelUtil.isFirstApiAfter(Build.VERSION_CODES.S_V2);
603     public static final boolean VNDK_IS_AT_LEAST_T =
604             SystemProperties.getInt("ro.vndk.version", 0) > Build.VERSION_CODES.S_V2;
605     public static final boolean BOARD_SDK_IS_AT_LEAST_T =
606             SystemProperties.getInt("ro.board.api_level", 0) > Build.VERSION_CODES.S_V2;
607     public static final boolean IS_HDR_EDITING_SUPPORTED = isHDREditingSupported();
608     private static final String LOG_TAG = CodecTestBase.class.getSimpleName();
609     enum SupportClass {
610         CODEC_ALL, // All codecs must support
611         CODEC_ANY, // At least one codec must support
612         CODEC_DEFAULT, // Default codec must support
613         CODEC_OPTIONAL // Codec support is optional
614     }
615     static final String HDR_STATIC_INFO =
616             "00 d0 84 80 3e c2 33 c4 86 4c 1d b8 0b 13 3d 42 40 e8 03 64 00 e8 03 2c 01";
617     static final String[] HDR_DYNAMIC_INFO = new String[]{
618             "b5 00 3c 00 01 04 00 40  00 0c 80 4e 20 27 10 00" +
619             "0a 00 00 24 08 00 00 28  00 00 50 00 28 c8 00 c9" +
620             "90 02 aa 58 05 ca d0 0c  0a f8 16 83 18 9c 18 00" +
621             "40 78 13 64 d5 7c 2e 2c  c3 59 de 79 6e c3 c2 00",
622 
623             "b5 00 3c 00 01 04 00 40  00 0c 80 4e 20 27 10 00" +
624             "0a 00 00 24 08 00 00 28  00 00 50 00 28 c8 00 c9" +
625             "90 02 aa 58 05 ca d0 0c  0a f8 16 83 18 9c 18 00" +
626             "40 78 13 64 d5 7c 2e 2c  c3 59 de 79 6e c3 c2 00",
627 
628             "b5 00 3c 00 01 04 00 40  00 0c 80 4e 20 27 10 00" +
629             "0e 80 00 24 08 00 00 28  00 00 50 00 28 c8 00 c9" +
630             "90 02 aa 58 05 ca d0 0c  0a f8 16 83 18 9c 18 00" +
631             "40 78 13 64 d5 7c 2e 2c  c3 59 de 79 6e c3 c2 00",
632 
633             "b5 00 3c 00 01 04 00 40  00 0c 80 4e 20 27 10 00" +
634             "0e 80 00 24 08 00 00 28  00 00 50 00 28 c8 00 c9" +
635             "90 02 aa 58 05 ca d0 0c  0a f8 16 83 18 9c 18 00" +
636             "40 78 13 64 d5 7c 2e 2c  c3 59 de 79 6e c3 c2 00",
637     };
638     boolean mTestDynamicMetadata = false;
639     static final String CODEC_PREFIX_KEY = "codec-prefix";
640     static final String MEDIA_TYPE_PREFIX_KEY = "media-type-prefix";
641     static final String MIME_SEL_KEY = "mime-sel";
642     static final Map<String, String> codecSelKeyMimeMap = new HashMap<>();
643     static final Map<String, String> mDefaultEncoders = new HashMap<>();
644     static final Map<String, String> mDefaultDecoders = new HashMap<>();
645     static final HashMap<String, int[]> mProfileMap = new HashMap<>();
646     static final HashMap<String, int[]> mProfileSdrMap = new HashMap<>();
647     static final HashMap<String, int[]> mProfileHlgMap = new HashMap<>();
648     static final HashMap<String, int[]> mProfileHdr10Map = new HashMap<>();
649     static final HashMap<String, int[]> mProfileHdr10PlusMap = new HashMap<>();
650     static final HashMap<String, int[]> mProfileHdrMap = new HashMap<>();
651     static final boolean ENABLE_LOGS = false;
652     static final int PER_TEST_TIMEOUT_LARGE_TEST_MS = 300000;
653     static final int PER_TEST_TIMEOUT_SMALL_TEST_MS = 60000;
654     static final int UNSPECIFIED = 0;
655     // Maintain Timeouts in sync with their counterpart in NativeMediaCommon.h
656     static final long Q_DEQ_TIMEOUT_US = 5000; // block at most 5ms while looking for io buffers
657     static final int RETRY_LIMIT = 100; // max poll counter before test aborts and returns error
658     static final String INVALID_CODEC = "unknown.codec_";
659     static final int[] MPEG2_PROFILES = new int[]{MPEG2ProfileSimple, MPEG2ProfileMain,
660             MPEG2Profile422, MPEG2ProfileSNR, MPEG2ProfileSpatial, MPEG2ProfileHigh};
661     static final int[] MPEG4_PROFILES = new int[]{MPEG4ProfileSimple, MPEG4ProfileSimpleScalable,
662             MPEG4ProfileCore, MPEG4ProfileMain, MPEG4ProfileNbit, MPEG4ProfileScalableTexture,
663             MPEG4ProfileSimpleFace, MPEG4ProfileSimpleFBA, MPEG4ProfileBasicAnimated,
664             MPEG4ProfileHybrid, MPEG4ProfileAdvancedRealTime, MPEG4ProfileCoreScalable,
665             MPEG4ProfileAdvancedCoding, MPEG4ProfileAdvancedCore, MPEG4ProfileAdvancedScalable,
666             MPEG4ProfileAdvancedSimple};
667     static final int[] H263_PROFILES = new int[]{H263ProfileBaseline, H263ProfileH320Coding,
668             H263ProfileBackwardCompatible, H263ProfileISWV2, H263ProfileISWV3,
669             H263ProfileHighCompression, H263ProfileInternet, H263ProfileInterlace,
670             H263ProfileHighLatency};
671     static final int[] VP8_PROFILES = new int[] {VP8ProfileMain};
672     static final int[] AVC_SDR_PROFILES = new int[]{AVCProfileBaseline, AVCProfileMain,
673             AVCProfileExtended, AVCProfileHigh, AVCProfileConstrainedBaseline,
674             AVCProfileConstrainedHigh};
675     static final int[] AVC_HLG_PROFILES = new int[]{AVCProfileHigh10};
676     static final int[] AVC_HDR_PROFILES = AVC_HLG_PROFILES;
677     static final int[] AVC_PROFILES = combine(AVC_SDR_PROFILES, AVC_HDR_PROFILES);
678     static final int[] VP9_SDR_PROFILES = new int[]{VP9Profile0};
679     static final int[] VP9_HLG_PROFILES = new int[]{VP9Profile2};
680     static final int[] VP9_HDR10_PROFILES = new int[]{VP9Profile2HDR};
681     static final int[] VP9_HDR10Plus_PROFILES = new int[]{VP9Profile2HDR10Plus};
682     static final int[] VP9_HDR_PROFILES =
683             combine(VP9_HLG_PROFILES, combine(VP9_HDR10_PROFILES, VP9_HDR10Plus_PROFILES));
684     static final int[] VP9_PROFILES = combine(VP9_SDR_PROFILES, VP9_HDR_PROFILES);
685     static final int[] HEVC_SDR_PROFILES = new int[]{HEVCProfileMain, HEVCProfileMainStill};
686     static final int[] HEVC_HLG_PROFILES = new int[]{HEVCProfileMain10};
687     static final int[] HEVC_HDR10_PROFILES = new int[]{HEVCProfileMain10HDR10};
688     static final int[] HEVC_HDR10Plus_PROFILES = new int[]{HEVCProfileMain10HDR10Plus};
689     static final int[] HEVC_HDR_PROFILES =
690             combine(HEVC_HLG_PROFILES, combine(HEVC_HDR10_PROFILES, HEVC_HDR10Plus_PROFILES));
691     static final int[] HEVC_PROFILES = combine(HEVC_SDR_PROFILES, HEVC_HDR_PROFILES);
692     static final int[] AV1_SDR_PROFILES = new int[]{AV1ProfileMain8};
693     static final int[] AV1_HLG_PROFILES = new int[]{AV1ProfileMain10};
694     static final int[] AV1_HDR10_PROFILES = new int[]{AV1ProfileMain10HDR10};
695     static final int[] AV1_HDR10Plus_PROFILES = new int[]{AV1ProfileMain10HDR10Plus};
696     static final int[] AV1_HDR_PROFILES =
697             combine(AV1_HLG_PROFILES, combine(AV1_HDR10_PROFILES, AV1_HDR10Plus_PROFILES));
698     static final int[] AV1_PROFILES = combine(AV1_SDR_PROFILES, AV1_HDR_PROFILES);
699     static final int[] AAC_PROFILES = new int[]{AACObjectMain, AACObjectLC, AACObjectSSR,
700             AACObjectLTP, AACObjectHE, AACObjectScalable, AACObjectERLC, AACObjectERScalable,
701             AACObjectLD, AACObjectELD, AACObjectXHE};
702     static final String mInpPrefix = WorkDir.getMediaDirString();
703     static final Context mContext = InstrumentationRegistry.getInstrumentation().getTargetContext();
704     static final PackageManager pm = mContext.getPackageManager();
705     static String mimeSelKeys;
706     static String codecPrefix;
707     static String mediaTypePrefix;
708 
709     CodecAsyncHandler mAsyncHandle;
710     boolean mIsCodecInAsyncMode;
711     boolean mSawInputEOS;
712     boolean mSawOutputEOS;
713     boolean mSignalEOSWithLastFrame;
714     int mInputCount;
715     int mOutputCount;
716     long mPrevOutputPts;
717     boolean mSignalledOutFormatChanged;
718     MediaFormat mOutFormat;
719     boolean mIsAudio;
720 
721     boolean mSaveToMem;
722     OutputManager mOutputBuff;
723 
724     String mCodecName;
725     MediaCodec mCodec;
726     Surface mSurface;
727 
728     static {
729         System.loadLibrary("ctsmediav2codec_jni");
730 
731         codecSelKeyMimeMap.put("vp8", MediaFormat.MIMETYPE_VIDEO_VP8);
732         codecSelKeyMimeMap.put("vp9", MediaFormat.MIMETYPE_VIDEO_VP9);
733         codecSelKeyMimeMap.put("av1", MediaFormat.MIMETYPE_VIDEO_AV1);
734         codecSelKeyMimeMap.put("avc", MediaFormat.MIMETYPE_VIDEO_AVC);
735         codecSelKeyMimeMap.put("hevc", MediaFormat.MIMETYPE_VIDEO_HEVC);
736         codecSelKeyMimeMap.put("mpeg4", MediaFormat.MIMETYPE_VIDEO_MPEG4);
737         codecSelKeyMimeMap.put("h263", MediaFormat.MIMETYPE_VIDEO_H263);
738         codecSelKeyMimeMap.put("mpeg2", MediaFormat.MIMETYPE_VIDEO_MPEG2);
739         codecSelKeyMimeMap.put("vraw", MediaFormat.MIMETYPE_VIDEO_RAW);
740         codecSelKeyMimeMap.put("amrnb", MediaFormat.MIMETYPE_AUDIO_AMR_NB);
741         codecSelKeyMimeMap.put("amrwb", MediaFormat.MIMETYPE_AUDIO_AMR_WB);
742         codecSelKeyMimeMap.put("mp3", MediaFormat.MIMETYPE_AUDIO_MPEG);
743         codecSelKeyMimeMap.put("aac", MediaFormat.MIMETYPE_AUDIO_AAC);
744         codecSelKeyMimeMap.put("vorbis", MediaFormat.MIMETYPE_AUDIO_VORBIS);
745         codecSelKeyMimeMap.put("opus", MediaFormat.MIMETYPE_AUDIO_OPUS);
746         codecSelKeyMimeMap.put("g711alaw", MediaFormat.MIMETYPE_AUDIO_G711_ALAW);
747         codecSelKeyMimeMap.put("g711mlaw", MediaFormat.MIMETYPE_AUDIO_G711_MLAW);
748         codecSelKeyMimeMap.put("araw", MediaFormat.MIMETYPE_AUDIO_RAW);
749         codecSelKeyMimeMap.put("flac", MediaFormat.MIMETYPE_AUDIO_FLAC);
750         codecSelKeyMimeMap.put("gsm", MediaFormat.MIMETYPE_AUDIO_MSGSM);
751 
752         android.os.Bundle args = InstrumentationRegistry.getArguments();
753         mimeSelKeys = args.getString(MIME_SEL_KEY);
754         codecPrefix = args.getString(CODEC_PREFIX_KEY);
755         mediaTypePrefix = args.getString(MEDIA_TYPE_PREFIX_KEY);
756 
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_SDR_PROFILES)757         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_SDR_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_SDR_PROFILES)758         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_SDR_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES)759         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES)760         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES)761         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES)762         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_SDR_PROFILES)763         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_SDR_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_SDR_PROFILES)764         mProfileSdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_SDR_PROFILES);
mProfileSdrMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES)765         mProfileSdrMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES);
766 
mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HLG_PROFILES)767         mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HLG_PROFILES);
mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HLG_PROFILES)768         mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HLG_PROFILES);
mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HLG_PROFILES)769         mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HLG_PROFILES);
mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HLG_PROFILES)770         mProfileHlgMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HLG_PROFILES);
771 
mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10_PROFILES)772         mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10_PROFILES);
mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10_PROFILES)773         mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10_PROFILES);
mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10_PROFILES)774         mProfileHdr10Map.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10_PROFILES);
775 
mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10Plus_PROFILES)776         mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR10Plus_PROFILES);
mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10Plus_PROFILES)777         mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR10Plus_PROFILES);
mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10Plus_PROFILES)778         mProfileHdr10PlusMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR10Plus_PROFILES);
779 
mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HDR_PROFILES)780         mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_HDR_PROFILES);
mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR_PROFILES)781         mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_HDR_PROFILES);
mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR_PROFILES)782         mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_HDR_PROFILES);
mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR_PROFILES)783         mProfileHdrMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_HDR_PROFILES);
784 
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_PROFILES)785         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, AVC_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_PROFILES)786         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, HEVC_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES)787         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_H263, H263_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES)788         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG2, MPEG2_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES)789         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_MPEG4, MPEG4_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES)790         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, VP8_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_PROFILES)791         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, VP9_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_PROFILES)792         mProfileMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, AV1_PROFILES);
mProfileMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES)793         mProfileMap.put(MediaFormat.MIMETYPE_AUDIO_AAC, AAC_PROFILES);
794     }
795 
combine(int[] first, int[] second)796     static int[] combine(int[] first, int[] second) {
797         int[] result = Arrays.copyOf(first, first.length + second.length);
798         System.arraycopy(second, 0, result, first.length, second.length);
799         return result;
800     }
801 
isCodecLossless(String mime)802     static boolean isCodecLossless(String mime) {
803         return mime.equals(MediaFormat.MIMETYPE_AUDIO_FLAC) ||
804                 mime.equals(MediaFormat.MIMETYPE_AUDIO_RAW);
805     }
806 
hasDecoder(String mime)807     static boolean hasDecoder(String mime) {
808         return CodecTestBase.selectCodecs(mime, null, null, false).size() != 0;
809     }
810 
hasEncoder(String mime)811     static boolean hasEncoder(String mime) {
812         return CodecTestBase.selectCodecs(mime, null, null, true).size() != 0;
813     }
814 
checkFormatSupport(String codecName, String mime, boolean isEncoder, ArrayList<MediaFormat> formats, String[] features, SupportClass supportRequirements)815     public static void checkFormatSupport(String codecName, String mime, boolean isEncoder,
816             ArrayList<MediaFormat> formats, String[] features, SupportClass supportRequirements)
817             throws IOException {
818         if (!areFormatsSupported(codecName, mime, formats)) {
819             switch (supportRequirements) {
820                 case CODEC_ALL:
821                     fail("format(s) not supported by codec: " + codecName
822                                     + " for mime : " + mime + " formats: " + formats);
823                     break;
824                 case CODEC_ANY:
825                     if (selectCodecs(mime, formats, features, isEncoder).isEmpty())
826                         fail("format(s) not supported by any component for mime : " + mime
827                                         + " formats: " + formats);
828                     break;
829                 case CODEC_DEFAULT:
830                     if (isDefaultCodec(codecName, mime, isEncoder))
831                         fail("format(s) not supported by default codec : " + codecName +
832                                 "for mime : " + mime + " formats: " + formats);
833                     break;
834                 case CODEC_OPTIONAL:
835                 default:
836                     // the later assumeTrue() ensures we skip the test for unsupported codecs
837                     break;
838             }
839             Assume.assumeTrue("format(s) not supported by codec: " + codecName + " for mime : " +
840                     mime, false);
841         }
842     }
843 
isFeatureSupported(String name, String mime, String feature)844     static boolean isFeatureSupported(String name, String mime, String feature) throws IOException {
845         MediaCodec codec = MediaCodec.createByCodecName(name);
846         MediaCodecInfo.CodecCapabilities codecCapabilities =
847                 codec.getCodecInfo().getCapabilitiesForType(mime);
848         boolean isSupported = codecCapabilities.isFeatureSupported(feature);
849         codec.release();
850         return isSupported;
851     }
852 
isHDREditingSupported()853     static boolean isHDREditingSupported() {
854         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
855         for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) {
856             if (!codecInfo.isEncoder()) {
857                 continue;
858             }
859             for (String mediaType : codecInfo.getSupportedTypes()) {
860                 CodecCapabilities caps = codecInfo.getCapabilitiesForType(mediaType);
861                 if (caps != null && caps.isFeatureSupported(FEATURE_HdrEditing)) {
862                     return true;
863                 }
864             }
865         }
866         return false;
867     }
868 
doesAnyFormatHaveHDRProfile(String mime, ArrayList<MediaFormat> formats)869     static boolean doesAnyFormatHaveHDRProfile(String mime, ArrayList<MediaFormat> formats) {
870         int[] profileArray = mProfileHdrMap.get(mime);
871         if (profileArray != null) {
872             for (MediaFormat format : formats) {
873                 assertEquals(mime, format.getString(MediaFormat.KEY_MIME));
874                 int profile = format.getInteger(MediaFormat.KEY_PROFILE, -1);
875                 if (IntStream.of(profileArray).anyMatch(x -> x == profile)) return true;
876             }
877         }
878         return false;
879     }
880 
doesCodecSupportHDRProfile(String codecName, String mediaType)881     static boolean doesCodecSupportHDRProfile(String codecName, String mediaType)
882             throws IOException {
883         int[] hdrProfiles = mProfileHdrMap.get(mediaType);
884         if (hdrProfiles == null) {
885             return false;
886         }
887         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
888         for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) {
889             if (!codecName.equals(codecInfo.getName())) {
890                 continue;
891             }
892             CodecCapabilities caps = codecInfo.getCapabilitiesForType(mediaType);
893             if (caps == null) {
894                 return false;
895             }
896             for (CodecProfileLevel pl : caps.profileLevels) {
897                 if (IntStream.of(hdrProfiles).anyMatch(x -> x == pl.profile)) {
898                     return true;
899                 }
900             }
901         }
902         return false;
903     }
904 
canDisplaySupportHDRContent()905     static boolean canDisplaySupportHDRContent() {
906         DisplayManager displayManager = mContext.getSystemService(DisplayManager.class);
907         return displayManager.getDisplay(Display.DEFAULT_DISPLAY).getHdrCapabilities()
908                 .getSupportedHdrTypes().length != 0;
909     }
910 
areFormatsSupported(String name, String mime, ArrayList<MediaFormat> formats)911     static boolean areFormatsSupported(String name, String mime, ArrayList<MediaFormat> formats)
912             throws IOException {
913         MediaCodec codec = MediaCodec.createByCodecName(name);
914         MediaCodecInfo.CodecCapabilities codecCapabilities =
915                 codec.getCodecInfo().getCapabilitiesForType(mime);
916         boolean isSupported = true;
917         if (formats != null) {
918             for (int i = 0; i < formats.size() && isSupported; i++) {
919                 isSupported = codecCapabilities.isFormatSupported(formats.get(i));
920             }
921         }
922         codec.release();
923         return isSupported;
924     }
925 
hasSupportForColorFormat(String name, String mime, int colorFormat)926     static boolean hasSupportForColorFormat(String name, String mime, int colorFormat)
927             throws IOException {
928         MediaCodec codec = MediaCodec.createByCodecName(name);
929         MediaCodecInfo.CodecCapabilities cap =
930                 codec.getCodecInfo().getCapabilitiesForType(mime);
931         boolean hasSupport = false;
932         for (int c : cap.colorFormats) {
933             if (c == colorFormat) {
934                 hasSupport = true;
935                 break;
936             }
937         }
938         codec.release();
939         return hasSupport;
940     }
941 
isDefaultCodec(String codecName, String mime, boolean isEncoder)942     static boolean isDefaultCodec(String codecName, String mime, boolean isEncoder)
943             throws IOException {
944         Map<String,String> mDefaultCodecs = isEncoder ? mDefaultEncoders:  mDefaultDecoders;
945         if (mDefaultCodecs.containsKey(mime)) {
946             return mDefaultCodecs.get(mime).equalsIgnoreCase(codecName);
947         }
948         MediaCodec codec = isEncoder ? MediaCodec.createEncoderByType(mime)
949                 : MediaCodec.createDecoderByType(mime);
950         boolean isDefault = codec.getName().equalsIgnoreCase(codecName);
951         mDefaultCodecs.put(mime, codec.getName());
952         codec.release();
953         return isDefault;
954     }
955 
isVendorCodec(String codecName)956     static boolean isVendorCodec(String codecName) {
957         MediaCodecList mcl = new MediaCodecList(MediaCodecList.ALL_CODECS);
958         for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) {
959             if (codecName.equals(codecInfo.getName())) {
960                 return codecInfo.isVendor();
961             }
962         }
963         return false;
964     }
965 
compileRequiredMimeList(boolean isEncoder, boolean needAudio, boolean needVideo)966     static ArrayList<String> compileRequiredMimeList(boolean isEncoder, boolean needAudio,
967             boolean needVideo) {
968         Set<String> list = new HashSet<>();
969         if (!isEncoder) {
970             if (MediaUtils.hasAudioOutput() && needAudio) {
971                 // sec 5.1.2
972                 list.add(MediaFormat.MIMETYPE_AUDIO_AAC);
973                 list.add(MediaFormat.MIMETYPE_AUDIO_FLAC);
974                 list.add(MediaFormat.MIMETYPE_AUDIO_MPEG);
975                 list.add(MediaFormat.MIMETYPE_AUDIO_VORBIS);
976                 list.add(MediaFormat.MIMETYPE_AUDIO_RAW);
977                 list.add(MediaFormat.MIMETYPE_AUDIO_OPUS);
978             }
979             if (MediaUtils.isHandheld() || MediaUtils.isTablet() || MediaUtils.isTv() ||
980                     MediaUtils.isAutomotive()) {
981                 // sec 2.2.2, 2.3.2, 2.5.2
982                 if (needAudio) {
983                     list.add(MediaFormat.MIMETYPE_AUDIO_AAC);
984                 }
985                 if (needVideo) {
986                     list.add(MediaFormat.MIMETYPE_VIDEO_AVC);
987                     list.add(MediaFormat.MIMETYPE_VIDEO_MPEG4);
988                     list.add(MediaFormat.MIMETYPE_VIDEO_H263);
989                     list.add(MediaFormat.MIMETYPE_VIDEO_VP8);
990                     list.add(MediaFormat.MIMETYPE_VIDEO_VP9);
991                 }
992             }
993             if (MediaUtils.isHandheld() || MediaUtils.isTablet()) {
994                 // sec 2.2.2
995                 if (needAudio) {
996                     list.add(MediaFormat.MIMETYPE_AUDIO_AMR_NB);
997                     list.add(MediaFormat.MIMETYPE_AUDIO_AMR_WB);
998                 }
999                 if (needVideo) {
1000                     list.add(MediaFormat.MIMETYPE_VIDEO_HEVC);
1001                 }
1002             }
1003             if (MediaUtils.isTv() && needVideo) {
1004                 // sec 2.3.2
1005                 list.add(MediaFormat.MIMETYPE_VIDEO_HEVC);
1006                 list.add(MediaFormat.MIMETYPE_VIDEO_MPEG2);
1007             }
1008         } else {
1009             if (MediaUtils.hasMicrophone() && needAudio) {
1010                 // sec 5.1.1
1011                 // TODO(b/154423550)
1012                 // list.add(MediaFormat.MIMETYPE_AUDIO_RAW);
1013                 list.add(MediaFormat.MIMETYPE_AUDIO_FLAC);
1014                 list.add(MediaFormat.MIMETYPE_AUDIO_OPUS);
1015             }
1016             if (MediaUtils.isHandheld() || MediaUtils.isTablet() || MediaUtils.isTv() ||
1017                     MediaUtils.isAutomotive()) {
1018                 // sec 2.2.2, 2.3.2, 2.5.2
1019                 if (needAudio) {
1020                     list.add(MediaFormat.MIMETYPE_AUDIO_AAC);
1021                 }
1022                 if (needVideo) {
1023                     list.add(MediaFormat.MIMETYPE_VIDEO_AVC);
1024                     list.add(MediaFormat.MIMETYPE_VIDEO_VP8);
1025                 }
1026             }
1027             if ((MediaUtils.isHandheld() || MediaUtils.isTablet()) && needAudio) {
1028                 // sec 2.2.2
1029                 list.add(MediaFormat.MIMETYPE_AUDIO_AMR_NB);
1030                 list.add(MediaFormat.MIMETYPE_AUDIO_AMR_WB);
1031             }
1032         }
1033         return new ArrayList<>(list);
1034     }
1035 
compileCompleteTestMimeList(boolean isEncoder, boolean needAudio, boolean needVideo)1036     static ArrayList<String> compileCompleteTestMimeList(boolean isEncoder, boolean needAudio,
1037             boolean needVideo) {
1038         ArrayList<String> mimes = new ArrayList<>();
1039         if (mimeSelKeys == null) {
1040             ArrayList<String> cddRequiredMimeList =
1041                     compileRequiredMimeList(isEncoder, needAudio, needVideo);
1042             MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1043             MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
1044             for (MediaCodecInfo codecInfo : codecInfos) {
1045                 if (codecInfo.isEncoder() != isEncoder) continue;
1046                 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue;
1047                 String[] types = codecInfo.getSupportedTypes();
1048                 for (String type : types) {
1049                     if (mediaTypePrefix != null && !type.startsWith(mediaTypePrefix)) {
1050                         continue;
1051                     }
1052                     if (!needAudio && type.startsWith("audio/")) continue;
1053                     if (!needVideo && type.startsWith("video/")) continue;
1054                     if (!mimes.contains(type)) {
1055                         mimes.add(type);
1056                     }
1057                 }
1058             }
1059             if (mediaTypePrefix != null) {
1060                 return mimes;
1061             }
1062             // feature_video_output is not exposed to package manager. Testing for video output
1063             // ports, such as VGA, HDMI, DisplayPort, or a wireless port for display is also not
1064             // direct.
1065             /* sec 5.2: device implementations include an embedded screen display with the
1066             diagonal length of at least 2.5 inches or include a video output port or declare the
1067             support of a camera */
1068             if (isEncoder && needVideo &&
1069                     (MediaUtils.hasCamera() || MediaUtils.getScreenSizeInInches() >= 2.5) &&
1070                     !mimes.contains(MediaFormat.MIMETYPE_VIDEO_AVC) &&
1071                     !mimes.contains(MediaFormat.MIMETYPE_VIDEO_VP8)) {
1072                 // Add required cdd mimes here so that respective codec tests fail.
1073                 mimes.add(MediaFormat.MIMETYPE_VIDEO_AVC);
1074                 mimes.add(MediaFormat.MIMETYPE_VIDEO_VP8);
1075                 Log.e(LOG_TAG,"device must support at least one of VP8 or AVC video encoders");
1076             }
1077             for (String mime : cddRequiredMimeList) {
1078                 if (!mimes.contains(mime)) {
1079                     // Add required cdd mimes here so that respective codec tests fail.
1080                     mimes.add(mime);
1081                     Log.e(LOG_TAG, "no codec found for mime " + mime + " as required by cdd");
1082                 }
1083             }
1084         } else {
1085             for (Map.Entry<String, String> entry : codecSelKeyMimeMap.entrySet()) {
1086                 String key = entry.getKey();
1087                 String value = entry.getValue();
1088                 if (mimeSelKeys.contains(key) && !mimes.contains(value)) mimes.add(value);
1089             }
1090         }
1091         return mimes;
1092     }
1093 
prepareParamList(List<Object[]> exhaustiveArgsList, boolean isEncoder, boolean needAudio, boolean needVideo, boolean mustTestAllCodecs)1094     static List<Object[]> prepareParamList(List<Object[]> exhaustiveArgsList, boolean isEncoder,
1095             boolean needAudio, boolean needVideo, boolean mustTestAllCodecs) {
1096         ArrayList<String> mimes = compileCompleteTestMimeList(isEncoder, needAudio, needVideo);
1097         ArrayList<String> cddRequiredMimeList =
1098                 compileRequiredMimeList(isEncoder, needAudio, needVideo);
1099         final List<Object[]> argsList = new ArrayList<>();
1100         int argLength = exhaustiveArgsList.get(0).length;
1101         for (String mime : mimes) {
1102             ArrayList<String> totalListOfCodecs = selectCodecs(mime, null, null, isEncoder);
1103             ArrayList<String> listOfCodecs = new ArrayList<>();
1104             if (codecPrefix != null) {
1105                 for (String codec : totalListOfCodecs) {
1106                     if (codec.startsWith(codecPrefix)) {
1107                         listOfCodecs.add(codec);
1108                     }
1109                 }
1110             } else {
1111                 listOfCodecs = totalListOfCodecs;
1112             }
1113             if (mustTestAllCodecs && listOfCodecs.size() == 0 && codecPrefix == null) {
1114                 listOfCodecs.add(INVALID_CODEC + mime);
1115             }
1116             boolean miss = true;
1117             for (Object[] arg : exhaustiveArgsList) {
1118                 if (mime.equals(arg[0])) {
1119                     for (String codec : listOfCodecs) {
1120                         Object[] arg_ = new Object[argLength + 1];
1121                         arg_[0] = codec;
1122                         System.arraycopy(arg, 0, arg_, 1, argLength);
1123                         argsList.add(arg_);
1124                     }
1125                     miss = false;
1126                 }
1127             }
1128             if (miss && mustTestAllCodecs) {
1129                 if (!cddRequiredMimeList.contains(mime)) {
1130                     Log.w(LOG_TAG, "no test vectors available for optional mime type " + mime);
1131                     continue;
1132                 }
1133                 for (String codec : listOfCodecs) {
1134                     Object[] arg_ = new Object[argLength + 1];
1135                     arg_[0] = codec;
1136                     arg_[1] = mime;
1137                     System.arraycopy(exhaustiveArgsList.get(0), 1, arg_, 2, argLength - 1);
1138                     argsList.add(arg_);
1139                 }
1140             }
1141         }
1142         return argsList;
1143     }
1144 
enqueueInput(int bufferIndex)1145     abstract void enqueueInput(int bufferIndex) throws IOException;
1146 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)1147     abstract void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info);
1148 
configureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, boolean isEncoder)1149     void configureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame,
1150             boolean isEncoder) {
1151         resetContext(isAsync, signalEOSWithLastFrame);
1152         mAsyncHandle.setCallBack(mCodec, isAsync);
1153         // signalEOS flag has nothing to do with configure. We are using this flag to try all
1154         // available configure apis
1155         if (signalEOSWithLastFrame) {
1156             mCodec.configure(format, mSurface, null,
1157                     isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0);
1158         } else {
1159             mCodec.configure(format, mSurface, isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0,
1160                     null);
1161         }
1162         if (ENABLE_LOGS) {
1163             Log.v(LOG_TAG, "codec configured");
1164         }
1165     }
1166 
flushCodec()1167     void flushCodec() {
1168         mCodec.flush();
1169         // TODO(b/147576107): is it ok to clearQueues right away or wait for some signal
1170         mAsyncHandle.clearQueues();
1171         mSawInputEOS = false;
1172         mSawOutputEOS = false;
1173         mInputCount = 0;
1174         mOutputCount = 0;
1175         mPrevOutputPts = Long.MIN_VALUE;
1176         if (ENABLE_LOGS) {
1177             Log.v(LOG_TAG, "codec flushed");
1178         }
1179     }
1180 
reConfigureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, boolean isEncoder)1181     void reConfigureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame,
1182             boolean isEncoder) {
1183         /* TODO(b/147348711) */
1184         if (false) mCodec.stop();
1185         else mCodec.reset();
1186         configureCodec(format, isAsync, signalEOSWithLastFrame, isEncoder);
1187     }
1188 
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)1189     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
1190         mAsyncHandle.resetContext();
1191         mIsCodecInAsyncMode = isAsync;
1192         mSawInputEOS = false;
1193         mSawOutputEOS = false;
1194         mSignalEOSWithLastFrame = signalEOSWithLastFrame;
1195         mInputCount = 0;
1196         mOutputCount = 0;
1197         mPrevOutputPts = Long.MIN_VALUE;
1198         mSignalledOutFormatChanged = false;
1199     }
1200 
enqueueEOS(int bufferIndex)1201     void enqueueEOS(int bufferIndex) {
1202         if (!mSawInputEOS) {
1203             mCodec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
1204             mSawInputEOS = true;
1205             if (ENABLE_LOGS) {
1206                 Log.v(LOG_TAG, "Queued End of Stream");
1207             }
1208         }
1209     }
1210 
doWork(int frameLimit)1211     void doWork(int frameLimit) throws InterruptedException, IOException {
1212         int frameCount = 0;
1213         if (mIsCodecInAsyncMode) {
1214             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
1215             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < frameLimit) {
1216                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
1217                 if (element != null) {
1218                     int bufferID = element.first;
1219                     MediaCodec.BufferInfo info = element.second;
1220                     if (info != null) {
1221                         // <id, info> corresponds to output callback. Handle it accordingly
1222                         dequeueOutput(bufferID, info);
1223                     } else {
1224                         // <id, null> corresponds to input callback. Handle it accordingly
1225                         enqueueInput(bufferID);
1226                         frameCount++;
1227                     }
1228                 }
1229             }
1230         } else {
1231             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
1232             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
1233             while (!mSawInputEOS && frameCount < frameLimit) {
1234                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
1235                 if (outputBufferId >= 0) {
1236                     dequeueOutput(outputBufferId, outInfo);
1237                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1238                     mOutFormat = mCodec.getOutputFormat();
1239                     mSignalledOutFormatChanged = true;
1240                 }
1241                 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
1242                 if (inputBufferId != -1) {
1243                     enqueueInput(inputBufferId);
1244                     frameCount++;
1245                 }
1246             }
1247         }
1248     }
1249 
queueEOS()1250     void queueEOS() throws InterruptedException {
1251         if (mIsCodecInAsyncMode) {
1252             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS) {
1253                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
1254                 if (element != null) {
1255                     int bufferID = element.first;
1256                     MediaCodec.BufferInfo info = element.second;
1257                     if (info != null) {
1258                         dequeueOutput(bufferID, info);
1259                     } else {
1260                         enqueueEOS(element.first);
1261                     }
1262                 }
1263             }
1264         } else {
1265             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
1266             while (!mSawInputEOS) {
1267                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
1268                 if (outputBufferId >= 0) {
1269                     dequeueOutput(outputBufferId, outInfo);
1270                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1271                     mOutFormat = mCodec.getOutputFormat();
1272                     mSignalledOutFormatChanged = true;
1273                 }
1274                 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
1275                 if (inputBufferId != -1) {
1276                     enqueueEOS(inputBufferId);
1277                 }
1278             }
1279         }
1280     }
1281 
waitForAllOutputs()1282     void waitForAllOutputs() throws InterruptedException {
1283         if (mIsCodecInAsyncMode) {
1284             while (!mAsyncHandle.hasSeenError() && !mSawOutputEOS) {
1285                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getOutput();
1286                 if (element != null) {
1287                     dequeueOutput(element.first, element.second);
1288                 }
1289             }
1290         } else {
1291             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
1292             while (!mSawOutputEOS) {
1293                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
1294                 if (outputBufferId >= 0) {
1295                     dequeueOutput(outputBufferId, outInfo);
1296                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1297                     mOutFormat = mCodec.getOutputFormat();
1298                     mSignalledOutFormatChanged = true;
1299                 }
1300             }
1301         }
1302     }
1303 
selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)1304     static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats,
1305             String[] features, boolean isEncoder) {
1306         MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1307         MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
1308         ArrayList<String> listOfCodecs = new ArrayList<>();
1309         for (MediaCodecInfo codecInfo : codecInfos) {
1310             if (codecInfo.isEncoder() != isEncoder) continue;
1311             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue;
1312             String[] types = codecInfo.getSupportedTypes();
1313             for (String type : types) {
1314                 if (type.equalsIgnoreCase(mime)) {
1315                     boolean isOk = true;
1316                     MediaCodecInfo.CodecCapabilities codecCapabilities =
1317                             codecInfo.getCapabilitiesForType(type);
1318                     if (formats != null) {
1319                         for (MediaFormat format : formats) {
1320                             if (!codecCapabilities.isFormatSupported(format)) {
1321                                 isOk = false;
1322                                 break;
1323                             }
1324                         }
1325                     }
1326                     if (features != null) {
1327                         for (String feature : features) {
1328                             if (!codecCapabilities.isFeatureSupported(feature)) {
1329                                 isOk = false;
1330                                 break;
1331                             }
1332                         }
1333                     }
1334                     if (isOk) listOfCodecs.add(codecInfo.getName());
1335                 }
1336             }
1337         }
1338         return listOfCodecs;
1339     }
1340 
getWidth(MediaFormat format)1341     static int getWidth(MediaFormat format) {
1342         int width = format.getInteger(MediaFormat.KEY_WIDTH, -1);
1343         if (format.containsKey("crop-left") && format.containsKey("crop-right")) {
1344             width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left");
1345         }
1346         return width;
1347     }
1348 
getHeight(MediaFormat format)1349     static int getHeight(MediaFormat format) {
1350         int height = format.getInteger(MediaFormat.KEY_HEIGHT, -1);
1351         if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) {
1352             height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top");
1353         }
1354         return height;
1355     }
1356 
loadByteArrayFromString(final String str)1357     byte[] loadByteArrayFromString(final String str) {
1358         if (str == null) {
1359             return null;
1360         }
1361         Pattern pattern = Pattern.compile("[0-9a-fA-F]{2}");
1362         Matcher matcher = pattern.matcher(str);
1363         // allocate a large enough byte array first
1364         byte[] tempArray = new byte[str.length() / 2];
1365         int i = 0;
1366         while (matcher.find()) {
1367             tempArray[i++] = (byte) Integer.parseInt(matcher.group(), 16);
1368         }
1369         return Arrays.copyOfRange(tempArray, 0, i);
1370     }
1371 
isFormatSimilar(MediaFormat inpFormat, MediaFormat outFormat)1372     boolean isFormatSimilar(MediaFormat inpFormat, MediaFormat outFormat) {
1373         if (inpFormat == null || outFormat == null) return false;
1374         String inpMime = inpFormat.getString(MediaFormat.KEY_MIME);
1375         String outMime = outFormat.getString(MediaFormat.KEY_MIME);
1376         // not comparing input and output mimes because for a codec, mime is raw on one side and
1377         // encoded type on the other
1378         if (outMime.startsWith("audio/")) {
1379             return inpFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT, -1) ==
1380                     outFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT, -2) &&
1381                     inpFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE, -1) ==
1382                             outFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE, -2) &&
1383                     inpMime.startsWith("audio/");
1384         } else if (outMime.startsWith("video/")) {
1385             return getWidth(inpFormat) == getWidth(outFormat) &&
1386                     getHeight(inpFormat) == getHeight(outFormat) && inpMime.startsWith("video/");
1387         }
1388         return true;
1389     }
1390 
validateMetrics(String codec)1391     PersistableBundle validateMetrics(String codec) {
1392         PersistableBundle metrics = mCodec.getMetrics();
1393         assertTrue("metrics is null", metrics != null);
1394         assertTrue(metrics.getString(MediaCodec.MetricsConstants.CODEC).equals(codec));
1395         if (mIsAudio) {
1396             assertTrue(metrics.getString(MediaCodec.MetricsConstants.MODE)
1397                     .equals(MediaCodec.MetricsConstants.MODE_AUDIO));
1398         } else {
1399             assertTrue(metrics.getString(MediaCodec.MetricsConstants.MODE)
1400                     .equals(MediaCodec.MetricsConstants.MODE_VIDEO));
1401         }
1402         return metrics;
1403     }
1404 
validateMetrics(String codec, MediaFormat format)1405     PersistableBundle validateMetrics(String codec, MediaFormat format) {
1406         PersistableBundle metrics = validateMetrics(codec);
1407         if (!mIsAudio) {
1408             assertTrue(metrics.getInt(MediaCodec.MetricsConstants.WIDTH) == getWidth(format));
1409             assertTrue(metrics.getInt(MediaCodec.MetricsConstants.HEIGHT) == getHeight(format));
1410         }
1411         assertTrue(metrics.getInt(MediaCodec.MetricsConstants.SECURE) == 0);
1412         return metrics;
1413     }
1414 
validateColorAspects(MediaFormat fmt, int range, int standard, int transfer)1415     void validateColorAspects(MediaFormat fmt, int range, int standard, int transfer) {
1416         int colorRange = fmt.getInteger(MediaFormat.KEY_COLOR_RANGE, UNSPECIFIED);
1417         int colorStandard = fmt.getInteger(MediaFormat.KEY_COLOR_STANDARD, UNSPECIFIED);
1418         int colorTransfer = fmt.getInteger(MediaFormat.KEY_COLOR_TRANSFER, UNSPECIFIED);
1419         if (range > UNSPECIFIED) {
1420             assertEquals("color range mismatch ", range, colorRange);
1421         }
1422         if (standard > UNSPECIFIED) {
1423             assertEquals("color standard mismatch ", standard, colorStandard);
1424         }
1425         if (transfer > UNSPECIFIED) {
1426             assertEquals("color transfer mismatch ", transfer, colorTransfer);
1427         }
1428     }
1429 
validateHDRStaticMetaData(MediaFormat fmt, ByteBuffer hdrStaticRef)1430     void validateHDRStaticMetaData(MediaFormat fmt, ByteBuffer hdrStaticRef) {
1431         ByteBuffer hdrStaticInfo = fmt.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO, null);
1432         assertNotNull("No HDR static metadata present in format : " + fmt, hdrStaticInfo);
1433         if (!hdrStaticRef.equals(hdrStaticInfo)) {
1434             StringBuilder refString = new StringBuilder("");
1435             StringBuilder testString = new StringBuilder("");
1436             byte[] ref = new byte[hdrStaticRef.capacity()];
1437             hdrStaticRef.get(ref);
1438             byte[] test = new byte[hdrStaticInfo.capacity()];
1439             hdrStaticInfo.get(test);
1440             for (int i = 0; i < Math.min(ref.length, test.length); i++) {
1441                 refString.append(String.format("%2x ", ref[i]));
1442                 testString.append(String.format("%2x ", test[i]));
1443             }
1444             fail("hdr static info mismatch" + "\n" + "ref static info : " + refString + "\n" +
1445                     "test static info : " + testString);
1446         }
1447     }
1448 
validateHDRDynamicMetaData(MediaFormat fmt, ByteBuffer hdrDynamicRef)1449     void validateHDRDynamicMetaData(MediaFormat fmt, ByteBuffer hdrDynamicRef) {
1450         ByteBuffer hdrDynamicInfo = fmt.getByteBuffer(MediaFormat.KEY_HDR10_PLUS_INFO, null);
1451         assertNotNull("No HDR dynamic metadata present in format : " + fmt, hdrDynamicInfo);
1452         if (!hdrDynamicRef.equals(hdrDynamicInfo)) {
1453             StringBuilder refString = new StringBuilder("");
1454             StringBuilder testString = new StringBuilder("");
1455             byte[] ref = new byte[hdrDynamicRef.capacity()];
1456             hdrDynamicRef.get(ref);
1457             byte[] test = new byte[hdrDynamicInfo.capacity()];
1458             hdrDynamicInfo.get(test);
1459             for (int i = 0; i < Math.min(ref.length, test.length); i++) {
1460                 refString.append(String.format("%2x ", ref[i]));
1461                 testString.append(String.format("%2x ", test[i]));
1462             }
1463             fail("hdr dynamic info mismatch" + "\n" + "ref dynamic info : " + refString + "\n" +
1464                     "test dynamic info : " + testString);
1465         }
1466     }
1467 
setUpSurface(CodecTestActivity activity)1468     public void setUpSurface(CodecTestActivity activity) throws InterruptedException {
1469         activity.waitTillSurfaceIsCreated();
1470         mSurface = activity.getSurface();
1471         assertTrue("Surface created is null.", mSurface != null);
1472         assertTrue("Surface created is invalid.", mSurface.isValid());
1473     }
1474 
tearDownSurface()1475     public void tearDownSurface() {
1476         if (mSurface != null) {
1477             mSurface.release();
1478             mSurface = null;
1479         }
1480     }
1481 
1482     @Before
isCodecNameValid()1483     public void isCodecNameValid() {
1484         if (mCodecName != null && mCodecName.startsWith(INVALID_CODEC)) {
1485             fail("no valid component available for current test ");
1486         }
1487     }
1488 
1489     @After
tearDown()1490     public void tearDown() {
1491         if (mCodec != null) {
1492             mCodec.release();
1493             mCodec = null;
1494         }
1495     }
1496 }
1497 
1498 class CodecDecoderTestBase extends CodecTestBase {
1499     private static final String LOG_TAG = CodecDecoderTestBase.class.getSimpleName();
1500 
1501     String mMime;
1502     String mTestFile;
1503     boolean mIsInterlaced;
1504     boolean mSkipChecksumVerification;
1505 
1506     ArrayList<ByteBuffer> mCsdBuffers;
1507     private int mCurrCsdIdx;
1508 
1509     private ByteBuffer flatBuffer = ByteBuffer.allocate(4 * Integer.BYTES);
1510 
1511     MediaExtractor mExtractor;
1512     CodecTestActivity mActivity;
1513 
CodecDecoderTestBase(String codecName, String mime, String testFile)1514     CodecDecoderTestBase(String codecName, String mime, String testFile) {
1515         mCodecName = codecName;
1516         mMime = mime;
1517         mTestFile = testFile;
1518         mAsyncHandle = new CodecAsyncHandler();
1519         mCsdBuffers = new ArrayList<>();
1520         mIsAudio = mMime.startsWith("audio/");
1521     }
1522 
setUpSource(String srcFile)1523     MediaFormat setUpSource(String srcFile) throws IOException {
1524         return setUpSource(mInpPrefix, srcFile);
1525     }
1526 
setUpSource(String prefix, String srcFile)1527     MediaFormat setUpSource(String prefix, String srcFile) throws IOException {
1528         Preconditions.assertTestFileExists(prefix + srcFile);
1529         mExtractor = new MediaExtractor();
1530         Preconditions.assertTestFileExists(prefix + srcFile);
1531         mExtractor.setDataSource(prefix + srcFile);
1532         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
1533             MediaFormat format = mExtractor.getTrackFormat(trackID);
1534             if (mMime.equalsIgnoreCase(format.getString(MediaFormat.KEY_MIME))) {
1535                 mExtractor.selectTrack(trackID);
1536                 if (mIsAudio) {
1537                     // as per cdd, pcm/wave must support PCM_{8, 16, 24, 32, float} and flac must
1538                     // support PCM_{16, float}. For raw media type let extractor manage the
1539                     // encoding type directly. For flac, basing on bits-per-sample select the type
1540                     if (mMime.equals(MediaFormat.MIMETYPE_AUDIO_FLAC)) {
1541                         if (format.getInteger("bits-per-sample", 16) > 16) {
1542                             format.setInteger(MediaFormat.KEY_PCM_ENCODING,
1543                                     AudioFormat.ENCODING_PCM_FLOAT);
1544                         }
1545                     }
1546                 } else {
1547                     ArrayList<MediaFormat> formatList = new ArrayList<>();
1548                     formatList.add(format);
1549                     boolean selectHBD = doesAnyFormatHaveHDRProfile(mMime, formatList) ||
1550                             srcFile.contains("10bit");
1551                     format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1552                             getColorFormat(mCodecName, mMime, mSurface != null, selectHBD));
1553                     if (selectHBD && (format.getInteger(MediaFormat.KEY_COLOR_FORMAT) !=
1554                             COLOR_FormatYUVP010)) {
1555                         mSkipChecksumVerification = true;
1556                     }
1557 
1558                     if ((format.getInteger(MediaFormat.KEY_COLOR_FORMAT) != COLOR_FormatYUVP010)
1559                             && selectHBD && mSurface == null) {
1560                         // Codecs that do not advertise P010 on devices with VNDK version < T, do
1561                         // not support decoding high bit depth clips when color format is set to
1562                         // COLOR_FormatYUV420Flexible in byte buffer mode. Since byte buffer mode
1563                         // for high bit depth decoding wasn't tested prior to Android T, skip this
1564                         // when device is older
1565                         assumeTrue("Skipping High Bit Depth tests on VNDK < T", VNDK_IS_AT_LEAST_T);
1566                     }
1567                 }
1568                 // TODO: determine this from the extractor format when it becomes exposed.
1569                 mIsInterlaced = srcFile.contains("_interlaced_");
1570                 return format;
1571             }
1572         }
1573         fail("No track with mime: " + mMime + " found in file: " + srcFile);
1574         return null;
1575     }
1576 
getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode)1577     int getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode)
1578             throws IOException {
1579         if (surfaceMode) return COLOR_FormatSurface;
1580         if (hbdMode) {
1581             MediaCodec codec = MediaCodec.createByCodecName(name);
1582             MediaCodecInfo.CodecCapabilities cap =
1583                     codec.getCodecInfo().getCapabilitiesForType(mediaType);
1584             codec.release();
1585             for (int c : cap.colorFormats) {
1586                 if (c == COLOR_FormatYUVP010) {
1587                     return c;
1588                 }
1589             }
1590         }
1591         return COLOR_FormatYUV420Flexible;
1592     }
1593 
hasCSD(MediaFormat format)1594     boolean hasCSD(MediaFormat format) {
1595         return format.containsKey("csd-0");
1596     }
1597 
flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio)1598     void flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio) {
1599         if (isAudio) {
1600             flatBuffer.putInt(info.size);
1601         }
1602         flatBuffer.putInt(info.flags & ~MediaCodec.BUFFER_FLAG_END_OF_STREAM)
1603                 .putLong(info.presentationTimeUs);
1604         flatBuffer.flip();
1605     }
1606 
enqueueCodecConfig(int bufferIndex)1607     void enqueueCodecConfig(int bufferIndex) {
1608         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
1609         ByteBuffer csdBuffer = mCsdBuffers.get(mCurrCsdIdx);
1610         inputBuffer.put((ByteBuffer) csdBuffer.rewind());
1611         mCodec.queueInputBuffer(bufferIndex, 0, csdBuffer.limit(), 0,
1612                 MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
1613         if (ENABLE_LOGS) {
1614             Log.v(LOG_TAG, "queued csd: id: " + bufferIndex + " size: " + csdBuffer.limit());
1615         }
1616     }
1617 
enqueueInput(int bufferIndex)1618     void enqueueInput(int bufferIndex) {
1619         if (mExtractor.getSampleSize() < 0) {
1620             enqueueEOS(bufferIndex);
1621         } else {
1622             ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
1623             mExtractor.readSampleData(inputBuffer, 0);
1624             int size = (int) mExtractor.getSampleSize();
1625             long pts = mExtractor.getSampleTime();
1626             int extractorFlags = mExtractor.getSampleFlags();
1627             int codecFlags = 0;
1628             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
1629                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
1630             }
1631             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) {
1632                 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME;
1633             }
1634             if (!mExtractor.advance() && mSignalEOSWithLastFrame) {
1635                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
1636                 mSawInputEOS = true;
1637             }
1638             if (ENABLE_LOGS) {
1639                 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts +
1640                         " flags: " + codecFlags);
1641             }
1642             mCodec.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
1643             if (size > 0 && (codecFlags & (MediaCodec.BUFFER_FLAG_CODEC_CONFIG |
1644                     MediaCodec.BUFFER_FLAG_PARTIAL_FRAME)) == 0) {
1645                 mOutputBuff.saveInPTS(pts);
1646                 mInputCount++;
1647             }
1648         }
1649     }
1650 
enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info)1651     void enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info) {
1652         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
1653         buffer.position(info.offset);
1654         for (int i = 0; i < info.size; i++) {
1655             inputBuffer.put(buffer.get());
1656         }
1657         if (ENABLE_LOGS) {
1658             Log.v(LOG_TAG, "input: id: " + bufferIndex + " flags: " + info.flags + " size: " +
1659                     info.size + " timestamp: " + info.presentationTimeUs);
1660         }
1661         mCodec.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs,
1662                 info.flags);
1663         if (info.size > 0 && ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) &&
1664                 ((info.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) == 0)) {
1665             mOutputBuff.saveInPTS(info.presentationTimeUs);
1666             mInputCount++;
1667         }
1668         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
1669             mSawInputEOS = true;
1670         }
1671     }
1672 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)1673     void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) {
1674         if (info.size > 0 && mSaveToMem) {
1675             ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex);
1676             flattenBufferInfo(info, mIsAudio);
1677             mOutputBuff.checksum(flatBuffer, flatBuffer.limit());
1678             if (mIsAudio) {
1679                 mOutputBuff.checksum(buf, info.size);
1680                 mOutputBuff.saveToMemory(buf, info);
1681             } else {
1682                 // tests both getOutputImage and getOutputBuffer. Can do time division
1683                 // multiplexing but lets allow it for now
1684                 Image img = mCodec.getOutputImage(bufferIndex);
1685                 assertTrue(img != null);
1686                 mOutputBuff.checksum(img);
1687                 int imgFormat = img.getFormat();
1688                 int bytesPerSample = (ImageFormat.getBitsPerPixel(imgFormat) * 2) / (8 * 3);
1689 
1690                 MediaFormat format = mCodec.getOutputFormat();
1691                 buf = mCodec.getOutputBuffer(bufferIndex);
1692                 int width = format.getInteger(MediaFormat.KEY_WIDTH);
1693                 int height = format.getInteger(MediaFormat.KEY_HEIGHT);
1694                 int stride = format.getInteger(MediaFormat.KEY_STRIDE);
1695                 mOutputBuff.checksum(buf, info.size, width, height, stride, bytesPerSample);
1696 
1697                 if (mTestDynamicMetadata) {
1698                     validateHDRDynamicMetaData(mCodec.getOutputFormat(), ByteBuffer
1699                             .wrap(loadByteArrayFromString(HDR_DYNAMIC_INFO[mOutputCount])));
1700 
1701                 }
1702             }
1703         }
1704 
1705         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
1706             mSawOutputEOS = true;
1707         }
1708         if (ENABLE_LOGS) {
1709             Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " +
1710                     info.size + " timestamp: " + info.presentationTimeUs);
1711         }
1712         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
1713             mOutputBuff.saveOutPTS(info.presentationTimeUs);
1714             mOutputCount++;
1715         }
1716         mCodec.releaseOutputBuffer(bufferIndex, false);
1717     }
1718 
doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)1719     void doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)
1720             throws InterruptedException {
1721         int frameCount = 0;
1722         if (mIsCodecInAsyncMode) {
1723             // output processing after queuing EOS is done in waitForAllOutputs()
1724             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < list.size()) {
1725                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
1726                 if (element != null) {
1727                     int bufferID = element.first;
1728                     MediaCodec.BufferInfo info = element.second;
1729                     if (info != null) {
1730                         dequeueOutput(bufferID, info);
1731                     } else {
1732                         enqueueInput(bufferID, buffer, list.get(frameCount));
1733                         frameCount++;
1734                     }
1735                 }
1736             }
1737         } else {
1738             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
1739             // output processing after queuing EOS is done in waitForAllOutputs()
1740             while (!mSawInputEOS && frameCount < list.size()) {
1741                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
1742                 if (outputBufferId >= 0) {
1743                     dequeueOutput(outputBufferId, outInfo);
1744                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1745                     mOutFormat = mCodec.getOutputFormat();
1746                     mSignalledOutFormatChanged = true;
1747                 }
1748                 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
1749                 if (inputBufferId != -1) {
1750                     enqueueInput(inputBufferId, buffer, list.get(frameCount));
1751                     frameCount++;
1752                 }
1753             }
1754         }
1755     }
1756 
queueCodecConfig()1757     void queueCodecConfig() throws InterruptedException {
1758         if (mIsCodecInAsyncMode) {
1759             for (mCurrCsdIdx = 0; !mAsyncHandle.hasSeenError() && mCurrCsdIdx < mCsdBuffers.size();
1760                  mCurrCsdIdx++) {
1761                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getInput();
1762                 if (element != null) {
1763                     enqueueCodecConfig(element.first);
1764                 }
1765             }
1766         } else {
1767             for (mCurrCsdIdx = 0; mCurrCsdIdx < mCsdBuffers.size(); mCurrCsdIdx++) {
1768                 enqueueCodecConfig(mCodec.dequeueInputBuffer(-1));
1769             }
1770         }
1771     }
1772 
decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)1773     void decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)
1774             throws IOException, InterruptedException {
1775         mSaveToMem = true;
1776         mOutputBuff = new OutputManager();
1777         mCodec = MediaCodec.createByCodecName(decoder);
1778         MediaFormat format = setUpSource(file);
1779         configureCodec(format, false, true, false);
1780         mCodec.start();
1781         mExtractor.seekTo(pts, mode);
1782         doWork(frameLimit);
1783         queueEOS();
1784         waitForAllOutputs();
1785         mCodec.stop();
1786         mCodec.release();
1787         mExtractor.release();
1788         mSaveToMem = false;
1789     }
1790 
1791     @Override
validateMetrics(String decoder, MediaFormat format)1792     PersistableBundle validateMetrics(String decoder, MediaFormat format) {
1793         PersistableBundle metrics = super.validateMetrics(decoder, format);
1794         assertTrue(metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE).equals(mMime));
1795         assertTrue(metrics.getInt(MediaCodec.MetricsConstants.ENCODER) == 0);
1796         return metrics;
1797     }
1798 
validateColorAspects(String decoder, String parent, String name, int range, int standard, int transfer, boolean ignoreColorBox)1799     void validateColorAspects(String decoder, String parent, String name, int range, int standard,
1800             int transfer, boolean ignoreColorBox)
1801             throws IOException, InterruptedException {
1802         mOutputBuff = new OutputManager();
1803         MediaFormat format = setUpSource(parent, name);
1804         if (ignoreColorBox) {
1805             format.removeKey(MediaFormat.KEY_COLOR_RANGE);
1806             format.removeKey(MediaFormat.KEY_COLOR_STANDARD);
1807             format.removeKey(MediaFormat.KEY_COLOR_TRANSFER);
1808         }
1809         if (decoder == null) {
1810             MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1811             decoder = codecList.findDecoderForFormat(format);
1812         }
1813         mCodec = MediaCodec.createByCodecName(decoder);
1814         configureCodec(format, true, true, false);
1815         mCodec.start();
1816         doWork(1);
1817         queueEOS();
1818         waitForAllOutputs();
1819         validateColorAspects(mCodec.getOutputFormat(), range, standard, transfer);
1820         mCodec.stop();
1821         mCodec.release();
1822         mExtractor.release();
1823     }
1824 
validateHDRStaticMetaData(String parent, String name, ByteBuffer HDRStatic, boolean ignoreContainerStaticInfo)1825     void validateHDRStaticMetaData(String parent, String name, ByteBuffer HDRStatic,
1826                                    boolean ignoreContainerStaticInfo)
1827             throws IOException, InterruptedException {
1828         mOutputBuff = new OutputManager();
1829         MediaFormat format = setUpSource(parent, name);
1830         if (ignoreContainerStaticInfo) {
1831             format.removeKey(MediaFormat.KEY_HDR_STATIC_INFO);
1832         }
1833         mCodec = MediaCodec.createByCodecName(mCodecName);
1834         configureCodec(format, true, true, false);
1835         mCodec.start();
1836         doWork(10);
1837         queueEOS();
1838         waitForAllOutputs();
1839         validateHDRStaticMetaData(mCodec.getOutputFormat(), HDRStatic);
1840         mCodec.stop();
1841         mCodec.release();
1842         mExtractor.release();
1843     }
1844 
validateHDRDynamicMetaData(String parent, String name, boolean ignoreContainerDynamicInfo)1845     void validateHDRDynamicMetaData(String parent, String name, boolean ignoreContainerDynamicInfo)
1846             throws IOException, InterruptedException {
1847         mOutputBuff = new OutputManager();
1848         MediaFormat format = setUpSource(parent, name);
1849         if (ignoreContainerDynamicInfo) {
1850             format.removeKey(MediaFormat.KEY_HDR10_PLUS_INFO);
1851         }
1852         mCodec = MediaCodec.createByCodecName(mCodecName);
1853         configureCodec(format, true, true, false);
1854         mCodec.start();
1855         doWork(10);
1856         queueEOS();
1857         waitForAllOutputs();
1858         mCodec.stop();
1859         mCodec.release();
1860         mExtractor.release();
1861     }
1862 }
1863 
1864 class CodecEncoderTestBase extends CodecTestBase {
1865     private static final String LOG_TAG = CodecEncoderTestBase.class.getSimpleName();
1866 
1867     // files are in WorkDir.getMediaDirString();
1868     private static final String INPUT_AUDIO_FILE = "bbb_2ch_44kHz_s16le.raw";
1869     private static final String INPUT_VIDEO_FILE = "bbb_cif_yuv420p_30fps.yuv";
1870     protected static final String INPUT_AUDIO_FILE_HBD = "audio/sd_2ch_48kHz_f32le.raw";
1871     protected static final String INPUT_VIDEO_FILE_HBD = "cosmat_cif_24fps_yuv420p16le.yuv";
1872 
1873     private final int INP_FRM_WIDTH = 352;
1874     private final int INP_FRM_HEIGHT = 288;
1875 
1876     final String mMime;
1877     final int[] mBitrates;
1878     final int[] mEncParamList1;
1879     final int[] mEncParamList2;
1880 
1881     final String mInputFile;
1882     byte[] mInputData;
1883     int mNumBytesSubmitted;
1884     long mInputOffsetPts;
1885 
1886     ArrayList<MediaFormat> mFormats;
1887     ArrayList<MediaCodec.BufferInfo> mInfoList;
1888 
1889     int mWidth, mHeight;
1890     int mFrameRate;
1891     int mMaxBFrames;
1892     int mChannels;
1893     int mSampleRate;
1894     int mBytesPerSample;
1895 
CodecEncoderTestBase(String encoder, String mime, int[] bitrates, int[] encoderInfo1, int[] encoderInfo2)1896     CodecEncoderTestBase(String encoder, String mime, int[] bitrates, int[] encoderInfo1,
1897             int[] encoderInfo2) {
1898         mMime = mime;
1899         mCodecName = encoder;
1900         mBitrates = bitrates;
1901         mEncParamList1 = encoderInfo1;
1902         mEncParamList2 = encoderInfo2;
1903         mFormats = new ArrayList<>();
1904         mInfoList = new ArrayList<>();
1905         mWidth = INP_FRM_WIDTH;
1906         mHeight = INP_FRM_HEIGHT;
1907         if (mime.equals(MediaFormat.MIMETYPE_VIDEO_MPEG4)) mFrameRate = 12;
1908         else if (mime.equals(MediaFormat.MIMETYPE_VIDEO_H263)) mFrameRate = 12;
1909         else mFrameRate = 30;
1910         mMaxBFrames = 0;
1911         mChannels = 1;
1912         mSampleRate = 8000;
1913         mAsyncHandle = new CodecAsyncHandler();
1914         mIsAudio = mMime.startsWith("audio/");
1915         mBytesPerSample = mIsAudio ? 2 : 1;
1916         mInputFile = mIsAudio ? INPUT_AUDIO_FILE : INPUT_VIDEO_FILE;
1917     }
1918 
1919     /**
1920      * Selects encoder input color format in byte buffer mode. As of now ndk tests support only
1921      * 420p, 420sp. COLOR_FormatYUV420Flexible although can represent any form of yuv, it doesn't
1922      * work in ndk due to lack of AMediaCodec_GetInputImage()
1923      */
findByteBufferColorFormat(String encoder, String mime)1924     static int findByteBufferColorFormat(String encoder, String mime) throws IOException {
1925         MediaCodec codec = MediaCodec.createByCodecName(encoder);
1926         MediaCodecInfo.CodecCapabilities cap = codec.getCodecInfo().getCapabilitiesForType(mime);
1927         int colorFormat = -1;
1928         for (int c : cap.colorFormats) {
1929             if (c == MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar ||
1930                     c == MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar) {
1931                 Log.v(LOG_TAG, "selecting color format: " + c);
1932                 colorFormat = c;
1933                 break;
1934             }
1935         }
1936         codec.release();
1937         return colorFormat;
1938     }
1939 
1940     @Override
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)1941     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
1942         super.resetContext(isAsync, signalEOSWithLastFrame);
1943         mNumBytesSubmitted = 0;
1944         mInputOffsetPts = 0;
1945     }
1946 
1947     @Override
flushCodec()1948     void flushCodec() {
1949         super.flushCodec();
1950         if (mIsAudio) {
1951             mInputOffsetPts = (mNumBytesSubmitted + 1024) * 1000000L /
1952                     (mBytesPerSample * mChannels * mSampleRate);
1953         } else {
1954             mInputOffsetPts = (mInputCount + 5) * 1000000L / mFrameRate;
1955         }
1956         mPrevOutputPts = mInputOffsetPts - 1;
1957         mNumBytesSubmitted = 0;
1958     }
1959 
setUpSource(String srcFile)1960     void setUpSource(String srcFile) throws IOException {
1961         String inpPath = mInpPrefix + srcFile;
1962         try (FileInputStream fInp = new FileInputStream(inpPath)) {
1963             int size = (int) new File(inpPath).length();
1964             mInputData = new byte[size];
1965             fInp.read(mInputData, 0, size);
1966         }
1967     }
1968 
fillImage(Image image)1969     void fillImage(Image image) {
1970         int format = image.getFormat();
1971         assertTrue("unexpected image format",
1972                 format == ImageFormat.YUV_420_888 || format == ImageFormat.YCBCR_P010);
1973         int bytesPerSample = (ImageFormat.getBitsPerPixel(format) * 2) / (8 * 3);  // YUV420
1974         assertEquals("Invalid bytes per sample", bytesPerSample, mBytesPerSample);
1975 
1976         int imageWidth = image.getWidth();
1977         int imageHeight = image.getHeight();
1978         Image.Plane[] planes = image.getPlanes();
1979         int offset = mNumBytesSubmitted;
1980         for (int i = 0; i < planes.length; ++i) {
1981             ByteBuffer buf = planes[i].getBuffer();
1982             int width = imageWidth;
1983             int height = imageHeight;
1984             int tileWidth = INP_FRM_WIDTH;
1985             int tileHeight = INP_FRM_HEIGHT;
1986             int rowStride = planes[i].getRowStride();
1987             int pixelStride = planes[i].getPixelStride();
1988             if (i != 0) {
1989                 width = imageWidth / 2;
1990                 height = imageHeight / 2;
1991                 tileWidth = INP_FRM_WIDTH / 2;
1992                 tileHeight = INP_FRM_HEIGHT / 2;
1993             }
1994             if (pixelStride == bytesPerSample) {
1995                 if (width == rowStride && width == tileWidth && height == tileHeight) {
1996                     buf.put(mInputData, offset, width * height * bytesPerSample);
1997                 } else {
1998                     for (int z = 0; z < height; z += tileHeight) {
1999                         int rowsToCopy = Math.min(height - z, tileHeight);
2000                         for (int y = 0; y < rowsToCopy; y++) {
2001                             for (int x = 0; x < width; x += tileWidth) {
2002                                 int colsToCopy = Math.min(width - x, tileWidth);
2003                                 buf.position((z + y) * rowStride + x * bytesPerSample);
2004                                 buf.put(mInputData, offset + y * tileWidth * bytesPerSample,
2005                                         colsToCopy * bytesPerSample);
2006                             }
2007                         }
2008                     }
2009                 }
2010             } else {
2011                 // do it pixel-by-pixel
2012                 for (int z = 0; z < height; z += tileHeight) {
2013                     int rowsToCopy = Math.min(height - z, tileHeight);
2014                     for (int y = 0; y < rowsToCopy; y++) {
2015                         int lineOffset = (z + y) * rowStride;
2016                         for (int x = 0; x < width; x += tileWidth) {
2017                             int colsToCopy = Math.min(width - x, tileWidth);
2018                             for (int w = 0; w < colsToCopy; w++) {
2019                                 for (int bytePos = 0; bytePos < bytesPerSample; bytePos++) {
2020                                     buf.position(lineOffset + (x + w) * pixelStride + bytePos);
2021                                     buf.put(mInputData[offset + y * tileWidth * bytesPerSample +
2022                                             w * bytesPerSample + bytePos]);
2023                                 }
2024                             }
2025                         }
2026                     }
2027                 }
2028             }
2029             offset += tileWidth * tileHeight * bytesPerSample;
2030         }
2031     }
2032 
fillByteBuffer(ByteBuffer inputBuffer)2033     void fillByteBuffer(ByteBuffer inputBuffer) {
2034         int offset = 0, frmOffset = mNumBytesSubmitted;
2035         for (int plane = 0; plane < 3; plane++) {
2036             int width = mWidth;
2037             int height = mHeight;
2038             int tileWidth = INP_FRM_WIDTH;
2039             int tileHeight = INP_FRM_HEIGHT;
2040             if (plane != 0) {
2041                 width = mWidth / 2;
2042                 height = mHeight / 2;
2043                 tileWidth = INP_FRM_WIDTH / 2;
2044                 tileHeight = INP_FRM_HEIGHT / 2;
2045             }
2046             for (int k = 0; k < height; k += tileHeight) {
2047                 int rowsToCopy = Math.min(height - k, tileHeight);
2048                 for (int j = 0; j < rowsToCopy; j++) {
2049                     for (int i = 0; i < width; i += tileWidth) {
2050                         int colsToCopy = Math.min(width - i, tileWidth);
2051                         inputBuffer.position(
2052                                 offset + (k + j) * width * mBytesPerSample + i * mBytesPerSample);
2053                         inputBuffer.put(mInputData, frmOffset + j * tileWidth * mBytesPerSample,
2054                                 colsToCopy * mBytesPerSample);
2055                     }
2056                 }
2057             }
2058             offset += width * height * mBytesPerSample;
2059             frmOffset += tileWidth * tileHeight * mBytesPerSample;
2060         }
2061     }
2062 
enqueueInput(int bufferIndex)2063     void enqueueInput(int bufferIndex) {
2064         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
2065         if (mNumBytesSubmitted >= mInputData.length) {
2066             enqueueEOS(bufferIndex);
2067         } else {
2068             int size;
2069             int flags = 0;
2070             long pts = mInputOffsetPts;
2071             if (mIsAudio) {
2072                 pts += mNumBytesSubmitted * 1000000L / (mBytesPerSample * mChannels * mSampleRate);
2073                 size = Math.min(inputBuffer.capacity(), mInputData.length - mNumBytesSubmitted);
2074                 assertTrue(size % (mBytesPerSample * mChannels) == 0);
2075                 inputBuffer.put(mInputData, mNumBytesSubmitted, size);
2076                 if (mNumBytesSubmitted + size >= mInputData.length && mSignalEOSWithLastFrame) {
2077                     flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
2078                     mSawInputEOS = true;
2079                 }
2080                 mNumBytesSubmitted += size;
2081             } else {
2082                 pts += mInputCount * 1000000L / mFrameRate;
2083                 size = mBytesPerSample * mWidth * mHeight * 3 / 2;
2084                 int frmSize = mBytesPerSample * INP_FRM_WIDTH * INP_FRM_HEIGHT * 3 / 2;
2085                 if (mNumBytesSubmitted + frmSize > mInputData.length) {
2086                     fail("received partial frame to encode");
2087                 } else {
2088                     Image img = mCodec.getInputImage(bufferIndex);
2089                     if (img != null) {
2090                         fillImage(img);
2091                     } else {
2092                         if (mWidth == INP_FRM_WIDTH && mHeight == INP_FRM_HEIGHT) {
2093                             inputBuffer.put(mInputData, mNumBytesSubmitted, size);
2094                         } else {
2095                             fillByteBuffer(inputBuffer);
2096                         }
2097                     }
2098                 }
2099                 if (mNumBytesSubmitted + frmSize >= mInputData.length && mSignalEOSWithLastFrame) {
2100                     flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
2101                     mSawInputEOS = true;
2102                 }
2103                 mNumBytesSubmitted += frmSize;
2104             }
2105             if (ENABLE_LOGS) {
2106                 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts +
2107                         " flags: " + flags);
2108             }
2109             mCodec.queueInputBuffer(bufferIndex, 0, size, pts, flags);
2110             mOutputBuff.saveInPTS(pts);
2111             mInputCount++;
2112         }
2113     }
2114 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)2115     void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) {
2116         if (ENABLE_LOGS) {
2117             Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " +
2118                     info.size + " timestamp: " + info.presentationTimeUs);
2119         }
2120         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
2121             mSawOutputEOS = true;
2122         }
2123         if (info.size > 0) {
2124             if (mSaveToMem) {
2125                 MediaCodec.BufferInfo copy = new MediaCodec.BufferInfo();
2126                 copy.set(mOutputBuff.getOutStreamSize(), info.size, info.presentationTimeUs,
2127                         info.flags);
2128                 mInfoList.add(copy);
2129 
2130                 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex);
2131                 mOutputBuff.saveToMemory(buf, info);
2132             }
2133             if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
2134                 mOutputBuff.saveOutPTS(info.presentationTimeUs);
2135                 mOutputCount++;
2136             }
2137         }
2138         mCodec.releaseOutputBuffer(bufferIndex, false);
2139     }
2140 
2141     @Override
validateMetrics(String codec, MediaFormat format)2142     PersistableBundle validateMetrics(String codec, MediaFormat format) {
2143         PersistableBundle metrics = super.validateMetrics(codec, format);
2144         assertTrue(metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE).equals(mMime));
2145         assertTrue(metrics.getInt(MediaCodec.MetricsConstants.ENCODER) == 1);
2146         return metrics;
2147     }
2148 
setUpParams(int limit)2149     void setUpParams(int limit) {
2150         int count = 0;
2151         for (int bitrate : mBitrates) {
2152             if (mIsAudio) {
2153                 for (int rate : mEncParamList1) {
2154                     for (int channels : mEncParamList2) {
2155                         MediaFormat format = new MediaFormat();
2156                         format.setString(MediaFormat.KEY_MIME, mMime);
2157                         if (mMime.equals(MediaFormat.MIMETYPE_AUDIO_FLAC)) {
2158                             format.setInteger(MediaFormat.KEY_FLAC_COMPRESSION_LEVEL, bitrate);
2159                         } else {
2160                             format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
2161                         }
2162                         format.setInteger(MediaFormat.KEY_SAMPLE_RATE, rate);
2163                         format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, channels);
2164                         mFormats.add(format);
2165                         count++;
2166                         if (count >= limit) return;
2167                     }
2168                 }
2169             } else {
2170                 assertTrue("Wrong number of height, width parameters",
2171                         mEncParamList1.length == mEncParamList2.length);
2172                 for (int i = 0; i < mEncParamList1.length; i++) {
2173                     MediaFormat format = new MediaFormat();
2174                     format.setString(MediaFormat.KEY_MIME, mMime);
2175                     format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
2176                     format.setInteger(MediaFormat.KEY_WIDTH, mEncParamList1[i]);
2177                     format.setInteger(MediaFormat.KEY_HEIGHT, mEncParamList2[i]);
2178                     format.setInteger(MediaFormat.KEY_FRAME_RATE, mFrameRate);
2179                     format.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
2180                     format.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
2181                     format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
2182                             MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
2183                     mFormats.add(format);
2184                     count++;
2185                     if (count >= limit) return;
2186                 }
2187             }
2188         }
2189     }
2190 
encodeToMemory(String file, String encoder, int frameLimit, MediaFormat format, boolean saveToMem)2191     void encodeToMemory(String file, String encoder, int frameLimit, MediaFormat format,
2192             boolean saveToMem) throws IOException, InterruptedException {
2193         mSaveToMem = saveToMem;
2194         mOutputBuff = new OutputManager();
2195         mInfoList.clear();
2196         mCodec = MediaCodec.createByCodecName(encoder);
2197         setUpSource(file);
2198         configureCodec(format, false, true, true);
2199         if (mIsAudio) {
2200             mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
2201             mChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
2202         } else {
2203             mWidth = format.getInteger(MediaFormat.KEY_WIDTH);
2204             mHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
2205         }
2206         mCodec.start();
2207         doWork(frameLimit);
2208         queueEOS();
2209         waitForAllOutputs();
2210         mCodec.stop();
2211         mCodec.release();
2212         mSaveToMem = false;
2213     }
2214 
decodeElementaryStream(String decoder, MediaFormat format, ByteBuffer elementaryStream, ArrayList<MediaCodec.BufferInfo> infos)2215     ByteBuffer decodeElementaryStream(String decoder, MediaFormat format,
2216             ByteBuffer elementaryStream, ArrayList<MediaCodec.BufferInfo> infos)
2217             throws IOException, InterruptedException {
2218         String mime = format.getString(MediaFormat.KEY_MIME);
2219         CodecDecoderTestBase cdtb = new CodecDecoderTestBase(decoder, mime, null);
2220         cdtb.mOutputBuff = new OutputManager();
2221         cdtb.mSaveToMem = true;
2222         cdtb.mCodec = MediaCodec.createByCodecName(decoder);
2223         cdtb.mCodec.configure(format, null, null, 0);
2224         cdtb.mCodec.start();
2225         cdtb.doWork(elementaryStream, infos);
2226         cdtb.queueEOS();
2227         cdtb.waitForAllOutputs();
2228         cdtb.mCodec.stop();
2229         cdtb.mCodec.release();
2230         return cdtb.mOutputBuff.getBuffer();
2231     }
2232 }
2233