• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediapc.cts;
18 
19 import static android.mediav2.common.cts.CodecTestBase.PROFILE_HLG_MAP;
20 import static android.mediapc.cts.CodecTestBase.areFormatsSupported;
21 
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24 
25 import android.media.MediaCodec;
26 import android.media.MediaCodecInfo;
27 import android.media.MediaExtractor;
28 import android.media.MediaFormat;
29 import android.util.Log;
30 import android.util.Pair;
31 import android.view.Surface;
32 
33 import java.io.IOException;
34 import java.nio.ByteBuffer;
35 import java.util.ArrayList;
36 import java.util.Objects;
37 import java.util.concurrent.Callable;
38 
39 public class CodecTranscoderTestBase {
40     private static final String LOG_TAG = CodecTranscoderTestBase.class.getSimpleName();
41     private static final boolean ENABLE_LOGS = false;
42     static final String mInpPrefix = WorkDir.getMediaDirString();
43     String mMime;
44     String mTestFile;
45     int mBitrate;
46     int mFrameRate;
47     boolean mUseHighBitDepth;
48     MediaExtractor mExtractor;
49     int mMaxBFrames;
50     int mLatency;
51 
52     MediaCodec mEncoder;
53     CodecAsyncHandler mAsyncHandleEncoder;
54     MediaCodec mDecoder;
55     CodecAsyncHandler mAsyncHandleDecoder;
56     Surface mSurface;
57 
58     boolean mSawDecInputEOS;
59     boolean mSawDecOutputEOS;
60     boolean mSawEncOutputEOS;
61     boolean mIsCodecInAsyncMode;
62     boolean mSignalEOSWithLastFrame;
63     boolean mReviseLatency;
64     int mDecInputCount;
65     int mDecOutputCount;
66     int mEncOutputCount;
67 
CodecTranscoderTestBase(String mime, String testfile, int bitrate, int frameRate, boolean useHighBitDepth)68     CodecTranscoderTestBase(String mime, String testfile, int bitrate, int frameRate,
69             boolean useHighBitDepth) {
70         mMime = mime;
71         mTestFile = testfile;
72         mBitrate = bitrate;
73         mFrameRate = frameRate;
74         mUseHighBitDepth = useHighBitDepth;
75         mMaxBFrames = 0;
76         mLatency = mMaxBFrames;
77         mReviseLatency = false;
78         mAsyncHandleDecoder = new CodecAsyncHandler();
79         mAsyncHandleEncoder = new CodecAsyncHandler();
80     }
81 
hasSeenError()82     boolean hasSeenError() {
83         return mAsyncHandleDecoder.hasSeenError() || mAsyncHandleEncoder.hasSeenError();
84     }
85 
setUpSource(String srcFile)86     MediaFormat setUpSource(String srcFile) throws IOException {
87         mExtractor = new MediaExtractor();
88         mExtractor.setDataSource(mInpPrefix + srcFile);
89         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
90             MediaFormat format = mExtractor.getTrackFormat(trackID);
91             String mime = format.getString(MediaFormat.KEY_MIME);
92             if (mime.startsWith("video/")) {
93                 mExtractor.selectTrack(trackID);
94                 format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
95                         MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
96                 format.setInteger(MediaFormat.KEY_PRIORITY, 1); // Best effort
97                 return format;
98             }
99         }
100         mExtractor.release();
101         fail("No video track found in file: " + srcFile);
102         return null;
103     }
104 
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)105     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
106         mAsyncHandleDecoder.resetContext();
107         mAsyncHandleEncoder.resetContext();
108         mIsCodecInAsyncMode = isAsync;
109         mSignalEOSWithLastFrame = signalEOSWithLastFrame;
110         mSawDecInputEOS = false;
111         mSawDecOutputEOS = false;
112         mSawEncOutputEOS = false;
113         mDecInputCount = 0;
114         mDecOutputCount = 0;
115         mEncOutputCount = 0;
116     }
117 
configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)118     void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync,
119             boolean signalEOSWithLastFrame) {
120         resetContext(isAsync, signalEOSWithLastFrame);
121         mAsyncHandleEncoder.setCallBack(mEncoder, isAsync);
122         mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
123         if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) {
124             mReviseLatency = true;
125             mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY);
126         }
127         mSurface = mEncoder.createInputSurface();
128         assertTrue("Surface is not valid", mSurface.isValid());
129         mAsyncHandleDecoder.setCallBack(mDecoder, isAsync);
130         mDecoder.configure(decFormat, mSurface, null, 0);
131         if (ENABLE_LOGS) {
132             Log.v(LOG_TAG, "codec configured");
133         }
134     }
135 
enqueueDecoderEOS(int bufferIndex)136     void enqueueDecoderEOS(int bufferIndex) {
137         if (!mSawDecInputEOS) {
138             mDecoder.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
139             mSawDecInputEOS = true;
140             if (ENABLE_LOGS) {
141                 Log.v(LOG_TAG, "Queued End of Stream");
142             }
143         }
144     }
145 
enqueueDecoderInput(int bufferIndex)146     void enqueueDecoderInput(int bufferIndex) {
147         if (mExtractor.getSampleSize() < 0) {
148             enqueueDecoderEOS(bufferIndex);
149         } else {
150             ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
151             int size = mExtractor.readSampleData(inputBuffer, 0);
152             long pts = mExtractor.getSampleTime();
153             int extractorFlags = mExtractor.getSampleFlags();
154             int codecFlags = 0;
155             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
156                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
157             }
158             if (!mExtractor.advance() && mSignalEOSWithLastFrame) {
159                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
160                 mSawDecInputEOS = true;
161             }
162             mDecoder.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
163             if (size > 0 && (codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
164                 mDecInputCount++;
165             }
166         }
167     }
168 
dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info)169     void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
170         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
171             mSawDecOutputEOS = true;
172         }
173         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
174             mDecOutputCount++;
175         }
176         mDecoder.releaseOutputBuffer(bufferIndex, mSurface != null);
177     }
178 
dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info)179     void dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
180         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
181             mSawEncOutputEOS = true;
182         }
183         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
184             mEncOutputCount++;
185         }
186         mEncoder.releaseOutputBuffer(bufferIndex, false);
187     }
188 
tryEncoderOutput(long timeOutUs)189     void tryEncoderOutput(long timeOutUs) throws InterruptedException {
190         if (mIsCodecInAsyncMode) {
191             if (!hasSeenError() && !mSawEncOutputEOS) {
192                 int retry = 0;
193                 while (mReviseLatency) {
194                     if (mAsyncHandleEncoder.hasOutputFormatChanged()) {
195                         mReviseLatency = false;
196                         int actualLatency = mAsyncHandleEncoder.getOutputFormat()
197                                 .getInteger(MediaFormat.KEY_LATENCY, mLatency);
198                         if (mLatency < actualLatency) {
199                             mLatency = actualLatency;
200                             return;
201                         }
202                     } else {
203                         if (retry > CodecTestBase.RETRY_LIMIT) throw new InterruptedException(
204                                 "did not receive output format changed for encoder after " +
205                                         CodecTestBase.Q_DEQ_TIMEOUT_US * CodecTestBase.RETRY_LIMIT +
206                                         " us");
207                         Thread.sleep(CodecTestBase.Q_DEQ_TIMEOUT_US / 1000);
208                         retry ++;
209                     }
210                 }
211                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleEncoder.getOutput();
212                 if (element != null) {
213                     dequeueEncoderOutput(element.first, element.second);
214                 }
215             }
216         } else {
217             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
218             if (!mSawEncOutputEOS) {
219                 int outputBufferId = mEncoder.dequeueOutputBuffer(outInfo, timeOutUs);
220                 if (outputBufferId >= 0) {
221                     dequeueEncoderOutput(outputBufferId, outInfo);
222                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
223                     mLatency = mEncoder.getOutputFormat()
224                             .getInteger(MediaFormat.KEY_LATENCY, mLatency);
225                 }
226             }
227         }
228     }
229 
waitForAllEncoderOutputs()230     void waitForAllEncoderOutputs() throws InterruptedException {
231         if (mIsCodecInAsyncMode) {
232             while (!hasSeenError() && !mSawEncOutputEOS) {
233                 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US);
234             }
235         } else {
236             while (!mSawEncOutputEOS) {
237                 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US);
238             }
239         }
240     }
241 
queueEOS()242     void queueEOS() throws InterruptedException {
243         if (mIsCodecInAsyncMode) {
244             while (!mAsyncHandleDecoder.hasSeenError() && !mSawDecInputEOS) {
245                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
246                 if (element != null) {
247                     int bufferID = element.first;
248                     MediaCodec.BufferInfo info = element.second;
249                     if (info != null) {
250                         dequeueDecoderOutput(bufferID, info);
251                     } else {
252                         enqueueDecoderEOS(element.first);
253                     }
254                 }
255             }
256         } else {
257             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
258             while (!mSawDecInputEOS) {
259                 int outputBufferId =
260                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
261                 if (outputBufferId >= 0) {
262                     dequeueDecoderOutput(outputBufferId, outInfo);
263                 }
264                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
265                 if (inputBufferId != -1) {
266                     enqueueDecoderEOS(inputBufferId);
267                 }
268             }
269         }
270         if (mIsCodecInAsyncMode) {
271             while (!hasSeenError() && !mSawDecOutputEOS) {
272                 Pair<Integer, MediaCodec.BufferInfo> decOp = mAsyncHandleDecoder.getOutput();
273                 if (decOp != null) dequeueDecoderOutput(decOp.first, decOp.second);
274                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
275                 if (mDecOutputCount - mEncOutputCount > mLatency) {
276                     tryEncoderOutput(-1);
277                 }
278             }
279         } else {
280             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
281             while (!mSawDecOutputEOS) {
282                 int outputBufferId =
283                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
284                 if (outputBufferId >= 0) {
285                     dequeueDecoderOutput(outputBufferId, outInfo);
286                 }
287                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
288                 if (mDecOutputCount - mEncOutputCount > mLatency) {
289                     tryEncoderOutput(-1);
290                 }
291             }
292         }
293     }
294 
doWork(int frameLimit)295     void doWork(int frameLimit) throws InterruptedException {
296         int frameCnt = 0;
297         if (mIsCodecInAsyncMode) {
298             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
299             while (!hasSeenError() && !mSawDecInputEOS && frameCnt < frameLimit) {
300                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
301                 if (element != null) {
302                     int bufferID = element.first;
303                     MediaCodec.BufferInfo info = element.second;
304                     if (info != null) {
305                         // <id, info> corresponds to output callback. Handle it accordingly
306                         dequeueDecoderOutput(bufferID, info);
307                     } else {
308                         // <id, null> corresponds to input callback. Handle it accordingly
309                         enqueueDecoderInput(bufferID);
310                         frameCnt++;
311                     }
312                 }
313                 // check decoder EOS
314                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
315                 // encoder output
316                 if (mDecOutputCount - mEncOutputCount > mLatency) {
317                     tryEncoderOutput(-1);
318                 }
319             }
320         } else {
321             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
322             while (!mSawDecInputEOS && frameCnt < frameLimit) {
323                 // decoder input
324                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
325                 if (inputBufferId != -1) {
326                     enqueueDecoderInput(inputBufferId);
327                     frameCnt++;
328                 }
329                 // decoder output
330                 int outputBufferId =
331                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
332                 if (outputBufferId >= 0) {
333                     dequeueDecoderOutput(outputBufferId, outInfo);
334                 }
335                 // check decoder EOS
336                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
337                 // encoder output
338                 if (mDecOutputCount - mEncOutputCount > mLatency) {
339                     tryEncoderOutput(-1);
340                 }
341             }
342         }
343     }
344 
setUpEncoderFormat(MediaFormat decoderFormat)345     MediaFormat setUpEncoderFormat(MediaFormat decoderFormat) {
346         MediaFormat encoderFormat = new MediaFormat();
347         encoderFormat.setString(MediaFormat.KEY_MIME, mMime);
348         encoderFormat.setInteger(MediaFormat.KEY_WIDTH,
349                 decoderFormat.getInteger(MediaFormat.KEY_WIDTH));
350         encoderFormat.setInteger(MediaFormat.KEY_HEIGHT,
351                 decoderFormat.getInteger(MediaFormat.KEY_HEIGHT));
352         encoderFormat.setInteger(MediaFormat.KEY_FRAME_RATE, mFrameRate);
353         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, mBitrate);
354         encoderFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
355         encoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
356                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
357         encoderFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
358         encoderFormat.setInteger(MediaFormat.KEY_PRIORITY,
359                 decoderFormat.getInteger(MediaFormat.KEY_PRIORITY));
360         if (mUseHighBitDepth) {
361             encoderFormat.setInteger(MediaFormat.KEY_PROFILE,
362                     Objects.requireNonNull(PROFILE_HLG_MAP.get(mMime))[0]);
363         }
364         return encoderFormat;
365     }
366 }
367 
368 /**
369  * The following class transcodes the given testFile and returns the achieved fps for transcoding.
370  */
371 class Transcode extends CodecTranscoderTestBase implements Callable<Double> {
372     private static final String LOG_TAG = Transcode.class.getSimpleName();
373 
374     private final String mDecoderName;
375     private final String mEncoderName;
376     private final boolean mIsAsync;
377 
Transcode(String mime, String testFile, String decoderName, String encoderName, boolean isAsync, boolean useHighBitDepth)378     Transcode(String mime, String testFile, String decoderName, String encoderName,
379             boolean isAsync, boolean useHighBitDepth) {
380         super(mime, testFile, 3000000, 30, useHighBitDepth);
381         mDecoderName = decoderName;
382         mEncoderName = encoderName;
383         mIsAsync = isAsync;
384     }
385 
doTranscode()386     public Double doTranscode() throws Exception {
387         MediaFormat decoderFormat = setUpSource(mTestFile);
388         ArrayList<MediaFormat> formats = new ArrayList<>();
389         formats.add(decoderFormat);
390         // If the decoder doesn't support the formats, then return 0 to indicate that decode failed
391         if (!areFormatsSupported(mDecoderName, formats)) {
392             return (Double) 0.0;
393         }
394 
395         mDecoder = MediaCodec.createByCodecName(mDecoderName);
396         MediaFormat encoderFormat = setUpEncoderFormat(decoderFormat);
397         mEncoder = MediaCodec.createByCodecName(mEncoderName);
398         mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
399         configureCodec(decoderFormat, encoderFormat, mIsAsync, false);
400         mEncoder.start();
401         mDecoder.start();
402         long start = System.currentTimeMillis();
403         doWork(Integer.MAX_VALUE);
404         queueEOS();
405         waitForAllEncoderOutputs();
406         long end = System.currentTimeMillis();
407         mSurface.release();
408         mDecoder.stop();
409         mDecoder.release();
410         mEncoder.stop();
411         mEncoder.release();
412         mExtractor.release();
413         double fps = mEncOutputCount / ((end - start) / 1000.0);
414         Log.d(LOG_TAG, "Mime: " + mMime + " Decoder: " + mDecoderName + " Encoder: " +
415                 mEncoderName + " Achieved fps: " + fps);
416         return fps;
417     }
418 
419     @Override
call()420     public Double call() throws Exception {
421         return doTranscode();
422     }
423 }
424 
425 /**
426  * The following class transcodes the given testFile until loadStatus is finished.
427  * If input reaches eos, it will rewind the input to start position.
428  */
429 class TranscodeLoad extends Transcode {
430     private final LoadStatus mLoadStatus;
431 
432     private long mMaxPts;
433     private long mBasePts;
434 
TranscodeLoad(String mime, String testFile, String decoderName, String encoderName, LoadStatus loadStatus)435     TranscodeLoad(String mime, String testFile, String decoderName, String encoderName,
436             LoadStatus loadStatus) {
437         super(mime, testFile, decoderName, encoderName, false, false);
438         mLoadStatus = loadStatus;
439         mMaxPts = 0;
440         mBasePts = 0;
441     }
442 
443     @Override
configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)444     void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync,
445             boolean signalEOSWithLastFrame) {
446         decFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
447         encFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
448         resetContext(isAsync, signalEOSWithLastFrame);
449         mAsyncHandleEncoder.setCallBack(mEncoder, isAsync);
450         mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
451         if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) {
452             mReviseLatency = true;
453             mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY);
454         }
455         mSurface = mEncoder.createInputSurface();
456         assertTrue("Surface is not valid", mSurface.isValid());
457         mAsyncHandleDecoder.setCallBack(mDecoder, isAsync);
458         mDecoder.configure(decFormat, mSurface, null, 0);
459     }
460 
461     @Override
enqueueDecoderInput(int bufferIndex)462     void enqueueDecoderInput(int bufferIndex) {
463         if (mExtractor.getSampleSize() < 0 || mLoadStatus.isLoadFinished()) {
464             enqueueDecoderEOS(bufferIndex);
465         } else {
466             ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
467             int size = mExtractor.readSampleData(inputBuffer, 0);
468             long pts = mExtractor.getSampleTime();
469             mMaxPts = Math.max(mMaxPts, mBasePts + pts);
470             int extractorFlags = mExtractor.getSampleFlags();
471             int codecFlags = 0;
472             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
473                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
474             }
475             mDecoder.queueInputBuffer(bufferIndex, 0, size, mBasePts + pts, codecFlags);
476             if (size > 0 && (codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
477                 mDecInputCount++;
478             }
479             // If eos is reached, seek to start position.
480             if (!mExtractor.advance()) {
481                 mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
482                 mBasePts = mMaxPts + 1000000L;
483             }
484         }
485     }
486 }
487 
488 /**
489  * The following class tells the status of the load whether it is finished or not.
490  */
491 class LoadStatus {
492     private boolean mLoadFinished;
493 
LoadStatus()494     public LoadStatus() { mLoadFinished = false; }
495 
setLoadFinished()496     public synchronized void setLoadFinished() { mLoadFinished = true; }
497 
isLoadFinished()498     public synchronized boolean isLoadFinished() { return mLoadFinished; }
499 }
500