1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts; 17 18 import android.media.AudioTrack; 19 import android.media.MediaCodec; 20 import android.media.MediaExtractor; 21 import android.media.MediaFormat; 22 import android.util.Log; 23 24 import java.nio.ByteBuffer; 25 import java.util.LinkedList; 26 27 /** 28 * Class for directly managing both audio and video playback by 29 * using {@link MediaCodec} and {@link AudioTrack}. 30 */ 31 public class CodecState { 32 private static final String TAG = CodecState.class.getSimpleName(); 33 34 private boolean mSawInputEOS, mSawOutputEOS; 35 private boolean mLimitQueueDepth; 36 private ByteBuffer[] mCodecInputBuffers; 37 private ByteBuffer[] mCodecOutputBuffers; 38 private int mTrackIndex; 39 private LinkedList<Integer> mAvailableInputBufferIndices; 40 private LinkedList<Integer> mAvailableOutputBufferIndices; 41 private LinkedList<MediaCodec.BufferInfo> mAvailableOutputBufferInfos; 42 private long mPresentationTimeUs; 43 private MediaCodec mCodec; 44 private MediaCodecCencPlayer mMediaCodecPlayer; 45 private MediaExtractor mExtractor; 46 private MediaFormat mFormat; 47 private MediaFormat mOutputFormat; 48 private NonBlockingAudioTrack mAudioTrack; 49 50 /** 51 * Manages audio and video playback using MediaCodec and AudioTrack. 52 */ CodecState( MediaCodecCencPlayer mediaCodecPlayer, MediaExtractor extractor, int trackIndex, MediaFormat format, MediaCodec codec, boolean limitQueueDepth)53 public CodecState( 54 MediaCodecCencPlayer mediaCodecPlayer, 55 MediaExtractor extractor, 56 int trackIndex, 57 MediaFormat format, 58 MediaCodec codec, 59 boolean limitQueueDepth) { 60 61 mMediaCodecPlayer = mediaCodecPlayer; 62 mExtractor = extractor; 63 mTrackIndex = trackIndex; 64 mFormat = format; 65 mSawInputEOS = mSawOutputEOS = false; 66 mLimitQueueDepth = limitQueueDepth; 67 68 mCodec = codec; 69 70 mAvailableInputBufferIndices = new LinkedList<Integer>(); 71 mAvailableOutputBufferIndices = new LinkedList<Integer>(); 72 mAvailableOutputBufferInfos = new LinkedList<MediaCodec.BufferInfo>(); 73 74 mPresentationTimeUs = 0; 75 } 76 release()77 public void release() { 78 mCodec.stop(); 79 mCodecInputBuffers = null; 80 mCodecOutputBuffers = null; 81 mOutputFormat = null; 82 83 mAvailableInputBufferIndices.clear(); 84 mAvailableOutputBufferIndices.clear(); 85 mAvailableOutputBufferInfos.clear(); 86 87 mAvailableInputBufferIndices = null; 88 mAvailableOutputBufferIndices = null; 89 mAvailableOutputBufferInfos = null; 90 91 mCodec.release(); 92 mCodec = null; 93 94 if (mAudioTrack != null) { 95 mAudioTrack.release(); 96 mAudioTrack = null; 97 } 98 } 99 start()100 public void start() { 101 mCodec.start(); 102 mCodecInputBuffers = mCodec.getInputBuffers(); 103 mCodecOutputBuffers = mCodec.getOutputBuffers(); 104 105 if (mAudioTrack != null) { 106 mAudioTrack.play(); 107 } 108 } 109 pause()110 public void pause() { 111 if (mAudioTrack != null) { 112 mAudioTrack.pause(); 113 } 114 } 115 getCurrentPositionUs()116 public long getCurrentPositionUs() { 117 return mPresentationTimeUs; 118 } 119 flush()120 public void flush() { 121 mAvailableInputBufferIndices.clear(); 122 mAvailableOutputBufferIndices.clear(); 123 mAvailableOutputBufferInfos.clear(); 124 125 mSawInputEOS = false; 126 mSawOutputEOS = false; 127 128 if (mAudioTrack != null 129 && mAudioTrack.getPlayState() == AudioTrack.PLAYSTATE_STOPPED) { 130 mAudioTrack.play(); 131 } 132 133 mCodec.flush(); 134 } 135 isEnded()136 public boolean isEnded() { 137 return mSawInputEOS && mSawOutputEOS; 138 } 139 140 /** 141 * doSomeWork() is the worker function that does all buffer handling and decoding works. 142 * It first reads data from {@link MediaExtractor} and pushes it into {@link MediaCodec}; 143 * it then dequeues buffer from {@link MediaCodec}, consumes it and pushes back to its own 144 * buffer queue for next round reading data from {@link MediaExtractor}. 145 */ doSomeWork()146 public void doSomeWork() { 147 int indexInput = mCodec.dequeueInputBuffer(0 /* timeoutUs */); 148 149 if (indexInput != MediaCodec.INFO_TRY_AGAIN_LATER) { 150 mAvailableInputBufferIndices.add(indexInput); 151 } 152 153 while (feedInputBuffer()) { 154 } 155 156 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 157 int indexOutput = mCodec.dequeueOutputBuffer(info, 0 /* timeoutUs */); 158 159 if (indexOutput == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 160 mOutputFormat = mCodec.getOutputFormat(); 161 onOutputFormatChanged(); 162 } else if (indexOutput == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 163 mCodecOutputBuffers = mCodec.getOutputBuffers(); 164 } else if (indexOutput != MediaCodec.INFO_TRY_AGAIN_LATER) { 165 mAvailableOutputBufferIndices.add(indexOutput); 166 mAvailableOutputBufferInfos.add(info); 167 } 168 169 while (drainOutputBuffer()) { 170 } 171 } 172 173 /** Returns true if more input data could be fed. */ feedInputBuffer()174 private boolean feedInputBuffer() throws MediaCodec.CryptoException, IllegalStateException { 175 if (mSawInputEOS || mAvailableInputBufferIndices.isEmpty()) { 176 return false; 177 } 178 179 // stalls read if audio queue is larger than 2MB full so we will not occupy too much heap 180 if (mLimitQueueDepth && mAudioTrack != null && 181 mAudioTrack.getNumBytesQueued() > 2 * 1024 * 1024) { 182 return false; 183 } 184 185 int index = mAvailableInputBufferIndices.peekFirst().intValue(); 186 187 ByteBuffer codecData = mCodecInputBuffers[index]; 188 189 int trackIndex = mExtractor.getSampleTrackIndex(); 190 191 if (trackIndex == mTrackIndex) { 192 int sampleSize = 193 mExtractor.readSampleData(codecData, 0 /* offset */); 194 195 long sampleTime = mExtractor.getSampleTime(); 196 197 int sampleFlags = mExtractor.getSampleFlags(); 198 199 if (sampleSize <= 0) { 200 Log.d(TAG, "sampleSize: " + sampleSize + " trackIndex:" + trackIndex + 201 " sampleTime:" + sampleTime + " sampleFlags:" + sampleFlags); 202 mSawInputEOS = true; 203 return false; 204 } 205 206 if ((sampleFlags & MediaExtractor.SAMPLE_FLAG_ENCRYPTED) != 0) { 207 MediaCodec.CryptoInfo info = new MediaCodec.CryptoInfo(); 208 mExtractor.getSampleCryptoInfo(info); 209 210 mCodec.queueSecureInputBuffer( 211 index, 0 /* offset */, info, sampleTime, 0 /* flags */); 212 } else { 213 mCodec.queueInputBuffer( 214 index, 0 /* offset */, sampleSize, sampleTime, 0 /* flags */); 215 } 216 217 mAvailableInputBufferIndices.removeFirst(); 218 mExtractor.advance(); 219 220 return true; 221 } else if (trackIndex < 0) { 222 Log.d(TAG, "saw input EOS on track " + mTrackIndex); 223 224 mSawInputEOS = true; 225 226 mCodec.queueInputBuffer( 227 index, 0 /* offset */, 0 /* sampleSize */, 228 0 /* sampleTime */, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 229 230 mAvailableInputBufferIndices.removeFirst(); 231 } 232 233 return false; 234 } 235 onOutputFormatChanged()236 private void onOutputFormatChanged() { 237 String mime = mOutputFormat.getString(MediaFormat.KEY_MIME); 238 // b/9250789 239 Log.d(TAG, "CodecState::onOutputFormatChanged " + mime); 240 241 if (mime.startsWith("audio/")) { 242 int sampleRate = 243 mOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); 244 245 int channelCount = 246 mOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 247 248 Log.d(TAG, "CodecState::onOutputFormatChanged Audio" + 249 " sampleRate:" + sampleRate + " channels:" + channelCount); 250 // We do sanity check here after we receive data from MediaExtractor and before 251 // we pass them down to AudioTrack. If MediaExtractor works properly, this 252 // sanity-check is not necessary, however, in our tests, we found that there 253 // are a few cases where ch=0 and samplerate=0 were returned by MediaExtractor. 254 if (channelCount < 1 || channelCount > 8 || 255 sampleRate < 8000 || sampleRate > 128000) { 256 return; 257 } 258 mAudioTrack = new NonBlockingAudioTrack(sampleRate, channelCount); 259 mAudioTrack.play(); 260 } 261 262 if (mime.startsWith("video/")) { 263 int width = mOutputFormat.getInteger(MediaFormat.KEY_WIDTH); 264 int height = mOutputFormat.getInteger(MediaFormat.KEY_HEIGHT); 265 Log.d(TAG, "CodecState::onOutputFormatChanged Video" + 266 " width:" + width + " height:" + height); 267 } 268 } 269 270 /** Returns true if more output data could be drained. */ drainOutputBuffer()271 private boolean drainOutputBuffer() { 272 if (mSawOutputEOS || mAvailableOutputBufferIndices.isEmpty()) { 273 return false; 274 } 275 276 int index = mAvailableOutputBufferIndices.peekFirst().intValue(); 277 MediaCodec.BufferInfo info = mAvailableOutputBufferInfos.peekFirst(); 278 279 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 280 Log.d(TAG, "saw output EOS on track " + mTrackIndex); 281 282 mSawOutputEOS = true; 283 284 return false; 285 } 286 287 long realTimeUs = 288 mMediaCodecPlayer.getRealTimeUsForMediaTime(info.presentationTimeUs); 289 290 long nowUs = mMediaCodecPlayer.getNowUs(); 291 292 long lateUs = nowUs - realTimeUs; 293 294 if (mAudioTrack != null) { 295 ByteBuffer buffer = mCodecOutputBuffers[index]; 296 buffer.clear(); 297 buffer.position(0 /* offset */); 298 299 byte[] audioCopy = new byte[info.size]; 300 buffer.get(audioCopy, 0, info.size); 301 302 mAudioTrack.write(audioCopy, info.size); 303 304 mCodec.releaseOutputBuffer(index, false /* render */); 305 306 mPresentationTimeUs = info.presentationTimeUs; 307 308 mAvailableOutputBufferIndices.removeFirst(); 309 mAvailableOutputBufferInfos.removeFirst(); 310 return true; 311 } else { 312 // video 313 boolean render; 314 315 if (lateUs < -45000) { 316 // too early; 317 return false; 318 } else if (lateUs > 30000) { 319 Log.d(TAG, "video late by " + lateUs + " us."); 320 render = false; 321 } else { 322 render = true; 323 mPresentationTimeUs = info.presentationTimeUs; 324 } 325 326 mCodec.releaseOutputBuffer(index, render); 327 328 mAvailableOutputBufferIndices.removeFirst(); 329 mAvailableOutputBufferInfos.removeFirst(); 330 return true; 331 } 332 } 333 getAudioTimeUs()334 public long getAudioTimeUs() { 335 if (mAudioTrack == null) { 336 return 0; 337 } 338 339 return mAudioTrack.getAudioTimeUs(); 340 } 341 process()342 public void process() { 343 if (mAudioTrack != null) { 344 mAudioTrack.process(); 345 } 346 } 347 } 348