1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.codec.cts; 18 19 import android.annotation.TargetApi; 20 import android.content.res.AssetFileDescriptor; 21 import android.media.MediaCodec; 22 import android.media.MediaCodecInfo; 23 import android.media.MediaCodecInfo.CodecCapabilities; 24 import android.media.MediaCodecInfo.CodecProfileLevel; 25 import android.media.MediaCodecList; 26 import android.media.MediaExtractor; 27 import android.media.MediaFormat; 28 import android.media.MediaMuxer; 29 import android.media.MediaPlayer; 30 import android.media.cts.InputSurface; 31 import android.media.cts.MediaStubActivity; 32 import android.media.cts.OutputSurface; 33 import android.media.cts.Preconditions; 34 import android.os.Environment; 35 import android.os.ParcelFileDescriptor; 36 import android.platform.test.annotations.AppModeFull; 37 import android.test.ActivityInstrumentationTestCase2; 38 import android.util.Log; 39 import android.view.Surface; 40 41 import android.media.MediaCodecInfo; 42 import android.media.MediaCodecInfo.CodecCapabilities; 43 import android.media.MediaCodecInfo.CodecProfileLevel; 44 45 import java.io.File; 46 import java.io.FileNotFoundException; 47 import java.io.IOException; 48 import java.nio.ByteBuffer; 49 import java.util.concurrent.atomic.AtomicReference; 50 import java.util.concurrent.CountDownLatch; 51 52 /** 53 * Test for the integration of MediaMuxer and MediaCodec's encoder. 54 * 55 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a 56 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write 57 * them into a file. 58 * 59 * <p>It does not currently check whether the result file is correct, but makes sure that nothing 60 * fails along the way. 61 * 62 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the 63 * MediaMuxer. 64 */ 65 @TargetApi(18) 66 @AppModeFull(reason = "Instant apps cannot access the SD card") 67 public class ExtractDecodeEditEncodeMuxTest 68 extends ActivityInstrumentationTestCase2<MediaStubActivity> { 69 70 private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); 71 private static final boolean VERBOSE = false; // lots of logging 72 static final String mInpPrefix = WorkDir.getMediaDirString(); 73 74 /** How long to wait for the next buffer to become available. */ 75 private static final int TIMEOUT_USEC = 10000; 76 77 /** Where to output the test files. */ 78 private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); 79 80 // parameters for the video encoder 81 private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps 82 private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // 30fps 83 private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames 84 private static final int OUTPUT_VIDEO_COLOR_FORMAT = 85 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 86 87 // parameters for the audio encoder 88 // Advanced Audio Coding 89 private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; 90 private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. 91 private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; 92 private static final int OUTPUT_AUDIO_AAC_PROFILE = 93 MediaCodecInfo.CodecProfileLevel.AACObjectHE; 94 private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. 95 96 /** 97 * Used for editing the frames. 98 * 99 * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer. 100 */ 101 private static final String FRAGMENT_SHADER = 102 "#extension GL_OES_EGL_image_external : require\n" + 103 "precision mediump float;\n" + 104 "varying vec2 vTextureCoord;\n" + 105 "uniform samplerExternalOES sTexture;\n" + 106 "void main() {\n" + 107 " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + 108 "}\n"; 109 110 /** Whether to copy the video from the test video. */ 111 private boolean mCopyVideo; 112 /** Whether to copy the audio from the test video. */ 113 private boolean mCopyAudio; 114 /** Whether to verify the audio format. */ 115 private boolean mVerifyAudioFormat; 116 /** Width of the output frames. */ 117 private int mWidth = -1; 118 /** Height of the output frames. */ 119 private int mHeight = -1; 120 121 /** The raw resource used as the input file. */ 122 private String mSourceRes; 123 124 /** The destination file for the encoded output. */ 125 private String mOutputFile; 126 127 private String mOutputVideoMimeType; 128 ExtractDecodeEditEncodeMuxTest()129 public ExtractDecodeEditEncodeMuxTest() { 130 super(MediaStubActivity.class); 131 } 132 testExtractDecodeEditEncodeMuxQCIF()133 public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { 134 if(!setSize(176, 144)) return; 135 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 136 setCopyVideo(); 137 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 138 TestWrapper.runTest(this); 139 } 140 testExtractDecodeEditEncodeMuxQVGA()141 public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { 142 if(!setSize(320, 240)) return; 143 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 144 setCopyVideo(); 145 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 146 TestWrapper.runTest(this); 147 } 148 testExtractDecodeEditEncodeMux720p()149 public void testExtractDecodeEditEncodeMux720p() throws Throwable { 150 if(!setSize(1280, 720)) return; 151 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 152 setCopyVideo(); 153 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 154 TestWrapper.runTest(this); 155 } 156 testExtractDecodeEditEncodeMux2160pHevc()157 public void testExtractDecodeEditEncodeMux2160pHevc() throws Throwable { 158 if(!setSize(3840, 2160)) return; 159 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 160 setCopyVideo(); 161 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_HEVC); 162 TestWrapper.runTest(this); 163 } 164 testExtractDecodeEditEncodeMuxAudio()165 public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { 166 if(!setSize(1280, 720)) return; 167 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 168 setCopyAudio(); 169 setVerifyAudioFormat(); 170 TestWrapper.runTest(this); 171 } 172 testExtractDecodeEditEncodeMuxAudioVideo()173 public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { 174 if(!setSize(1280, 720)) return; 175 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 176 setCopyAudio(); 177 setCopyVideo(); 178 setVerifyAudioFormat(); 179 TestWrapper.runTest(this); 180 } 181 182 /** Wraps testExtractDecodeEditEncodeMux() */ 183 private static class TestWrapper implements Runnable { 184 private Throwable mThrowable; 185 private ExtractDecodeEditEncodeMuxTest mTest; 186 TestWrapper(ExtractDecodeEditEncodeMuxTest test)187 private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { 188 mTest = test; 189 } 190 191 @Override run()192 public void run() { 193 try { 194 mTest.extractDecodeEditEncodeMux(); 195 } catch (Throwable th) { 196 mThrowable = th; 197 } 198 } 199 200 /** 201 * Entry point. 202 */ runTest(ExtractDecodeEditEncodeMuxTest test)203 public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { 204 test.setOutputFile(); 205 TestWrapper wrapper = new TestWrapper(test); 206 Thread th = new Thread(wrapper, "codec test"); 207 th.start(); 208 th.join(); 209 if (wrapper.mThrowable != null) { 210 throw wrapper.mThrowable; 211 } 212 } 213 } 214 215 /** 216 * Sets the test to copy the video stream. 217 */ setCopyVideo()218 private void setCopyVideo() { 219 mCopyVideo = true; 220 } 221 222 /** 223 * Sets the test to copy the video stream. 224 */ setCopyAudio()225 private void setCopyAudio() { 226 mCopyAudio = true; 227 } 228 229 /** 230 * Sets the test to verify the output audio format. 231 */ setVerifyAudioFormat()232 private void setVerifyAudioFormat() { 233 mVerifyAudioFormat = true; 234 } 235 236 /** 237 * Sets the desired frame size and returns whether the given resolution is 238 * supported. 239 * 240 * <p>If decoding/encoding using AVC as the codec, checks that the resolution 241 * is supported. For other codecs, always return {@code true}. 242 */ setSize(int width, int height)243 private boolean setSize(int width, int height) { 244 if ((width % 16) != 0 || (height % 16) != 0) { 245 Log.w(TAG, "WARNING: width or height not multiple of 16"); 246 } 247 mWidth = width; 248 mHeight = height; 249 250 // TODO: remove this logic in setSize as it is now handled when configuring codecs 251 return true; 252 } 253 254 /** 255 * Sets the raw resource used as the source video. 256 */ setSource(String res)257 private void setSource(String res) { 258 mSourceRes = res; 259 } 260 261 /** 262 * Sets the name of the output file based on the other parameters. 263 * 264 * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(String)}. 265 */ setOutputFile()266 private void setOutputFile() { 267 StringBuilder sb = new StringBuilder(); 268 sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); 269 sb.append("/cts-media-"); 270 sb.append(getClass().getSimpleName()); 271 assertTrue("should have called setSource() first", mSourceRes != null); 272 sb.append('-'); 273 sb.append(mSourceRes); 274 if (mCopyVideo) { 275 assertTrue("should have called setSize() first", mWidth != -1); 276 assertTrue("should have called setSize() first", mHeight != -1); 277 sb.append('-'); 278 sb.append("video"); 279 sb.append('-'); 280 sb.append(mWidth); 281 sb.append('x'); 282 sb.append(mHeight); 283 } 284 if (mCopyAudio) { 285 sb.append('-'); 286 sb.append("audio"); 287 } 288 sb.append(".mp4"); 289 mOutputFile = sb.toString(); 290 } 291 setVideoMimeType(String mimeType)292 private void setVideoMimeType(String mimeType) { 293 mOutputVideoMimeType = mimeType; 294 } 295 296 /** 297 * Tests encoding and subsequently decoding video from frames generated into a buffer. 298 * <p> 299 * We encode several frames of a video test pattern using MediaCodec, then decode the output 300 * with MediaCodec and do some simple checks. 301 */ extractDecodeEditEncodeMux()302 private void extractDecodeEditEncodeMux() throws Exception { 303 // Exception that may be thrown during release. 304 Exception exception = null; 305 306 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 307 308 // We avoid the device-specific limitations on width and height by using values 309 // that are multiples of 16, which all tested devices seem to be able to handle. 310 MediaFormat outputVideoFormat = 311 MediaFormat.createVideoFormat(mOutputVideoMimeType, mWidth, mHeight); 312 313 // Set some properties. Failing to specify some of these can cause the MediaCodec 314 // configure() call to throw an unhelpful exception. 315 outputVideoFormat.setInteger( 316 MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT); 317 outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE); 318 outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); 319 outputVideoFormat.setInteger( 320 MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); 321 if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); 322 323 String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat); 324 if (videoEncoderName == null) { 325 // Don't fail CTS if they don't have an AVC codec (not here, anyway). 326 Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat); 327 return; 328 } 329 if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName); 330 331 MediaFormat outputAudioFormat = 332 MediaFormat.createAudioFormat( 333 OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, 334 OUTPUT_AUDIO_CHANNEL_COUNT); 335 outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE); 336 // TODO: Bug workaround --- uncomment once fixed. 337 // outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); 338 339 String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat); 340 if (audioEncoderName == null) { 341 // Don't fail CTS if they don't have an AAC codec (not here, anyway). 342 Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat); 343 return; 344 } 345 if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName); 346 347 MediaExtractor videoExtractor = null; 348 MediaExtractor audioExtractor = null; 349 OutputSurface outputSurface = null; 350 MediaCodec videoDecoder = null; 351 MediaCodec audioDecoder = null; 352 MediaCodec videoEncoder = null; 353 MediaCodec audioEncoder = null; 354 MediaMuxer muxer = null; 355 356 InputSurface inputSurface = null; 357 358 try { 359 if (mCopyVideo) { 360 videoExtractor = createExtractor(); 361 int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); 362 assertTrue("missing video track in test video", videoInputTrack != -1); 363 MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack); 364 365 // Create a MediaCodec for the desired codec, then configure it as an encoder with 366 // our desired properties. Request a Surface to use for input. 367 AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>(); 368 videoEncoder = createVideoEncoder( 369 videoEncoderName, outputVideoFormat, inputSurfaceReference); 370 inputSurface = new InputSurface(inputSurfaceReference.get()); 371 inputSurface.makeCurrent(); 372 // Create a MediaCodec for the decoder, based on the extractor's format. 373 outputSurface = new OutputSurface(); 374 outputSurface.changeFragmentShader(FRAGMENT_SHADER); 375 videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface()); 376 } 377 378 if (mCopyAudio) { 379 audioExtractor = createExtractor(); 380 int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); 381 assertTrue("missing audio track in test video", audioInputTrack != -1); 382 MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack); 383 384 // Create a MediaCodec for the desired codec, then configure it as an encoder with 385 // our desired properties. Request a Surface to use for input. 386 audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat); 387 // Create a MediaCodec for the decoder, based on the extractor's format. 388 audioDecoder = createAudioDecoder(mcl, inputFormat); 389 } 390 391 // Creates a muxer but do not start or add tracks just yet. 392 muxer = createMuxer(); 393 394 doExtractDecodeEditEncodeMux( 395 videoExtractor, 396 audioExtractor, 397 videoDecoder, 398 videoEncoder, 399 audioDecoder, 400 audioEncoder, 401 muxer, 402 inputSurface, 403 outputSurface); 404 } finally { 405 if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); 406 // Try to release everything we acquired, even if one of the releases fails, in which 407 // case we save the first exception we got and re-throw at the end (unless something 408 // other exception has already been thrown). This guarantees the first exception thrown 409 // is reported as the cause of the error, everything is (attempted) to be released, and 410 // all other exceptions appear in the logs. 411 try { 412 if (videoExtractor != null) { 413 videoExtractor.release(); 414 } 415 } catch(Exception e) { 416 Log.e(TAG, "error while releasing videoExtractor", e); 417 if (exception == null) { 418 exception = e; 419 } 420 } 421 try { 422 if (audioExtractor != null) { 423 audioExtractor.release(); 424 } 425 } catch(Exception e) { 426 Log.e(TAG, "error while releasing audioExtractor", e); 427 if (exception == null) { 428 exception = e; 429 } 430 } 431 try { 432 if (videoDecoder != null) { 433 videoDecoder.stop(); 434 videoDecoder.release(); 435 } 436 } catch(Exception e) { 437 Log.e(TAG, "error while releasing videoDecoder", e); 438 if (exception == null) { 439 exception = e; 440 } 441 } 442 try { 443 if (outputSurface != null) { 444 outputSurface.release(); 445 } 446 } catch(Exception e) { 447 Log.e(TAG, "error while releasing outputSurface", e); 448 if (exception == null) { 449 exception = e; 450 } 451 } 452 try { 453 if (videoEncoder != null) { 454 videoEncoder.stop(); 455 videoEncoder.release(); 456 } 457 } catch(Exception e) { 458 Log.e(TAG, "error while releasing videoEncoder", e); 459 if (exception == null) { 460 exception = e; 461 } 462 } 463 try { 464 if (audioDecoder != null) { 465 audioDecoder.stop(); 466 audioDecoder.release(); 467 } 468 } catch(Exception e) { 469 Log.e(TAG, "error while releasing audioDecoder", e); 470 if (exception == null) { 471 exception = e; 472 } 473 } 474 try { 475 if (audioEncoder != null) { 476 audioEncoder.stop(); 477 audioEncoder.release(); 478 } 479 } catch(Exception e) { 480 Log.e(TAG, "error while releasing audioEncoder", e); 481 if (exception == null) { 482 exception = e; 483 } 484 } 485 try { 486 if (muxer != null) { 487 muxer.stop(); 488 muxer.release(); 489 } 490 } catch(Exception e) { 491 Log.e(TAG, "error while releasing muxer", e); 492 if (exception == null) { 493 exception = e; 494 } 495 } 496 try { 497 if (inputSurface != null) { 498 inputSurface.release(); 499 } 500 } catch(Exception e) { 501 Log.e(TAG, "error while releasing inputSurface", e); 502 if (exception == null) { 503 exception = e; 504 } 505 } 506 } 507 if (exception != null) { 508 throw exception; 509 } 510 511 MediaExtractor mediaExtractor = null; 512 try { 513 mediaExtractor = new MediaExtractor(); 514 mediaExtractor.setDataSource(mOutputFile); 515 516 assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0), 517 mediaExtractor.getTrackCount()); 518 if (mVerifyAudioFormat) { 519 boolean foundAudio = false; 520 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 521 MediaFormat trackFormat = mediaExtractor.getTrackFormat(i); 522 if (isAudioFormat(trackFormat)) { 523 foundAudio = true; 524 int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ; 525 526 // SBR mode halves the sample rate in the format. 527 // Query output profile. KEY_PROFILE gets precedence over KEY_AAC_PROFILE 528 int aac_profile = trackFormat.getInteger(MediaFormat.KEY_AAC_PROFILE, -1); 529 int profile = trackFormat.getInteger(MediaFormat.KEY_PROFILE, aac_profile); 530 531 if (profile == MediaCodecInfo.CodecProfileLevel.AACObjectHE) { 532 expectedSampleRate /= 2; 533 } 534 assertEquals("sample rates should match", expectedSampleRate, 535 trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 536 } 537 } 538 539 assertTrue("output should have an audio track", foundAudio || !mCopyAudio); 540 } 541 } catch (IOException e) { 542 throw new IllegalStateException("exception verifying output file", e); 543 } finally { 544 if (mediaExtractor != null) { 545 mediaExtractor.release(); 546 } 547 } 548 549 // TODO: Check the generated output file's video format and sample data. 550 551 MediaStubActivity activity = getActivity(); 552 final MediaPlayer mp = new MediaPlayer(); 553 final Exception[] exceptionHolder = { null }; 554 final CountDownLatch playbackEndSignal = new CountDownLatch(1); 555 mp.setOnErrorListener(new MediaPlayer.OnErrorListener() { 556 @Override 557 public boolean onError(MediaPlayer origin, int what, int extra) { 558 exceptionHolder[0] = new RuntimeException("error playing output file: what=" + what 559 + " extra=" + extra); 560 // Returning false would trigger onCompletion() so that 561 // playbackEndSignal.await() can stop waiting. 562 return false; 563 } 564 }); 565 mp.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { 566 @Override 567 public void onCompletion(MediaPlayer origin) { 568 playbackEndSignal.countDown(); 569 } 570 }); 571 try { 572 mp.setDataSource(mOutputFile); 573 mp.setDisplay(activity.getSurfaceHolder()); 574 mp.prepare(); 575 mp.start(); 576 playbackEndSignal.await(); 577 } catch (Exception e) { 578 exceptionHolder[0] = e; 579 } finally { 580 mp.release(); 581 } 582 583 if (exceptionHolder[0] != null) { 584 throw exceptionHolder[0]; 585 } 586 } 587 getAssetFileDescriptorFor(final String res)588 protected AssetFileDescriptor getAssetFileDescriptorFor(final String res) 589 throws FileNotFoundException { 590 Preconditions.assertTestFileExists(mInpPrefix + res); 591 File inpFile = new File(mInpPrefix + res); 592 ParcelFileDescriptor parcelFD = 593 ParcelFileDescriptor.open(inpFile, ParcelFileDescriptor.MODE_READ_ONLY); 594 return new AssetFileDescriptor(parcelFD, 0, parcelFD.getStatSize()); 595 } 596 597 /** 598 * Creates an extractor that reads its frames from {@link #mSourceRes}. 599 */ createExtractor()600 private MediaExtractor createExtractor() throws IOException { 601 MediaExtractor extractor; 602 AssetFileDescriptor srcFd = getAssetFileDescriptorFor(mSourceRes); 603 extractor = new MediaExtractor(); 604 extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(), 605 srcFd.getLength()); 606 return extractor; 607 } 608 609 /** 610 * Creates a decoder for the given format, which outputs to the given surface. 611 * 612 * @param inputFormat the format of the stream to decode 613 * @param surface into which to decode the frames 614 */ createVideoDecoder( MediaCodecList mcl, MediaFormat inputFormat, Surface surface)615 private MediaCodec createVideoDecoder( 616 MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException { 617 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 618 decoder.configure(inputFormat, surface, null, 0); 619 decoder.start(); 620 return decoder; 621 } 622 623 /** 624 * Creates an encoder for the given format using the specified codec, taking input from a 625 * surface. 626 * 627 * <p>The surface to use as input is stored in the given reference. 628 * 629 * @param codecInfo of the codec to use 630 * @param format of the stream to be produced 631 * @param surfaceReference to store the surface to use as input 632 */ createVideoEncoder( String codecName, MediaFormat format, AtomicReference<Surface> surfaceReference)633 private MediaCodec createVideoEncoder( 634 String codecName, 635 MediaFormat format, 636 AtomicReference<Surface> surfaceReference) 637 throws IOException { 638 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 639 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 640 // Must be called before start() is. 641 surfaceReference.set(encoder.createInputSurface()); 642 encoder.start(); 643 return encoder; 644 } 645 646 /** 647 * Creates a decoder for the given format. 648 * 649 * @param inputFormat the format of the stream to decode 650 */ createAudioDecoder( MediaCodecList mcl, MediaFormat inputFormat)651 private MediaCodec createAudioDecoder( 652 MediaCodecList mcl, MediaFormat inputFormat) throws IOException { 653 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 654 decoder.configure(inputFormat, null, null, 0); 655 decoder.start(); 656 return decoder; 657 } 658 659 /** 660 * Creates an encoder for the given format using the specified codec. 661 * 662 * @param codecInfo of the codec to use 663 * @param format of the stream to be produced 664 */ createAudioEncoder(String codecName, MediaFormat format)665 private MediaCodec createAudioEncoder(String codecName, MediaFormat format) 666 throws IOException { 667 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 668 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 669 encoder.start(); 670 return encoder; 671 } 672 673 /** 674 * Creates a muxer to write the encoded frames. 675 * 676 * <p>The muxer is not started as it needs to be started only after all streams have been added. 677 */ createMuxer()678 private MediaMuxer createMuxer() throws IOException { 679 return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 680 } 681 getAndSelectVideoTrackIndex(MediaExtractor extractor)682 private int getAndSelectVideoTrackIndex(MediaExtractor extractor) { 683 for (int index = 0; index < extractor.getTrackCount(); ++index) { 684 if (VERBOSE) { 685 Log.d(TAG, "format for track " + index + " is " 686 + getMimeTypeFor(extractor.getTrackFormat(index))); 687 } 688 if (isVideoFormat(extractor.getTrackFormat(index))) { 689 extractor.selectTrack(index); 690 return index; 691 } 692 } 693 return -1; 694 } 695 getAndSelectAudioTrackIndex(MediaExtractor extractor)696 private int getAndSelectAudioTrackIndex(MediaExtractor extractor) { 697 for (int index = 0; index < extractor.getTrackCount(); ++index) { 698 if (VERBOSE) { 699 Log.d(TAG, "format for track " + index + " is " 700 + getMimeTypeFor(extractor.getTrackFormat(index))); 701 } 702 if (isAudioFormat(extractor.getTrackFormat(index))) { 703 extractor.selectTrack(index); 704 return index; 705 } 706 } 707 return -1; 708 } 709 710 /** 711 * Does the actual work for extracting, decoding, encoding and muxing. 712 */ doExtractDecodeEditEncodeMux( MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface)713 private void doExtractDecodeEditEncodeMux( 714 MediaExtractor videoExtractor, 715 MediaExtractor audioExtractor, 716 MediaCodec videoDecoder, 717 MediaCodec videoEncoder, 718 MediaCodec audioDecoder, 719 MediaCodec audioEncoder, 720 MediaMuxer muxer, 721 InputSurface inputSurface, 722 OutputSurface outputSurface) { 723 ByteBuffer[] videoDecoderInputBuffers = null; 724 ByteBuffer[] videoDecoderOutputBuffers = null; 725 ByteBuffer[] videoEncoderOutputBuffers = null; 726 MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null; 727 MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null; 728 if (mCopyVideo) { 729 videoDecoderInputBuffers = videoDecoder.getInputBuffers(); 730 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 731 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 732 videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 733 videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 734 } 735 ByteBuffer[] audioDecoderInputBuffers = null; 736 ByteBuffer[] audioDecoderOutputBuffers = null; 737 ByteBuffer[] audioEncoderInputBuffers = null; 738 ByteBuffer[] audioEncoderOutputBuffers = null; 739 MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null; 740 MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null; 741 if (mCopyAudio) { 742 audioDecoderInputBuffers = audioDecoder.getInputBuffers(); 743 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 744 audioEncoderInputBuffers = audioEncoder.getInputBuffers(); 745 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 746 audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 747 audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 748 } 749 // We will get these from the decoders when notified of a format change. 750 MediaFormat decoderOutputVideoFormat = null; 751 MediaFormat decoderOutputAudioFormat = null; 752 // We will get these from the encoders when notified of a format change. 753 MediaFormat encoderOutputVideoFormat = null; 754 MediaFormat encoderOutputAudioFormat = null; 755 // We will determine these once we have the output format. 756 int outputVideoTrack = -1; 757 int outputAudioTrack = -1; 758 // Whether things are done on the video side. 759 boolean videoExtractorDone = false; 760 boolean videoDecoderDone = false; 761 boolean videoEncoderDone = false; 762 // Whether things are done on the audio side. 763 boolean audioExtractorDone = false; 764 boolean audioDecoderDone = false; 765 boolean audioEncoderDone = false; 766 // The audio decoder output buffer to process, -1 if none. 767 int pendingAudioDecoderOutputBufferIndex = -1; 768 769 boolean muxing = false; 770 771 int videoExtractedFrameCount = 0; 772 int videoDecodedFrameCount = 0; 773 int videoEncodedFrameCount = 0; 774 775 int audioExtractedFrameCount = 0; 776 int audioDecodedFrameCount = 0; 777 int audioEncodedFrameCount = 0; 778 779 while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) { 780 if (VERBOSE) { 781 Log.d(TAG, String.format( 782 "loop: " 783 784 + "V(%b){" 785 + "extracted:%d(done:%b) " 786 + "decoded:%d(done:%b) " 787 + "encoded:%d(done:%b)} " 788 789 + "A(%b){" 790 + "extracted:%d(done:%b) " 791 + "decoded:%d(done:%b) " 792 + "encoded:%d(done:%b) " 793 + "pending:%d} " 794 795 + "muxing:%b(V:%d,A:%d)", 796 797 mCopyVideo, 798 videoExtractedFrameCount, videoExtractorDone, 799 videoDecodedFrameCount, videoDecoderDone, 800 videoEncodedFrameCount, videoEncoderDone, 801 802 mCopyAudio, 803 audioExtractedFrameCount, audioExtractorDone, 804 audioDecodedFrameCount, audioDecoderDone, 805 audioEncodedFrameCount, audioEncoderDone, 806 pendingAudioDecoderOutputBufferIndex, 807 808 muxing, outputVideoTrack, outputAudioTrack)); 809 } 810 811 // Extract video from file and feed to decoder. 812 // Do not extract video if we have determined the output format but we are not yet 813 // ready to mux the frames. 814 while (mCopyVideo && !videoExtractorDone 815 && (encoderOutputVideoFormat == null || muxing)) { 816 int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC); 817 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 818 if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); 819 break; 820 } 821 if (VERBOSE) { 822 Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); 823 } 824 ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex]; 825 int size = videoExtractor.readSampleData(decoderInputBuffer, 0); 826 long presentationTime = videoExtractor.getSampleTime(); 827 int flags = videoExtractor.getSampleFlags(); 828 if (VERBOSE) { 829 Log.d(TAG, "video extractor: returned buffer of size " + size); 830 Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); 831 } 832 videoExtractorDone = !videoExtractor.advance(); 833 if (videoExtractorDone) { 834 if (VERBOSE) Log.d(TAG, "video extractor: EOS"); 835 flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM; 836 } 837 if (size >= 0) { 838 videoDecoder.queueInputBuffer( 839 decoderInputBufferIndex, 840 0, 841 size, 842 presentationTime, 843 flags); 844 videoExtractedFrameCount++; 845 } 846 // We extracted a frame, let's try something else next. 847 break; 848 } 849 850 // Extract audio from file and feed to decoder. 851 // Do not extract audio if we have determined the output format but we are not yet 852 // ready to mux the frames. 853 while (mCopyAudio && !audioExtractorDone 854 && (encoderOutputAudioFormat == null || muxing)) { 855 int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC); 856 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 857 if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); 858 break; 859 } 860 if (VERBOSE) { 861 Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); 862 } 863 ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex]; 864 int size = audioExtractor.readSampleData(decoderInputBuffer, 0); 865 long presentationTime = audioExtractor.getSampleTime(); 866 if (VERBOSE) { 867 Log.d(TAG, "audio extractor: returned buffer of size " + size); 868 Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); 869 } 870 if (size >= 0) { 871 audioDecoder.queueInputBuffer( 872 decoderInputBufferIndex, 873 0, 874 size, 875 presentationTime, 876 audioExtractor.getSampleFlags()); 877 } 878 audioExtractorDone = !audioExtractor.advance(); 879 if (audioExtractorDone) { 880 if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); 881 audioDecoder.queueInputBuffer( 882 decoderInputBufferIndex, 883 0, 884 0, 885 0, 886 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 887 } 888 audioExtractedFrameCount++; 889 // We extracted a frame, let's try something else next. 890 break; 891 } 892 893 // Poll output frames from the video decoder and feed the encoder. 894 while (mCopyVideo && !videoDecoderDone 895 && (encoderOutputVideoFormat == null || muxing)) { 896 int decoderOutputBufferIndex = 897 videoDecoder.dequeueOutputBuffer( 898 videoDecoderOutputBufferInfo, TIMEOUT_USEC); 899 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 900 if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); 901 break; 902 } 903 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 904 if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); 905 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 906 break; 907 } 908 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 909 decoderOutputVideoFormat = videoDecoder.getOutputFormat(); 910 if (VERBOSE) { 911 Log.d(TAG, "video decoder: output format changed: " 912 + decoderOutputVideoFormat); 913 } 914 break; 915 } 916 if (VERBOSE) { 917 Log.d(TAG, "video decoder: returned output buffer: " 918 + decoderOutputBufferIndex); 919 Log.d(TAG, "video decoder: returned buffer of size " 920 + videoDecoderOutputBufferInfo.size); 921 } 922 ByteBuffer decoderOutputBuffer = 923 videoDecoderOutputBuffers[decoderOutputBufferIndex]; 924 if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 925 != 0) { 926 if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); 927 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 928 break; 929 } 930 if (VERBOSE) { 931 Log.d(TAG, "video decoder: returned buffer for time " 932 + videoDecoderOutputBufferInfo.presentationTimeUs); 933 } 934 boolean render = videoDecoderOutputBufferInfo.size != 0; 935 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); 936 if (render) { 937 if (VERBOSE) Log.d(TAG, "output surface: await new image"); 938 outputSurface.awaitNewImage(); 939 // Edit the frame and send it to the encoder. 940 if (VERBOSE) Log.d(TAG, "output surface: draw image"); 941 outputSurface.drawImage(); 942 inputSurface.setPresentationTime( 943 videoDecoderOutputBufferInfo.presentationTimeUs * 1000); 944 if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); 945 inputSurface.swapBuffers(); 946 if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); 947 videoDecodedFrameCount++; 948 } 949 if ((videoDecoderOutputBufferInfo.flags 950 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 951 if (VERBOSE) Log.d(TAG, "video decoder: EOS"); 952 videoDecoderDone = true; 953 videoEncoder.signalEndOfInputStream(); 954 } 955 // We extracted a pending frame, let's try something else next. 956 break; 957 } 958 959 // Poll output frames from the audio decoder. 960 // Do not poll if we already have a pending buffer to feed to the encoder. 961 while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1 962 && (encoderOutputAudioFormat == null || muxing)) { 963 int decoderOutputBufferIndex = 964 audioDecoder.dequeueOutputBuffer( 965 audioDecoderOutputBufferInfo, TIMEOUT_USEC); 966 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 967 if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); 968 break; 969 } 970 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 971 if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); 972 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 973 break; 974 } 975 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 976 decoderOutputAudioFormat = audioDecoder.getOutputFormat(); 977 if (VERBOSE) { 978 Log.d(TAG, "audio decoder: output format changed: " 979 + decoderOutputAudioFormat); 980 } 981 break; 982 } 983 if (VERBOSE) { 984 Log.d(TAG, "audio decoder: returned output buffer: " 985 + decoderOutputBufferIndex); 986 } 987 if (VERBOSE) { 988 Log.d(TAG, "audio decoder: returned buffer of size " 989 + audioDecoderOutputBufferInfo.size); 990 } 991 ByteBuffer decoderOutputBuffer = 992 audioDecoderOutputBuffers[decoderOutputBufferIndex]; 993 if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 994 != 0) { 995 if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); 996 audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 997 break; 998 } 999 if (VERBOSE) { 1000 Log.d(TAG, "audio decoder: returned buffer for time " 1001 + audioDecoderOutputBufferInfo.presentationTimeUs); 1002 } 1003 if (VERBOSE) { 1004 Log.d(TAG, "audio decoder: output buffer is now pending: " 1005 + pendingAudioDecoderOutputBufferIndex); 1006 } 1007 pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; 1008 audioDecodedFrameCount++; 1009 // We extracted a pending frame, let's try something else next. 1010 break; 1011 } 1012 1013 // Feed the pending decoded audio buffer to the audio encoder. 1014 while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) { 1015 if (VERBOSE) { 1016 Log.d(TAG, "audio decoder: attempting to process pending buffer: " 1017 + pendingAudioDecoderOutputBufferIndex); 1018 } 1019 int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC); 1020 if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1021 if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); 1022 break; 1023 } 1024 if (VERBOSE) { 1025 Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); 1026 } 1027 ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex]; 1028 int size = audioDecoderOutputBufferInfo.size; 1029 long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs; 1030 if (VERBOSE) { 1031 Log.d(TAG, "audio decoder: processing pending buffer: " 1032 + pendingAudioDecoderOutputBufferIndex); 1033 } 1034 if (VERBOSE) { 1035 Log.d(TAG, "audio decoder: pending buffer of size " + size); 1036 Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); 1037 } 1038 if (size >= 0) { 1039 ByteBuffer decoderOutputBuffer = 1040 audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex] 1041 .duplicate(); 1042 decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset); 1043 decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size); 1044 encoderInputBuffer.position(0); 1045 encoderInputBuffer.put(decoderOutputBuffer); 1046 1047 audioEncoder.queueInputBuffer( 1048 encoderInputBufferIndex, 1049 0, 1050 size, 1051 presentationTime, 1052 audioDecoderOutputBufferInfo.flags); 1053 } 1054 audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false); 1055 pendingAudioDecoderOutputBufferIndex = -1; 1056 if ((audioDecoderOutputBufferInfo.flags 1057 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1058 if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); 1059 audioDecoderDone = true; 1060 } 1061 // We enqueued a pending frame, let's try something else next. 1062 break; 1063 } 1064 1065 // Poll frames from the video encoder and send them to the muxer. 1066 while (mCopyVideo && !videoEncoderDone 1067 && (encoderOutputVideoFormat == null || muxing)) { 1068 int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer( 1069 videoEncoderOutputBufferInfo, TIMEOUT_USEC); 1070 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1071 if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); 1072 break; 1073 } 1074 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1075 if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); 1076 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 1077 break; 1078 } 1079 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1080 if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); 1081 if (outputVideoTrack >= 0) { 1082 fail("video encoder changed its output format again?"); 1083 } 1084 encoderOutputVideoFormat = videoEncoder.getOutputFormat(); 1085 break; 1086 } 1087 assertTrue("should have added track before processing output", muxing); 1088 if (VERBOSE) { 1089 Log.d(TAG, "video encoder: returned output buffer: " 1090 + encoderOutputBufferIndex); 1091 Log.d(TAG, "video encoder: returned buffer of size " 1092 + videoEncoderOutputBufferInfo.size); 1093 } 1094 ByteBuffer encoderOutputBuffer = 1095 videoEncoderOutputBuffers[encoderOutputBufferIndex]; 1096 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1097 != 0) { 1098 if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); 1099 // Simply ignore codec config buffers. 1100 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1101 break; 1102 } 1103 if (VERBOSE) { 1104 Log.d(TAG, "video encoder: returned buffer for time " 1105 + videoEncoderOutputBufferInfo.presentationTimeUs); 1106 } 1107 if (videoEncoderOutputBufferInfo.size != 0) { 1108 muxer.writeSampleData( 1109 outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo); 1110 videoEncodedFrameCount++; 1111 } 1112 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1113 != 0) { 1114 if (VERBOSE) Log.d(TAG, "video encoder: EOS"); 1115 videoEncoderDone = true; 1116 } 1117 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1118 // We enqueued an encoded frame, let's try something else next. 1119 break; 1120 } 1121 1122 // Poll frames from the audio encoder and send them to the muxer. 1123 while (mCopyAudio && !audioEncoderDone 1124 && (encoderOutputAudioFormat == null || muxing)) { 1125 int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer( 1126 audioEncoderOutputBufferInfo, TIMEOUT_USEC); 1127 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1128 if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); 1129 break; 1130 } 1131 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1132 if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); 1133 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 1134 break; 1135 } 1136 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1137 if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); 1138 if (outputAudioTrack >= 0) { 1139 fail("audio encoder changed its output format again?"); 1140 } 1141 1142 encoderOutputAudioFormat = audioEncoder.getOutputFormat(); 1143 break; 1144 } 1145 assertTrue("should have added track before processing output", muxing); 1146 if (VERBOSE) { 1147 Log.d(TAG, "audio encoder: returned output buffer: " 1148 + encoderOutputBufferIndex); 1149 Log.d(TAG, "audio encoder: returned buffer of size " 1150 + audioEncoderOutputBufferInfo.size); 1151 } 1152 ByteBuffer encoderOutputBuffer = 1153 audioEncoderOutputBuffers[encoderOutputBufferIndex]; 1154 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1155 != 0) { 1156 if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); 1157 // Simply ignore codec config buffers. 1158 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1159 break; 1160 } 1161 if (VERBOSE) { 1162 Log.d(TAG, "audio encoder: returned buffer for time " 1163 + audioEncoderOutputBufferInfo.presentationTimeUs); 1164 } 1165 if (audioEncoderOutputBufferInfo.size != 0) { 1166 muxer.writeSampleData( 1167 outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo); 1168 } 1169 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1170 != 0) { 1171 if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); 1172 audioEncoderDone = true; 1173 } 1174 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1175 audioEncodedFrameCount++; 1176 // We enqueued an encoded frame, let's try something else next. 1177 break; 1178 } 1179 1180 if (!muxing 1181 && (!mCopyAudio || encoderOutputAudioFormat != null) 1182 && (!mCopyVideo || encoderOutputVideoFormat != null)) { 1183 if (mCopyVideo) { 1184 Log.d(TAG, "muxer: adding video track."); 1185 outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat); 1186 } 1187 if (mCopyAudio) { 1188 Log.d(TAG, "muxer: adding audio track."); 1189 outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat); 1190 } 1191 Log.d(TAG, "muxer: starting"); 1192 muxer.start(); 1193 muxing = true; 1194 } 1195 } 1196 1197 // Basic validation checks. 1198 if (mCopyVideo) { 1199 assertEquals("encoded and decoded video frame counts should match", 1200 videoDecodedFrameCount, videoEncodedFrameCount); 1201 assertTrue("decoded frame count should be less than extracted frame count", 1202 videoDecodedFrameCount <= videoExtractedFrameCount); 1203 } 1204 if (mCopyAudio) { 1205 assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex); 1206 } 1207 } 1208 isVideoFormat(MediaFormat format)1209 private static boolean isVideoFormat(MediaFormat format) { 1210 return getMimeTypeFor(format).startsWith("video/"); 1211 } 1212 isAudioFormat(MediaFormat format)1213 private static boolean isAudioFormat(MediaFormat format) { 1214 return getMimeTypeFor(format).startsWith("audio/"); 1215 } 1216 getMimeTypeFor(MediaFormat format)1217 private static String getMimeTypeFor(MediaFormat format) { 1218 return format.getString(MediaFormat.KEY_MIME); 1219 } 1220 1221 /** 1222 * Returns the first codec capable of encoding the specified MIME type, or null if no match was 1223 * found. 1224 */ selectCodec(String mimeType)1225 private static MediaCodecInfo selectCodec(String mimeType) { 1226 int numCodecs = MediaCodecList.getCodecCount(); 1227 for (int i = 0; i < numCodecs; i++) { 1228 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1229 1230 if (codecInfo.isAlias()) { 1231 continue; 1232 } 1233 if (!codecInfo.isEncoder()) { 1234 continue; 1235 } 1236 1237 String[] types = codecInfo.getSupportedTypes(); 1238 for (int j = 0; j < types.length; j++) { 1239 if (types[j].equalsIgnoreCase(mimeType)) { 1240 return codecInfo; 1241 } 1242 } 1243 } 1244 return null; 1245 } 1246 } 1247