1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.annotation.UnsupportedAppUsage; 23 import android.graphics.ImageFormat; 24 import android.graphics.Rect; 25 import android.graphics.SurfaceTexture; 26 import android.media.MediaCodecInfo.CodecCapabilities; 27 import android.os.Build; 28 import android.os.Bundle; 29 import android.os.Handler; 30 import android.os.IHwBinder; 31 import android.os.Looper; 32 import android.os.Message; 33 import android.os.PersistableBundle; 34 import android.view.Surface; 35 36 import java.io.IOException; 37 import java.lang.annotation.Retention; 38 import java.lang.annotation.RetentionPolicy; 39 import java.nio.ByteBuffer; 40 import java.nio.ByteOrder; 41 import java.nio.ReadOnlyBufferException; 42 import java.util.Arrays; 43 import java.util.HashMap; 44 import java.util.Map; 45 import java.util.concurrent.locks.Lock; 46 import java.util.concurrent.locks.ReentrantLock; 47 48 /** 49 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 50 It is part of the Android low-level multimedia support infrastructure (normally used together 51 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 52 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 53 <p> 54 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 55 data="../../../images/media/mediacodec_buffers.svg"><img 56 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 57 alt="MediaCodec buffer flow diagram"></object></center> 58 <p> 59 In broad terms, a codec processes input data to generate output data. It processes data 60 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 61 (or receive) an empty input buffer, fill it up with data and send it to the codec for 62 processing. The codec uses up the data and transforms it into one of its empty output buffers. 63 Finally, you request (or receive) a filled output buffer, consume its contents and release it 64 back to the codec. 65 66 <h3>Data Types</h3> 67 <p> 68 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 69 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 70 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 71 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 72 You normally cannot access the raw video data when using a Surface, but you can use the 73 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 74 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 75 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 76 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 77 OutputImage(int)}. 78 79 <h4>Compressed Buffers</h4> 80 <p> 81 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 82 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 83 compressed video frame. For audio data this is normally a single access unit (an encoded audio 84 segment typically containing a few milliseconds of audio as dictated by the format type), but 85 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 86 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 87 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 88 89 <h4>Raw Audio Buffers</h4> 90 <p> 91 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 92 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 93 in native byte order. 94 Raw audio buffers in the float PCM encoding are only possible 95 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 96 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 97 {@link #configure configure(…)} 98 and confirmed by {@link #getOutputFormat} for decoders 99 or {@link #getInputFormat} for encoders. 100 A sample method to check for float PCM in the MediaFormat is as follows: 101 102 <pre class=prettyprint> 103 static boolean isPcmFloat(MediaFormat format) { 104 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 105 == AudioFormat.ENCODING_PCM_FLOAT; 106 }</pre> 107 108 In order to extract, in a short array, 109 one channel of a buffer containing 16 bit signed integer audio data, 110 the following code may be used: 111 112 <pre class=prettyprint> 113 // Assumes the buffer PCM encoding is 16 bit. 114 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 115 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 116 MediaFormat format = codec.getOutputFormat(bufferId); 117 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 118 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 119 if (channelIx < 0 || channelIx >= numChannels) { 120 return null; 121 } 122 short[] res = new short[samples.remaining() / numChannels]; 123 for (int i = 0; i < res.length; ++i) { 124 res[i] = samples.get(i * numChannels + channelIx); 125 } 126 return res; 127 }</pre> 128 129 <h4>Raw Video Buffers</h4> 130 <p> 131 In ByteBuffer mode video buffers are laid out according to their {@linkplain 132 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 133 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 134 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 135 Video codecs may support three kinds of color formats: 136 <ul> 137 <li><strong>native raw video format:</strong> This is marked by {@link 138 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 139 <li><strong>flexible YUV buffers</strong> (such as {@link 140 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 141 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 142 OutputImage(int)}.</li> 143 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 144 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 145 For color formats that are equivalent to a flexible format, you can still use {@link 146 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 147 </ul> 148 <p> 149 All video codecs support flexible YUV 4:2:0 buffers since {@link 150 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 151 152 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 153 <p> 154 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 155 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 156 values to understand the layout of the raw output buffers. 157 <p class=note> 158 Note that on some devices the slice-height is advertised as 0. This could mean either that the 159 slice-height is the same as the frame height, or that the slice-height is the frame height 160 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 161 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 162 plane in planar formats is also not specified or defined, though usually it is half of the slice 163 height. 164 <p> 165 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 166 video frames; however, for most encondings the video (picture) only occupies a portion of the 167 video frame. This is represented by the 'crop rectangle'. 168 <p> 169 You need to use the following keys to get the crop rectangle of raw output images from the 170 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 171 entire video frame.The crop rectangle is understood in the context of the output frame 172 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 173 <table style="width: 0%"> 174 <thead> 175 <tr> 176 <th>Format Key</th> 177 <th>Type</th> 178 <th>Description</th> 179 </tr> 180 </thead> 181 <tbody> 182 <tr> 183 <td>{@code "crop-left"}</td> 184 <td>Integer</td> 185 <td>The left-coordinate (x) of the crop rectangle</td> 186 </tr><tr> 187 <td>{@code "crop-top"}</td> 188 <td>Integer</td> 189 <td>The top-coordinate (y) of the crop rectangle</td> 190 </tr><tr> 191 <td>{@code "crop-right"}</td> 192 <td>Integer</td> 193 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 194 </tr><tr> 195 <td>{@code "crop-bottom"}</td> 196 <td>Integer</td> 197 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 198 </tr><tr> 199 <td colspan=3> 200 The right and bottom coordinates can be understood as the coordinates of the right-most 201 valid column/bottom-most valid row of the cropped output image. 202 </td> 203 </tr> 204 </tbody> 205 </table> 206 <p> 207 The size of the video frame (before rotation) can be calculated as such: 208 <pre class=prettyprint> 209 MediaFormat format = decoder.getOutputFormat(…); 210 int width = format.getInteger(MediaFormat.KEY_WIDTH); 211 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 212 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 213 } 214 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 215 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 216 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 217 } 218 </pre> 219 <p class=note> 220 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 221 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 222 most devices it pointed to the top-left pixel of the entire frame. 223 224 <h3>States</h3> 225 <p> 226 During its life a codec conceptually exists in one of three states: Stopped, Executing or 227 Released. The Stopped collective state is actually the conglomeration of three states: 228 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 229 three sub-states: Flushed, Running and End-of-Stream. 230 <p> 231 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 232 data="../../../images/media/mediacodec_states.svg"><img 233 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 234 alt="MediaCodec state diagram"></object></center> 235 <p> 236 When you create a codec using one of the factory methods, the codec is in the Uninitialized 237 state. First, you need to configure it via {@link #configure configure(…)}, which brings 238 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 239 state you can process data through the buffer queue manipulation described above. 240 <p> 241 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 242 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 243 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 244 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 245 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 246 codec no longer accepts further input buffers, but still generates output buffers until the 247 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 248 while in the Executing state using {@link #flush}. 249 <p> 250 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 251 again. When you are done using a codec, you must release it by calling {@link #release}. 252 <p> 253 On rare occasions the codec may encounter an error and move to the Error state. This is 254 communicated using an invalid return value from a queuing operation, or sometimes via an 255 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 256 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 257 terminal Released state. 258 259 <h3>Creation</h3> 260 <p> 261 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 262 decoding a file or a stream, you can get the desired format from {@link 263 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 264 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 265 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 266 name of a codec that can handle that specific media format. Finally, create the codec using 267 {@link #createByCodecName}. 268 <p class=note> 269 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 270 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 271 MediaFormat#KEY_FRAME_RATE frame rate}. Use 272 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 273 to clear any existing frame rate setting in the format. 274 <p> 275 You can also create the preferred codec for a specific MIME type using {@link 276 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 277 This, however, cannot be used to inject features, and may create a codec that cannot handle the 278 specific desired media format. 279 280 <h4>Creating secure decoders</h4> 281 <p> 282 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 283 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 284 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 285 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 286 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 287 <p> 288 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 289 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 290 291 <h3>Initialization</h3> 292 <p> 293 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 294 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 295 specific media format. This is when you can specify the output {@link Surface} for video 296 producers – codecs that generate raw video data (e.g. video decoders). This is also when 297 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 298 some codecs can operate in multiple modes, you must specify whether you want it to work as a 299 decoder or an encoder. 300 <p> 301 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 302 output format in the Configured state. You can use this to verify the resulting configuration, 303 e.g. color formats, before starting the codec. 304 <p> 305 If you want to process raw input video buffers natively with a video consumer – a codec 306 that processes raw video input, such as a video encoder – create a destination Surface for 307 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 308 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 309 surface} by calling {@link #setInputSurface}. 310 311 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 312 <p> 313 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 314 to be prefixed by a number of buffers containing setup data, or codec specific data. When 315 processing such compressed formats, this data must be submitted to the codec after {@link 316 #start} and before any frame data. Such data must be marked using the flag {@link 317 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 318 <p> 319 Codec-specific data can also be included in the format passed to {@link #configure configure} in 320 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 321 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 322 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 323 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 324 specific data, you can choose to submit it using the specified number of buffers in the correct 325 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 326 codec-specific data and submit it as a single codec-config buffer. 327 <p> 328 Android uses the following codec-specific data buffers. These are also required to be set in 329 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 330 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 331 {@code "\x00\x00\x00\x01"}. 332 <p> 333 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 334 <table> 335 <thead> 336 <th>Format</th> 337 <th>CSD buffer #0</th> 338 <th>CSD buffer #1</th> 339 <th>CSD buffer #2</th> 340 </thead> 341 <tbody class=mid> 342 <tr> 343 <td>AAC</td> 344 <td>Decoder-specific information from ESDS<sup>*</sup></td> 345 <td class=NA>Not Used</td> 346 <td class=NA>Not Used</td> 347 </tr> 348 <tr> 349 <td>VORBIS</td> 350 <td>Identification header</td> 351 <td>Setup header</td> 352 <td class=NA>Not Used</td> 353 </tr> 354 <tr> 355 <td>OPUS</td> 356 <td>Identification header</td> 357 <td>Pre-skip in nanosecs<br> 358 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 359 This overrides the pre-skip value in the identification header.</td> 360 <td>Seek Pre-roll in nanosecs<br> 361 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 362 </tr> 363 <tr> 364 <td>FLAC</td> 365 <td>mandatory metadata block (called the STREAMINFO block),<br> 366 optionally followed by any number of other metadata blocks</td> 367 <td class=NA>Not Used</td> 368 <td class=NA>Not Used</td> 369 </tr> 370 <tr> 371 <td>MPEG-4</td> 372 <td>Decoder-specific information from ESDS<sup>*</sup></td> 373 <td class=NA>Not Used</td> 374 <td class=NA>Not Used</td> 375 </tr> 376 <tr> 377 <td>H.264 AVC</td> 378 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 379 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 380 <td class=NA>Not Used</td> 381 </tr> 382 <tr> 383 <td>H.265 HEVC</td> 384 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 385 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 386 PPS (Picture Parameter Sets<sup>*</sup>)</td> 387 <td class=NA>Not Used</td> 388 <td class=NA>Not Used</td> 389 </tr> 390 <tr> 391 <td>VP9</td> 392 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 393 (optional)</td> 394 <td class=NA>Not Used</td> 395 <td class=NA>Not Used</td> 396 </tr> 397 </tbody> 398 </table> 399 400 <p class=note> 401 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 402 after start, before any output buffer or output format change has been returned, as the codec 403 specific data may be lost during the flush. You must resubmit the data using buffers marked with 404 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 405 <p> 406 Encoders (or codecs that generate compressed data) will create and return the codec specific data 407 before any valid output buffer in output buffers marked with the {@linkplain 408 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 409 meaningful timestamps. 410 411 <h3>Data Processing</h3> 412 <p> 413 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 414 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 415 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 416 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 417 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 418 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 419 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 420 <p> 421 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 422 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 423 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 424 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 425 <p> 426 The codec in turn will return a read-only output buffer via the {@link 427 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 428 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 429 output buffer has been processed, call one of the {@link #releaseOutputBuffer 430 releaseOutputBuffer} methods to return the buffer to the codec. 431 <p> 432 While you are not required to resubmit/release buffers immediately to the codec, holding onto 433 input and/or output buffers may stall the codec, and this behavior is device dependent. 434 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 435 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 436 hold onto to available buffers as little as possible. 437 <p> 438 Depending on the API version, you can process data in three ways: 439 <table> 440 <thead> 441 <tr> 442 <th>Processing Mode</th> 443 <th>API version <= 20<br>Jelly Bean/KitKat</th> 444 <th>API version >= 21<br>Lollipop and later</th> 445 </tr> 446 </thead> 447 <tbody> 448 <tr> 449 <td>Synchronous API using buffer arrays</td> 450 <td>Supported</td> 451 <td>Deprecated</td> 452 </tr> 453 <tr> 454 <td>Synchronous API using buffers</td> 455 <td class=NA>Not Available</td> 456 <td>Supported</td> 457 </tr> 458 <tr> 459 <td>Asynchronous API using buffers</td> 460 <td class=NA>Not Available</td> 461 <td>Supported</td> 462 </tr> 463 </tbody> 464 </table> 465 466 <h4>Asynchronous Processing using Buffers</h4> 467 <p> 468 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 469 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 470 mode changes the state transitions slightly, because you must call {@link #start} after {@link 471 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 472 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 473 sub-state and start passing available input buffers via the callback. 474 <p> 475 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 476 data="../../../images/media/mediacodec_async_states.svg"><img 477 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 478 alt="MediaCodec state diagram for asynchronous operation"></object></center> 479 <p> 480 MediaCodec is typically used like this in asynchronous mode: 481 <pre class=prettyprint> 482 MediaCodec codec = MediaCodec.createByCodecName(name); 483 MediaFormat mOutputFormat; // member variable 484 codec.setCallback(new MediaCodec.Callback() { 485 {@literal @Override} 486 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 487 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 488 // fill inputBuffer with valid data 489 … 490 codec.queueInputBuffer(inputBufferId, …); 491 } 492 493 {@literal @Override} 494 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 495 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 496 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 497 // bufferFormat is equivalent to mOutputFormat 498 // outputBuffer is ready to be processed or rendered. 499 … 500 codec.releaseOutputBuffer(outputBufferId, …); 501 } 502 503 {@literal @Override} 504 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 505 // Subsequent data will conform to new format. 506 // Can ignore if using getOutputFormat(outputBufferId) 507 mOutputFormat = format; // option B 508 } 509 510 {@literal @Override} 511 void onError(…) { 512 … 513 } 514 }); 515 codec.configure(format, …); 516 mOutputFormat = codec.getOutputFormat(); // option B 517 codec.start(); 518 // wait for processing to complete 519 codec.stop(); 520 codec.release();</pre> 521 522 <h4>Synchronous Processing using Buffers</h4> 523 <p> 524 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 525 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 526 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 527 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 528 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 529 getInput}/{@link #getOutputBuffers OutputBuffers()}. 530 531 <p class=note> 532 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 533 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 534 #start} or after having dequeued an output buffer ID with the value of {@link 535 #INFO_OUTPUT_FORMAT_CHANGED}. 536 <p> 537 MediaCodec is typically used like this in synchronous mode: 538 <pre> 539 MediaCodec codec = MediaCodec.createByCodecName(name); 540 codec.configure(format, …); 541 MediaFormat outputFormat = codec.getOutputFormat(); // option B 542 codec.start(); 543 for (;;) { 544 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 545 if (inputBufferId >= 0) { 546 ByteBuffer inputBuffer = codec.getInputBuffer(…); 547 // fill inputBuffer with valid data 548 … 549 codec.queueInputBuffer(inputBufferId, …); 550 } 551 int outputBufferId = codec.dequeueOutputBuffer(…); 552 if (outputBufferId >= 0) { 553 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 554 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 555 // bufferFormat is identical to outputFormat 556 // outputBuffer is ready to be processed or rendered. 557 … 558 codec.releaseOutputBuffer(outputBufferId, …); 559 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 560 // Subsequent data will conform to new format. 561 // Can ignore if using getOutputFormat(outputBufferId) 562 outputFormat = codec.getOutputFormat(); // option B 563 } 564 } 565 codec.stop(); 566 codec.release();</pre> 567 568 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 569 <p> 570 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 571 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 572 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 573 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 574 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 575 between the size of the arrays and the number of input and output buffers used by the system, 576 although the array size provides an upper bound. 577 <pre> 578 MediaCodec codec = MediaCodec.createByCodecName(name); 579 codec.configure(format, …); 580 codec.start(); 581 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 582 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 583 for (;;) { 584 int inputBufferId = codec.dequeueInputBuffer(…); 585 if (inputBufferId >= 0) { 586 // fill inputBuffers[inputBufferId] with valid data 587 … 588 codec.queueInputBuffer(inputBufferId, …); 589 } 590 int outputBufferId = codec.dequeueOutputBuffer(…); 591 if (outputBufferId >= 0) { 592 // outputBuffers[outputBufferId] is ready to be processed or rendered. 593 … 594 codec.releaseOutputBuffer(outputBufferId, …); 595 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 596 outputBuffers = codec.getOutputBuffers(); 597 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 598 // Subsequent data will conform to new format. 599 MediaFormat format = codec.getOutputFormat(); 600 } 601 } 602 codec.stop(); 603 codec.release();</pre> 604 605 <h4>End-of-stream Handling</h4> 606 <p> 607 When you reach the end of the input data, you must signal it to the codec by specifying the 608 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 609 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 610 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 611 be ignored. 612 <p> 613 The codec will continue to return output buffers until it eventually signals the end of the 614 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 615 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 616 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 617 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 618 <p> 619 Do not submit additional input buffers after signaling the end of the input stream, unless the 620 codec has been flushed, or stopped and restarted. 621 622 <h4>Using an Output Surface</h4> 623 <p> 624 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 625 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 626 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 627 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 628 null}-s. 629 <p> 630 When using an output Surface, you can select whether or not to render each output buffer on the 631 surface. You have three choices: 632 <ul> 633 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 634 releaseOutputBuffer(bufferId, false)}.</li> 635 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 636 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 637 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 638 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 639 </ul> 640 <p> 641 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 642 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 643 It was not defined prior to that. 644 <p> 645 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 646 dynamically using {@link #setOutputSurface setOutputSurface}. 647 <p> 648 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 649 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 650 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 651 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 652 was undefined, with the exception that View surfaces (SuerfaceView or TextureView) always dropped 653 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 654 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 655 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 656 setting the key {@code "allow-frame-drop"} to {@code 0} in their configure format. 657 658 <h4>Transformations When Rendering onto Surface</h4> 659 660 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 661 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 662 mode} will be automatically applied with one exception: 663 <p class=note> 664 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 665 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 666 and simple way to identify software decoders, or if they apply the rotation other than by trying 667 it out. 668 <p> 669 There are also some caveats. 670 <p class=note> 671 Note that the pixel aspect ratio is not considered when displaying the output onto the 672 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 673 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 674 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 675 square pixels (pixel aspect ratio or 1:1). 676 <p class=note> 677 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 678 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 679 by 90 or 270 degrees. 680 <p class=note> 681 When setting the video scaling mode, note that it must be reset after each time the output 682 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 683 do this after each time the output format changes. 684 685 <h4>Using an Input Surface</h4> 686 <p> 687 When using an input Surface, there are no accessible input buffers, as buffers are automatically 688 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 689 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 690 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 691 <p> 692 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 693 submitting data to the codec immediately after this call. 694 <p> 695 696 <h3>Seeking & Adaptive Playback Support</h3> 697 <p> 698 Video decoders (and in general codecs that consume compressed video data) behave differently 699 regarding seek and format change whether or not they support and are configured for adaptive 700 playback. You can check if a decoder supports {@linkplain 701 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 702 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 703 playback support for video decoders is only activated if you configure the codec to decode onto a 704 {@link Surface}. 705 706 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 707 <p> 708 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 709 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 710 completely on its own (for most codecs this means an I-frame), and no frames that are to be 711 displayed after a key frame refer to frames before the key frame. 712 <p> 713 The following table summarizes suitable key frames for various video formats. 714 <table> 715 <thead> 716 <tr> 717 <th>Format</th> 718 <th>Suitable key frame</th> 719 </tr> 720 </thead> 721 <tbody class=mid> 722 <tr> 723 <td>VP9/VP8</td> 724 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 725 <i>(There is no specific name for such key frame.)</i></td> 726 </tr> 727 <tr> 728 <td>H.265 HEVC</td> 729 <td>IDR or CRA</td> 730 </tr> 731 <tr> 732 <td>H.264 AVC</td> 733 <td>IDR</td> 734 </tr> 735 <tr> 736 <td>MPEG-4<br>H.263<br>MPEG-2</td> 737 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 738 <i>(There is no specific name for such key frame.)</td> 739 </tr> 740 </tbody> 741 </table> 742 743 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 744 Surface)</h4> 745 <p> 746 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 747 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 748 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 749 before you call {@code flush}. It is important that the input data after a flush starts at a 750 suitable stream boundary/key frame. 751 <p class=note> 752 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 753 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 754 #configure configure(…)} - {@link #start} cycle is necessary. 755 756 <p class=note> 757 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 758 generally, before the first output buffer or output format change is received – you 759 will need to resubmit the codec-specific-data to the codec. See the <a 760 href="#CSD">codec-specific-data section</a> for more info. 761 762 <h4>For decoders that support and are configured for adaptive playback</h4> 763 <p> 764 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 765 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 766 discontinuity must start at a suitable stream boundary/key frame. 767 <p> 768 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 769 picture size or configuration mid-stream. To do this you must package the entire new 770 codec-specific configuration data together with the key frame into a single buffer (including 771 any start codes), and submit it as a <strong>regular</strong> input buffer. 772 <p> 773 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 774 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 775 onOutputFormatChanged} callback just after the picture-size change takes place and before any 776 frames with the new size have been returned. 777 <p class=note> 778 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 779 {@link #flush} shortly after you have changed the picture size. If you have not received 780 confirmation of the picture size change, you will need to repeat the request for the new picture 781 size. 782 783 <h3>Error handling</h3> 784 <p> 785 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 786 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 787 which you must catch or declare to pass up. MediaCodec methods throw {@code 788 IllegalStateException} when the method is called from a codec state that does not allow it; this 789 is typically due to incorrect application API usage. Methods involving secure buffers may throw 790 {@link CryptoException}, which has further error information obtainable from {@link 791 CryptoException#getErrorCode}. 792 <p> 793 Internal codec errors result in a {@link CodecException}, which may be due to media content 794 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 795 correctly using the API. The recommended action when receiving a {@code CodecException} 796 can be determined by calling {@link CodecException#isRecoverable} and {@link 797 CodecException#isTransient}: 798 <ul> 799 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 800 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 801 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 802 temporarily unavailable and the method may be retried at a later time.</li> 803 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 804 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 805 reset} or {@linkplain #release released}.</li> 806 </ul> 807 <p> 808 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 809 810 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 811 <p> 812 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 813 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 814 815 <style> 816 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 817 .api > tr > th { vertical-align: bottom; } 818 .api > tr > td { vertical-align: middle; } 819 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 820 .fn { text-align: left; } 821 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 822 .deg45 { 823 white-space: nowrap; background: none; border: none; vertical-align: bottom; 824 width: 30px; height: 83px; 825 } 826 .deg45 > div { 827 transform: skew(-45deg, 0deg) translate(1px, -67px); 828 transform-origin: bottom left 0; 829 width: 30px; height: 20px; 830 } 831 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 832 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 833 </style> 834 835 <table align="right" style="width: 0%"> 836 <thead> 837 <tr><th>Symbol</th><th>Meaning</th></tr> 838 </thead> 839 <tbody class=sml> 840 <tr><td>●</td><td>Supported</td></tr> 841 <tr><td>⁕</td><td>Semantics changed</td></tr> 842 <tr><td>○</td><td>Experimental support</td></tr> 843 <tr><td>[ ]</td><td>Deprecated</td></tr> 844 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 845 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 846 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 847 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 848 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 849 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 850 </tbody> 851 </table> 852 853 <table style="width: 100%;"> 854 <thead class=api> 855 <tr> 856 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 857 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 858 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 859 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 860 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 861 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 862 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 863 <th></th> 864 <th colspan="8">SDK Version</th> 865 </tr> 866 <tr> 867 <th colspan="7">State</th> 868 <th>Method</th> 869 <th>16</th> 870 <th>17</th> 871 <th>18</th> 872 <th>19</th> 873 <th>20</th> 874 <th>21</th> 875 <th>22</th> 876 <th>23</th> 877 </tr> 878 </thead> 879 <tbody class=api> 880 <tr> 881 <td></td> 882 <td></td> 883 <td></td> 884 <td></td> 885 <td></td> 886 <td></td> 887 <td></td> 888 <td class=fn>{@link #createByCodecName createByCodecName}</td> 889 <td>●</td> 890 <td>●</td> 891 <td>●</td> 892 <td>●</td> 893 <td>●</td> 894 <td>●</td> 895 <td>●</td> 896 <td>●</td> 897 </tr> 898 <tr> 899 <td></td> 900 <td></td> 901 <td></td> 902 <td></td> 903 <td></td> 904 <td></td> 905 <td></td> 906 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 907 <td>●</td> 908 <td>●</td> 909 <td>●</td> 910 <td>●</td> 911 <td>●</td> 912 <td>●</td> 913 <td>●</td> 914 <td>●</td> 915 </tr> 916 <tr> 917 <td></td> 918 <td></td> 919 <td></td> 920 <td></td> 921 <td></td> 922 <td></td> 923 <td></td> 924 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 925 <td>●</td> 926 <td>●</td> 927 <td>●</td> 928 <td>●</td> 929 <td>●</td> 930 <td>●</td> 931 <td>●</td> 932 <td>●</td> 933 </tr> 934 <tr> 935 <td></td> 936 <td></td> 937 <td></td> 938 <td></td> 939 <td></td> 940 <td></td> 941 <td></td> 942 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 943 <td></td> 944 <td></td> 945 <td></td> 946 <td></td> 947 <td></td> 948 <td></td> 949 <td></td> 950 <td>●</td> 951 </tr> 952 <tr> 953 <td>16+</td> 954 <td>-</td> 955 <td>-</td> 956 <td>-</td> 957 <td>-</td> 958 <td>-</td> 959 <td>-</td> 960 <td class=fn>{@link #configure configure}</td> 961 <td>●</td> 962 <td>●</td> 963 <td>●</td> 964 <td>●</td> 965 <td>●</td> 966 <td>⁕</td> 967 <td>●</td> 968 <td>●</td> 969 </tr> 970 <tr> 971 <td>-</td> 972 <td>18+</td> 973 <td>-</td> 974 <td>-</td> 975 <td>-</td> 976 <td>-</td> 977 <td>-</td> 978 <td class=fn>{@link #createInputSurface createInputSurface}</td> 979 <td></td> 980 <td></td> 981 <td>⎋</td> 982 <td>⎋</td> 983 <td>⎋</td> 984 <td>⎋</td> 985 <td>⎋</td> 986 <td>⎋</td> 987 </tr> 988 <tr> 989 <td>-</td> 990 <td>-</td> 991 <td>16+</td> 992 <td>16+</td> 993 <td>(16+)</td> 994 <td>-</td> 995 <td>-</td> 996 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 997 <td>●</td> 998 <td>●</td> 999 <td>▧</td> 1000 <td>▧</td> 1001 <td>▧</td> 1002 <td>⁕▧↩</td> 1003 <td>▧↩</td> 1004 <td>▧↩</td> 1005 </tr> 1006 <tr> 1007 <td>-</td> 1008 <td>-</td> 1009 <td>16+</td> 1010 <td>16+</td> 1011 <td>16+</td> 1012 <td>-</td> 1013 <td>-</td> 1014 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1015 <td>●</td> 1016 <td>●</td> 1017 <td>●</td> 1018 <td>●</td> 1019 <td>●</td> 1020 <td>⁕↩</td> 1021 <td>↩</td> 1022 <td>↩</td> 1023 </tr> 1024 <tr> 1025 <td>-</td> 1026 <td>-</td> 1027 <td>16+</td> 1028 <td>16+</td> 1029 <td>16+</td> 1030 <td>-</td> 1031 <td>-</td> 1032 <td class=fn>{@link #flush flush}</td> 1033 <td>●</td> 1034 <td>●</td> 1035 <td>●</td> 1036 <td>●</td> 1037 <td>●</td> 1038 <td>●</td> 1039 <td>●</td> 1040 <td>●</td> 1041 </tr> 1042 <tr> 1043 <td>18+</td> 1044 <td>18+</td> 1045 <td>18+</td> 1046 <td>18+</td> 1047 <td>18+</td> 1048 <td>18+</td> 1049 <td>-</td> 1050 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1051 <td></td> 1052 <td></td> 1053 <td>●</td> 1054 <td>●</td> 1055 <td>●</td> 1056 <td>●</td> 1057 <td>●</td> 1058 <td>●</td> 1059 </tr> 1060 <tr> 1061 <td>-</td> 1062 <td>-</td> 1063 <td>(21+)</td> 1064 <td>21+</td> 1065 <td>(21+)</td> 1066 <td>-</td> 1067 <td>-</td> 1068 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1069 <td></td> 1070 <td></td> 1071 <td></td> 1072 <td></td> 1073 <td></td> 1074 <td>●</td> 1075 <td>●</td> 1076 <td>●</td> 1077 </tr> 1078 <tr> 1079 <td>-</td> 1080 <td>-</td> 1081 <td>16+</td> 1082 <td>(16+)</td> 1083 <td>(16+)</td> 1084 <td>-</td> 1085 <td>-</td> 1086 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1087 <td>●</td> 1088 <td>●</td> 1089 <td>●</td> 1090 <td>●</td> 1091 <td>●</td> 1092 <td>[⁕↩]</td> 1093 <td>[↩]</td> 1094 <td>[↩]</td> 1095 </tr> 1096 <tr> 1097 <td>-</td> 1098 <td>21+</td> 1099 <td>(21+)</td> 1100 <td>(21+)</td> 1101 <td>(21+)</td> 1102 <td>-</td> 1103 <td>-</td> 1104 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1105 <td></td> 1106 <td></td> 1107 <td></td> 1108 <td></td> 1109 <td></td> 1110 <td>●</td> 1111 <td>●</td> 1112 <td>●</td> 1113 </tr> 1114 <tr> 1115 <td>-</td> 1116 <td>-</td> 1117 <td>(21+)</td> 1118 <td>21+</td> 1119 <td>(21+)</td> 1120 <td>-</td> 1121 <td>-</td> 1122 <td class=fn>{@link #getInputImage getInputImage}</td> 1123 <td></td> 1124 <td></td> 1125 <td></td> 1126 <td></td> 1127 <td></td> 1128 <td>○</td> 1129 <td>●</td> 1130 <td>●</td> 1131 </tr> 1132 <tr> 1133 <td>18+</td> 1134 <td>18+</td> 1135 <td>18+</td> 1136 <td>18+</td> 1137 <td>18+</td> 1138 <td>18+</td> 1139 <td>-</td> 1140 <td class=fn>{@link #getName getName}</td> 1141 <td></td> 1142 <td></td> 1143 <td>●</td> 1144 <td>●</td> 1145 <td>●</td> 1146 <td>●</td> 1147 <td>●</td> 1148 <td>●</td> 1149 </tr> 1150 <tr> 1151 <td>-</td> 1152 <td>-</td> 1153 <td>(21+)</td> 1154 <td>21+</td> 1155 <td>21+</td> 1156 <td>-</td> 1157 <td>-</td> 1158 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1159 <td></td> 1160 <td></td> 1161 <td></td> 1162 <td></td> 1163 <td></td> 1164 <td>●</td> 1165 <td>●</td> 1166 <td>●</td> 1167 </tr> 1168 <tr> 1169 <td>-</td> 1170 <td>-</td> 1171 <td>16+</td> 1172 <td>16+</td> 1173 <td>16+</td> 1174 <td>-</td> 1175 <td>-</td> 1176 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1177 <td>●</td> 1178 <td>●</td> 1179 <td>●</td> 1180 <td>●</td> 1181 <td>●</td> 1182 <td>[⁕↩]</td> 1183 <td>[↩]</td> 1184 <td>[↩]</td> 1185 </tr> 1186 <tr> 1187 <td>-</td> 1188 <td>21+</td> 1189 <td>16+</td> 1190 <td>16+</td> 1191 <td>16+</td> 1192 <td>-</td> 1193 <td>-</td> 1194 <td class=fn>{@link #getOutputFormat()}</td> 1195 <td>●</td> 1196 <td>●</td> 1197 <td>●</td> 1198 <td>●</td> 1199 <td>●</td> 1200 <td>●</td> 1201 <td>●</td> 1202 <td>●</td> 1203 </tr> 1204 <tr> 1205 <td>-</td> 1206 <td>-</td> 1207 <td>(21+)</td> 1208 <td>21+</td> 1209 <td>21+</td> 1210 <td>-</td> 1211 <td>-</td> 1212 <td class=fn>{@link #getOutputFormat(int)}</td> 1213 <td></td> 1214 <td></td> 1215 <td></td> 1216 <td></td> 1217 <td></td> 1218 <td>●</td> 1219 <td>●</td> 1220 <td>●</td> 1221 </tr> 1222 <tr> 1223 <td>-</td> 1224 <td>-</td> 1225 <td>(21+)</td> 1226 <td>21+</td> 1227 <td>21+</td> 1228 <td>-</td> 1229 <td>-</td> 1230 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1231 <td></td> 1232 <td></td> 1233 <td></td> 1234 <td></td> 1235 <td></td> 1236 <td>○</td> 1237 <td>●</td> 1238 <td>●</td> 1239 </tr> 1240 <tr> 1241 <td>-</td> 1242 <td>-</td> 1243 <td>-</td> 1244 <td>16+</td> 1245 <td>(16+)</td> 1246 <td>-</td> 1247 <td>-</td> 1248 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1249 <td>●</td> 1250 <td>●</td> 1251 <td>●</td> 1252 <td>●</td> 1253 <td>●</td> 1254 <td>⁕</td> 1255 <td>●</td> 1256 <td>●</td> 1257 </tr> 1258 <tr> 1259 <td>-</td> 1260 <td>-</td> 1261 <td>-</td> 1262 <td>16+</td> 1263 <td>(16+)</td> 1264 <td>-</td> 1265 <td>-</td> 1266 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1267 <td>●</td> 1268 <td>●</td> 1269 <td>●</td> 1270 <td>●</td> 1271 <td>●</td> 1272 <td>⁕</td> 1273 <td>●</td> 1274 <td>●</td> 1275 </tr> 1276 <tr> 1277 <td>16+</td> 1278 <td>16+</td> 1279 <td>16+</td> 1280 <td>16+</td> 1281 <td>16+</td> 1282 <td>16+</td> 1283 <td>16+</td> 1284 <td class=fn>{@link #release release}</td> 1285 <td>●</td> 1286 <td>●</td> 1287 <td>●</td> 1288 <td>●</td> 1289 <td>●</td> 1290 <td>●</td> 1291 <td>●</td> 1292 <td>●</td> 1293 </tr> 1294 <tr> 1295 <td>-</td> 1296 <td>-</td> 1297 <td>-</td> 1298 <td>16+</td> 1299 <td>16+</td> 1300 <td>-</td> 1301 <td>-</td> 1302 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1303 <td>●</td> 1304 <td>●</td> 1305 <td>●</td> 1306 <td>●</td> 1307 <td>●</td> 1308 <td>⁕</td> 1309 <td>●</td> 1310 <td>⁕</td> 1311 </tr> 1312 <tr> 1313 <td>-</td> 1314 <td>-</td> 1315 <td>-</td> 1316 <td>21+</td> 1317 <td>21+</td> 1318 <td>-</td> 1319 <td>-</td> 1320 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1321 <td></td> 1322 <td></td> 1323 <td></td> 1324 <td></td> 1325 <td></td> 1326 <td>⎆</td> 1327 <td>⎆</td> 1328 <td>⎆</td> 1329 </tr> 1330 <tr> 1331 <td>21+</td> 1332 <td>21+</td> 1333 <td>21+</td> 1334 <td>21+</td> 1335 <td>21+</td> 1336 <td>21+</td> 1337 <td>-</td> 1338 <td class=fn>{@link #reset reset}</td> 1339 <td></td> 1340 <td></td> 1341 <td></td> 1342 <td></td> 1343 <td></td> 1344 <td>●</td> 1345 <td>●</td> 1346 <td>●</td> 1347 </tr> 1348 <tr> 1349 <td>21+</td> 1350 <td>-</td> 1351 <td>-</td> 1352 <td>-</td> 1353 <td>-</td> 1354 <td>-</td> 1355 <td>-</td> 1356 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1357 <td></td> 1358 <td></td> 1359 <td></td> 1360 <td></td> 1361 <td></td> 1362 <td>●</td> 1363 <td>●</td> 1364 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1365 </tr> 1366 <tr> 1367 <td>-</td> 1368 <td>23+</td> 1369 <td>-</td> 1370 <td>-</td> 1371 <td>-</td> 1372 <td>-</td> 1373 <td>-</td> 1374 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1375 <td></td> 1376 <td></td> 1377 <td></td> 1378 <td></td> 1379 <td></td> 1380 <td></td> 1381 <td></td> 1382 <td>⎋</td> 1383 </tr> 1384 <tr> 1385 <td>23+</td> 1386 <td>23+</td> 1387 <td>23+</td> 1388 <td>23+</td> 1389 <td>23+</td> 1390 <td>(23+)</td> 1391 <td>(23+)</td> 1392 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1393 <td></td> 1394 <td></td> 1395 <td></td> 1396 <td></td> 1397 <td></td> 1398 <td></td> 1399 <td></td> 1400 <td>○ ⎆</td> 1401 </tr> 1402 <tr> 1403 <td>-</td> 1404 <td>23+</td> 1405 <td>23+</td> 1406 <td>23+</td> 1407 <td>23+</td> 1408 <td>-</td> 1409 <td>-</td> 1410 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1411 <td></td> 1412 <td></td> 1413 <td></td> 1414 <td></td> 1415 <td></td> 1416 <td></td> 1417 <td></td> 1418 <td>⎆</td> 1419 </tr> 1420 <tr> 1421 <td>19+</td> 1422 <td>19+</td> 1423 <td>19+</td> 1424 <td>19+</td> 1425 <td>19+</td> 1426 <td>(19+)</td> 1427 <td>-</td> 1428 <td class=fn>{@link #setParameters setParameters}</td> 1429 <td></td> 1430 <td></td> 1431 <td></td> 1432 <td>●</td> 1433 <td>●</td> 1434 <td>●</td> 1435 <td>●</td> 1436 <td>●</td> 1437 </tr> 1438 <tr> 1439 <td>-</td> 1440 <td>(16+)</td> 1441 <td>(16+)</td> 1442 <td>16+</td> 1443 <td>(16+)</td> 1444 <td>(16+)</td> 1445 <td>-</td> 1446 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1447 <td>⎆</td> 1448 <td>⎆</td> 1449 <td>⎆</td> 1450 <td>⎆</td> 1451 <td>⎆</td> 1452 <td>⎆</td> 1453 <td>⎆</td> 1454 <td>⎆</td> 1455 </tr> 1456 <tr> 1457 <td>(29+)</td> 1458 <td>29+</td> 1459 <td>29+</td> 1460 <td>29+</td> 1461 <td>(29+)</td> 1462 <td>(29+)</td> 1463 <td>-</td> 1464 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1465 <td></td> 1466 <td></td> 1467 <td></td> 1468 <td></td> 1469 <td></td> 1470 <td></td> 1471 <td></td> 1472 <td></td> 1473 </tr> 1474 <tr> 1475 <td>-</td> 1476 <td>-</td> 1477 <td>18+</td> 1478 <td>18+</td> 1479 <td>-</td> 1480 <td>-</td> 1481 <td>-</td> 1482 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1483 <td></td> 1484 <td></td> 1485 <td>⎋</td> 1486 <td>⎋</td> 1487 <td>⎋</td> 1488 <td>⎋</td> 1489 <td>⎋</td> 1490 <td>⎋</td> 1491 </tr> 1492 <tr> 1493 <td>-</td> 1494 <td>16+</td> 1495 <td>21+(⇄)</td> 1496 <td>-</td> 1497 <td>-</td> 1498 <td>-</td> 1499 <td>-</td> 1500 <td class=fn>{@link #start start}</td> 1501 <td>●</td> 1502 <td>●</td> 1503 <td>●</td> 1504 <td>●</td> 1505 <td>●</td> 1506 <td>⁕</td> 1507 <td>●</td> 1508 <td>●</td> 1509 </tr> 1510 <tr> 1511 <td>-</td> 1512 <td>-</td> 1513 <td>16+</td> 1514 <td>16+</td> 1515 <td>16+</td> 1516 <td>-</td> 1517 <td>-</td> 1518 <td class=fn>{@link #stop stop}</td> 1519 <td>●</td> 1520 <td>●</td> 1521 <td>●</td> 1522 <td>●</td> 1523 <td>●</td> 1524 <td>●</td> 1525 <td>●</td> 1526 <td>●</td> 1527 </tr> 1528 </tbody> 1529 </table> 1530 */ 1531 final public class MediaCodec { 1532 /** 1533 * Per buffer metadata includes an offset and size specifying 1534 * the range of valid data in the associated codec (output) buffer. 1535 */ 1536 public final static class BufferInfo { 1537 /** 1538 * Update the buffer metadata information. 1539 * 1540 * @param newOffset the start-offset of the data in the buffer. 1541 * @param newSize the amount of data (in bytes) in the buffer. 1542 * @param newTimeUs the presentation timestamp in microseconds. 1543 * @param newFlags buffer flags associated with the buffer. This 1544 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1545 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1546 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1547 public void set( 1548 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1549 offset = newOffset; 1550 size = newSize; 1551 presentationTimeUs = newTimeUs; 1552 flags = newFlags; 1553 } 1554 1555 /** 1556 * The start-offset of the data in the buffer. 1557 */ 1558 public int offset; 1559 1560 /** 1561 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1562 * the buffer has no data in it and can be discarded. The only 1563 * use of a 0-size buffer is to carry the end-of-stream marker. 1564 */ 1565 public int size; 1566 1567 /** 1568 * The presentation timestamp in microseconds for the buffer. 1569 * This is derived from the presentation timestamp passed in 1570 * with the corresponding input buffer. This should be ignored for 1571 * a 0-sized buffer. 1572 */ 1573 public long presentationTimeUs; 1574 1575 /** 1576 * Buffer flags associated with the buffer. A combination of 1577 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1578 * 1579 * <p>Encoded buffers that are key frames are marked with 1580 * {@link #BUFFER_FLAG_KEY_FRAME}. 1581 * 1582 * <p>The last output buffer corresponding to the input buffer 1583 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1584 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1585 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1586 * marker. 1587 */ 1588 @BufferFlag 1589 public int flags; 1590 1591 /** @hide */ 1592 @NonNull dup()1593 public BufferInfo dup() { 1594 BufferInfo copy = new BufferInfo(); 1595 copy.set(offset, size, presentationTimeUs, flags); 1596 return copy; 1597 } 1598 }; 1599 1600 // The follow flag constants MUST stay in sync with their equivalents 1601 // in MediaCodec.h ! 1602 1603 /** 1604 * This indicates that the (encoded) buffer marked as such contains 1605 * the data for a key frame. 1606 * 1607 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1608 */ 1609 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1610 1611 /** 1612 * This indicates that the (encoded) buffer marked as such contains 1613 * the data for a key frame. 1614 */ 1615 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1616 1617 /** 1618 * This indicated that the buffer marked as such contains codec 1619 * initialization / codec specific data instead of media data. 1620 */ 1621 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1622 1623 /** 1624 * This signals the end of stream, i.e. no buffers will be available 1625 * after this, unless of course, {@link #flush} follows. 1626 */ 1627 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1628 1629 /** 1630 * This indicates that the buffer only contains part of a frame, 1631 * and the decoder should batch the data until a buffer without 1632 * this flag appears before decoding the frame. 1633 */ 1634 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1635 1636 /** 1637 * This indicates that the buffer contains non-media data for the 1638 * muxer to process. 1639 * 1640 * All muxer data should start with a FOURCC header that determines the type of data. 1641 * 1642 * For example, when it contains Exif data sent to a MediaMuxer track of 1643 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1644 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1645 * 1646 * @hide 1647 */ 1648 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1649 1650 /** @hide */ 1651 @IntDef( 1652 flag = true, 1653 value = { 1654 BUFFER_FLAG_SYNC_FRAME, 1655 BUFFER_FLAG_KEY_FRAME, 1656 BUFFER_FLAG_CODEC_CONFIG, 1657 BUFFER_FLAG_END_OF_STREAM, 1658 BUFFER_FLAG_PARTIAL_FRAME, 1659 BUFFER_FLAG_MUXER_DATA, 1660 }) 1661 @Retention(RetentionPolicy.SOURCE) 1662 public @interface BufferFlag {} 1663 1664 private EventHandler mEventHandler; 1665 private EventHandler mOnFrameRenderedHandler; 1666 private EventHandler mCallbackHandler; 1667 private Callback mCallback; 1668 private OnFrameRenderedListener mOnFrameRenderedListener; 1669 private final Object mListenerLock = new Object(); 1670 private MediaCodecInfo mCodecInfo; 1671 private final Object mCodecInfoLock = new Object(); 1672 private MediaCrypto mCrypto; 1673 1674 private static final int EVENT_CALLBACK = 1; 1675 private static final int EVENT_SET_CALLBACK = 2; 1676 private static final int EVENT_FRAME_RENDERED = 3; 1677 1678 private static final int CB_INPUT_AVAILABLE = 1; 1679 private static final int CB_OUTPUT_AVAILABLE = 2; 1680 private static final int CB_ERROR = 3; 1681 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1682 1683 private class EventHandler extends Handler { 1684 private MediaCodec mCodec; 1685 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1686 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1687 super(looper); 1688 mCodec = codec; 1689 } 1690 1691 @Override handleMessage(@onNull Message msg)1692 public void handleMessage(@NonNull Message msg) { 1693 switch (msg.what) { 1694 case EVENT_CALLBACK: 1695 { 1696 handleCallback(msg); 1697 break; 1698 } 1699 case EVENT_SET_CALLBACK: 1700 { 1701 mCallback = (MediaCodec.Callback) msg.obj; 1702 break; 1703 } 1704 case EVENT_FRAME_RENDERED: 1705 synchronized (mListenerLock) { 1706 Map<String, Object> map = (Map<String, Object>)msg.obj; 1707 for (int i = 0; ; ++i) { 1708 Object mediaTimeUs = map.get(i + "-media-time-us"); 1709 Object systemNano = map.get(i + "-system-nano"); 1710 if (mediaTimeUs == null || systemNano == null 1711 || mOnFrameRenderedListener == null) { 1712 break; 1713 } 1714 mOnFrameRenderedListener.onFrameRendered( 1715 mCodec, (long)mediaTimeUs, (long)systemNano); 1716 } 1717 break; 1718 } 1719 default: 1720 { 1721 break; 1722 } 1723 } 1724 } 1725 handleCallback(@onNull Message msg)1726 private void handleCallback(@NonNull Message msg) { 1727 if (mCallback == null) { 1728 return; 1729 } 1730 1731 switch (msg.arg1) { 1732 case CB_INPUT_AVAILABLE: 1733 { 1734 int index = msg.arg2; 1735 synchronized(mBufferLock) { 1736 validateInputByteBuffer(mCachedInputBuffers, index); 1737 } 1738 mCallback.onInputBufferAvailable(mCodec, index); 1739 break; 1740 } 1741 1742 case CB_OUTPUT_AVAILABLE: 1743 { 1744 int index = msg.arg2; 1745 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1746 synchronized(mBufferLock) { 1747 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1748 } 1749 mCallback.onOutputBufferAvailable( 1750 mCodec, index, info); 1751 break; 1752 } 1753 1754 case CB_ERROR: 1755 { 1756 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1757 break; 1758 } 1759 1760 case CB_OUTPUT_FORMAT_CHANGE: 1761 { 1762 mCallback.onOutputFormatChanged(mCodec, 1763 new MediaFormat((Map<String, Object>) msg.obj)); 1764 break; 1765 } 1766 1767 default: 1768 { 1769 break; 1770 } 1771 } 1772 } 1773 } 1774 1775 private boolean mHasSurface = false; 1776 1777 /** 1778 * Instantiate the preferred decoder supporting input data of the given mime type. 1779 * 1780 * The following is a partial list of defined mime types and their semantics: 1781 * <ul> 1782 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1783 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1784 * <li>"video/avc" - H.264/AVC video 1785 * <li>"video/hevc" - H.265/HEVC video 1786 * <li>"video/mp4v-es" - MPEG4 video 1787 * <li>"video/3gpp" - H.263 video 1788 * <li>"audio/3gpp" - AMR narrowband audio 1789 * <li>"audio/amr-wb" - AMR wideband audio 1790 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1791 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1792 * <li>"audio/vorbis" - vorbis audio 1793 * <li>"audio/g711-alaw" - G.711 alaw audio 1794 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1795 * </ul> 1796 * 1797 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1798 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1799 * given format. 1800 * 1801 * @param type The mime type of the input data. 1802 * @throws IOException if the codec cannot be created. 1803 * @throws IllegalArgumentException if type is not a valid mime type. 1804 * @throws NullPointerException if type is null. 1805 */ 1806 @NonNull createDecoderByType(@onNull String type)1807 public static MediaCodec createDecoderByType(@NonNull String type) 1808 throws IOException { 1809 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1810 } 1811 1812 /** 1813 * Instantiate the preferred encoder supporting output data of the given mime type. 1814 * 1815 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1816 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1817 * given format. 1818 * 1819 * @param type The desired mime type of the output data. 1820 * @throws IOException if the codec cannot be created. 1821 * @throws IllegalArgumentException if type is not a valid mime type. 1822 * @throws NullPointerException if type is null. 1823 */ 1824 @NonNull createEncoderByType(@onNull String type)1825 public static MediaCodec createEncoderByType(@NonNull String type) 1826 throws IOException { 1827 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1828 } 1829 1830 /** 1831 * If you know the exact name of the component you want to instantiate 1832 * use this method to instantiate it. Use with caution. 1833 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1834 * @param name The name of the codec to be instantiated. 1835 * @throws IOException if the codec cannot be created. 1836 * @throws IllegalArgumentException if name is not valid. 1837 * @throws NullPointerException if name is null. 1838 */ 1839 @NonNull createByCodecName(@onNull String name)1840 public static MediaCodec createByCodecName(@NonNull String name) 1841 throws IOException { 1842 return new MediaCodec( 1843 name, false /* nameIsType */, false /* unused */); 1844 } 1845 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1846 private MediaCodec( 1847 @NonNull String name, boolean nameIsType, boolean encoder) { 1848 Looper looper; 1849 if ((looper = Looper.myLooper()) != null) { 1850 mEventHandler = new EventHandler(this, looper); 1851 } else if ((looper = Looper.getMainLooper()) != null) { 1852 mEventHandler = new EventHandler(this, looper); 1853 } else { 1854 mEventHandler = null; 1855 } 1856 mCallbackHandler = mEventHandler; 1857 mOnFrameRenderedHandler = mEventHandler; 1858 1859 mBufferLock = new Object(); 1860 1861 // save name used at creation 1862 mNameAtCreation = nameIsType ? null : name; 1863 1864 native_setup(name, nameIsType, encoder); 1865 } 1866 1867 private String mNameAtCreation; 1868 1869 @Override finalize()1870 protected void finalize() { 1871 native_finalize(); 1872 mCrypto = null; 1873 } 1874 1875 /** 1876 * Returns the codec to its initial (Uninitialized) state. 1877 * 1878 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1879 * error has occured to reset the codec to its initial state after creation. 1880 * 1881 * @throws CodecException if an unrecoverable error has occured and the codec 1882 * could not be reset. 1883 * @throws IllegalStateException if in the Released state. 1884 */ reset()1885 public final void reset() { 1886 freeAllTrackedBuffers(); // free buffers first 1887 native_reset(); 1888 mCrypto = null; 1889 } 1890 native_reset()1891 private native final void native_reset(); 1892 1893 /** 1894 * Free up resources used by the codec instance. 1895 * 1896 * Make sure you call this when you're done to free up any opened 1897 * component instance instead of relying on the garbage collector 1898 * to do this for you at some point in the future. 1899 */ release()1900 public final void release() { 1901 freeAllTrackedBuffers(); // free buffers first 1902 native_release(); 1903 mCrypto = null; 1904 } 1905 native_release()1906 private native final void native_release(); 1907 1908 /** 1909 * If this codec is to be used as an encoder, pass this flag. 1910 */ 1911 public static final int CONFIGURE_FLAG_ENCODE = 1; 1912 1913 /** @hide */ 1914 @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE }) 1915 @Retention(RetentionPolicy.SOURCE) 1916 public @interface ConfigureFlag {} 1917 1918 /** 1919 * Configures a component. 1920 * 1921 * @param format The format of the input data (decoder) or the desired 1922 * format of the output data (encoder). Passing {@code null} 1923 * as {@code format} is equivalent to passing an 1924 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1925 * @param surface Specify a surface on which to render the output of this 1926 * decoder. Pass {@code null} as {@code surface} if the 1927 * codec does not generate raw video output (e.g. not a video 1928 * decoder) and/or if you want to configure the codec for 1929 * {@link ByteBuffer} output. 1930 * @param crypto Specify a crypto object to facilitate secure decryption 1931 * of the media data. Pass {@code null} as {@code crypto} for 1932 * non-secure codecs. 1933 * Please note that {@link MediaCodec} does NOT take ownership 1934 * of the {@link MediaCrypto} object; it is the application's 1935 * responsibility to properly cleanup the {@link MediaCrypto} object 1936 * when not in use. 1937 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1938 * component as an encoder. 1939 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1940 * or the format is unacceptable (e.g. missing a mandatory key), 1941 * or the flags are not set properly 1942 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1943 * @throws IllegalStateException if not in the Uninitialized state. 1944 * @throws CryptoException upon DRM error. 1945 * @throws CodecException upon codec error. 1946 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)1947 public void configure( 1948 @Nullable MediaFormat format, 1949 @Nullable Surface surface, @Nullable MediaCrypto crypto, 1950 @ConfigureFlag int flags) { 1951 configure(format, surface, crypto, null, flags); 1952 } 1953 1954 /** 1955 * Configure a component to be used with a descrambler. 1956 * @param format The format of the input data (decoder) or the desired 1957 * format of the output data (encoder). Passing {@code null} 1958 * as {@code format} is equivalent to passing an 1959 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1960 * @param surface Specify a surface on which to render the output of this 1961 * decoder. Pass {@code null} as {@code surface} if the 1962 * codec does not generate raw video output (e.g. not a video 1963 * decoder) and/or if you want to configure the codec for 1964 * {@link ByteBuffer} output. 1965 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1966 * component as an encoder. 1967 * @param descrambler Specify a descrambler object to facilitate secure 1968 * descrambling of the media data, or null for non-secure codecs. 1969 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1970 * or the format is unacceptable (e.g. missing a mandatory key), 1971 * or the flags are not set properly 1972 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1973 * @throws IllegalStateException if not in the Uninitialized state. 1974 * @throws CryptoException upon DRM error. 1975 * @throws CodecException upon codec error. 1976 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)1977 public void configure( 1978 @Nullable MediaFormat format, @Nullable Surface surface, 1979 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 1980 configure(format, surface, null, 1981 descrambler != null ? descrambler.getBinder() : null, flags); 1982 } 1983 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)1984 private void configure( 1985 @Nullable MediaFormat format, @Nullable Surface surface, 1986 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 1987 @ConfigureFlag int flags) { 1988 if (crypto != null && descramblerBinder != null) { 1989 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 1990 } 1991 1992 String[] keys = null; 1993 Object[] values = null; 1994 1995 if (format != null) { 1996 Map<String, Object> formatMap = format.getMap(); 1997 keys = new String[formatMap.size()]; 1998 values = new Object[formatMap.size()]; 1999 2000 int i = 0; 2001 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2002 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2003 int sessionId = 0; 2004 try { 2005 sessionId = (Integer)entry.getValue(); 2006 } 2007 catch (Exception e) { 2008 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2009 } 2010 keys[i] = "audio-hw-sync"; 2011 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2012 } else { 2013 keys[i] = entry.getKey(); 2014 values[i] = entry.getValue(); 2015 } 2016 ++i; 2017 } 2018 } 2019 2020 mHasSurface = surface != null; 2021 mCrypto = crypto; 2022 2023 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2024 } 2025 2026 /** 2027 * Dynamically sets the output surface of a codec. 2028 * <p> 2029 * This can only be used if the codec was configured with an output surface. The 2030 * new output surface should have a compatible usage type to the original output surface. 2031 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2032 * to ImageReader (software readable) output. 2033 * @param surface the output surface to use. It must not be {@code null}. 2034 * @throws IllegalStateException if the codec does not support setting the output 2035 * surface in the current state. 2036 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2037 */ setOutputSurface(@onNull Surface surface)2038 public void setOutputSurface(@NonNull Surface surface) { 2039 if (!mHasSurface) { 2040 throw new IllegalStateException("codec was not configured for an output surface"); 2041 } 2042 native_setSurface(surface); 2043 } 2044 native_setSurface(@onNull Surface surface)2045 private native void native_setSurface(@NonNull Surface surface); 2046 2047 /** 2048 * Create a persistent input surface that can be used with codecs that normally have an input 2049 * surface, such as video encoders. A persistent input can be reused by subsequent 2050 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2051 * most one codec or recorder instance concurrently. 2052 * <p> 2053 * The application is responsible for calling release() on the Surface when done. 2054 * 2055 * @return an input surface that can be used with {@link #setInputSurface}. 2056 */ 2057 @NonNull createPersistentInputSurface()2058 public static Surface createPersistentInputSurface() { 2059 return native_createPersistentInputSurface(); 2060 } 2061 2062 static class PersistentSurface extends Surface { 2063 @SuppressWarnings("unused") PersistentSurface()2064 PersistentSurface() {} // used by native 2065 2066 @Override release()2067 public void release() { 2068 native_releasePersistentInputSurface(this); 2069 super.release(); 2070 } 2071 2072 private long mPersistentObject; 2073 }; 2074 2075 /** 2076 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2077 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2078 * lieu of {@link #createInputSurface}. 2079 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2080 * @throws IllegalStateException if not in the Configured state or does not require an input 2081 * surface. 2082 * @throws IllegalArgumentException if the surface was not created by 2083 * {@link #createPersistentInputSurface}. 2084 */ setInputSurface(@onNull Surface surface)2085 public void setInputSurface(@NonNull Surface surface) { 2086 if (!(surface instanceof PersistentSurface)) { 2087 throw new IllegalArgumentException("not a PersistentSurface"); 2088 } 2089 native_setInputSurface(surface); 2090 } 2091 2092 @NonNull native_createPersistentInputSurface()2093 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2094 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2095 private native final void native_setInputSurface(@NonNull Surface surface); 2096 native_setCallback(@ullable Callback cb)2097 private native final void native_setCallback(@Nullable Callback cb); 2098 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2099 private native final void native_configure( 2100 @Nullable String[] keys, @Nullable Object[] values, 2101 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2102 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2103 2104 /** 2105 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2106 * may only be called after {@link #configure} and before {@link #start}. 2107 * <p> 2108 * The application is responsible for calling release() on the Surface when 2109 * done. 2110 * <p> 2111 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2112 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2113 * unexpected results. 2114 * @throws IllegalStateException if not in the Configured state. 2115 */ 2116 @NonNull createInputSurface()2117 public native final Surface createInputSurface(); 2118 2119 /** 2120 * After successfully configuring the component, call {@code start}. 2121 * <p> 2122 * Call {@code start} also if the codec is configured in asynchronous mode, 2123 * and it has just been flushed, to resume requesting input buffers. 2124 * @throws IllegalStateException if not in the Configured state 2125 * or just after {@link #flush} for a codec that is configured 2126 * in asynchronous mode. 2127 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2128 * for start may be attributed to future method calls. 2129 */ start()2130 public final void start() { 2131 native_start(); 2132 synchronized(mBufferLock) { 2133 cacheBuffers(true /* input */); 2134 cacheBuffers(false /* input */); 2135 } 2136 } native_start()2137 private native final void native_start(); 2138 2139 /** 2140 * Finish the decode/encode session, note that the codec instance 2141 * remains active and ready to be {@link #start}ed again. 2142 * To ensure that it is available to other client call {@link #release} 2143 * and don't just rely on garbage collection to eventually do this for you. 2144 * @throws IllegalStateException if in the Released state. 2145 */ stop()2146 public final void stop() { 2147 native_stop(); 2148 freeAllTrackedBuffers(); 2149 2150 synchronized (mListenerLock) { 2151 if (mCallbackHandler != null) { 2152 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2153 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2154 } 2155 if (mOnFrameRenderedHandler != null) { 2156 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2157 } 2158 } 2159 } 2160 native_stop()2161 private native final void native_stop(); 2162 2163 /** 2164 * Flush both input and output ports of the component. 2165 * <p> 2166 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2167 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2168 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2169 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2170 * invalid, and all buffers are owned by the codec. 2171 * <p> 2172 * If the codec is configured in asynchronous mode, call {@link #start} 2173 * after {@code flush} has returned to resume codec operations. The codec 2174 * will not request input buffers until this has happened. 2175 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2176 * callbacks that were not handled prior to calling {@code flush}. 2177 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2178 * should be discarded.</strong> 2179 * <p> 2180 * If the codec is configured in synchronous mode, codec will resume 2181 * automatically if it is configured with an input surface. Otherwise, it 2182 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2183 * 2184 * @throws IllegalStateException if not in the Executing state. 2185 * @throws MediaCodec.CodecException upon codec error. 2186 */ flush()2187 public final void flush() { 2188 synchronized(mBufferLock) { 2189 invalidateByteBuffers(mCachedInputBuffers); 2190 invalidateByteBuffers(mCachedOutputBuffers); 2191 mDequeuedInputBuffers.clear(); 2192 mDequeuedOutputBuffers.clear(); 2193 } 2194 native_flush(); 2195 } 2196 native_flush()2197 private native final void native_flush(); 2198 2199 /** 2200 * Thrown when an internal codec error occurs. 2201 */ 2202 public final static class CodecException extends IllegalStateException { 2203 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2204 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2205 super(detailMessage); 2206 mErrorCode = errorCode; 2207 mActionCode = actionCode; 2208 2209 // TODO get this from codec 2210 final String sign = errorCode < 0 ? "neg_" : ""; 2211 mDiagnosticInfo = 2212 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2213 } 2214 2215 /** 2216 * Returns true if the codec exception is a transient issue, 2217 * perhaps due to resource constraints, and that the method 2218 * (or encoding/decoding) may be retried at a later time. 2219 */ 2220 public boolean isTransient() { 2221 return mActionCode == ACTION_TRANSIENT; 2222 } 2223 2224 /** 2225 * Returns true if the codec cannot proceed further, 2226 * but can be recovered by stopping, configuring, 2227 * and starting again. 2228 */ 2229 public boolean isRecoverable() { 2230 return mActionCode == ACTION_RECOVERABLE; 2231 } 2232 2233 /** 2234 * Retrieve the error code associated with a CodecException 2235 */ 2236 public int getErrorCode() { 2237 return mErrorCode; 2238 } 2239 2240 /** 2241 * Retrieve a developer-readable diagnostic information string 2242 * associated with the exception. Do not show this to end-users, 2243 * since this string will not be localized or generally 2244 * comprehensible to end-users. 2245 */ 2246 public @NonNull String getDiagnosticInfo() { 2247 return mDiagnosticInfo; 2248 } 2249 2250 /** 2251 * This indicates required resource was not able to be allocated. 2252 */ 2253 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2254 2255 /** 2256 * This indicates the resource manager reclaimed the media resource used by the codec. 2257 * <p> 2258 * With this exception, the codec must be released, as it has moved to terminal state. 2259 */ 2260 public static final int ERROR_RECLAIMED = 1101; 2261 2262 /** @hide */ 2263 @IntDef({ 2264 ERROR_INSUFFICIENT_RESOURCE, 2265 ERROR_RECLAIMED, 2266 }) 2267 @Retention(RetentionPolicy.SOURCE) 2268 public @interface ReasonCode {} 2269 2270 /* Must be in sync with android_media_MediaCodec.cpp */ 2271 private final static int ACTION_TRANSIENT = 1; 2272 private final static int ACTION_RECOVERABLE = 2; 2273 2274 private final String mDiagnosticInfo; 2275 private final int mErrorCode; 2276 private final int mActionCode; 2277 } 2278 2279 /** 2280 * Thrown when a crypto error occurs while queueing a secure input buffer. 2281 */ 2282 public final static class CryptoException extends RuntimeException { 2283 public CryptoException(int errorCode, @Nullable String detailMessage) { 2284 super(detailMessage); 2285 mErrorCode = errorCode; 2286 } 2287 2288 /** 2289 * This indicates that the requested key was not found when trying to 2290 * perform a decrypt operation. The operation can be retried after adding 2291 * the correct decryption key. 2292 */ 2293 public static final int ERROR_NO_KEY = 1; 2294 2295 /** 2296 * This indicates that the key used for decryption is no longer 2297 * valid due to license term expiration. The operation can be retried 2298 * after updating the expired keys. 2299 */ 2300 public static final int ERROR_KEY_EXPIRED = 2; 2301 2302 /** 2303 * This indicates that a required crypto resource was not able to be 2304 * allocated while attempting the requested operation. The operation 2305 * can be retried if the app is able to release resources. 2306 */ 2307 public static final int ERROR_RESOURCE_BUSY = 3; 2308 2309 /** 2310 * This indicates that the output protection levels supported by the 2311 * device are not sufficient to meet the requirements set by the 2312 * content owner in the license policy. 2313 */ 2314 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4; 2315 2316 /** 2317 * This indicates that decryption was attempted on a session that is 2318 * not opened, which could be due to a failure to open the session, 2319 * closing the session prematurely, or the session being reclaimed 2320 * by the resource manager. 2321 */ 2322 public static final int ERROR_SESSION_NOT_OPENED = 5; 2323 2324 /** 2325 * This indicates that an operation was attempted that could not be 2326 * supported by the crypto system of the device in its current 2327 * configuration. It may occur when the license policy requires 2328 * device security features that aren't supported by the device, 2329 * or due to an internal error in the crypto system that prevents 2330 * the specified security policy from being met. 2331 */ 2332 public static final int ERROR_UNSUPPORTED_OPERATION = 6; 2333 2334 /** 2335 * This indicates that the security level of the device is not 2336 * sufficient to meet the requirements set by the content owner 2337 * in the license policy. 2338 */ 2339 public static final int ERROR_INSUFFICIENT_SECURITY = 7; 2340 2341 /** 2342 * This indicates that the video frame being decrypted exceeds 2343 * the size of the device's protected output buffers. When 2344 * encountering this error the app should try playing content 2345 * of a lower resolution. 2346 */ 2347 public static final int ERROR_FRAME_TOO_LARGE = 8; 2348 2349 /** 2350 * This error indicates that session state has been 2351 * invalidated. It can occur on devices that are not capable 2352 * of retaining crypto session state across device 2353 * suspend/resume. The session must be closed and a new 2354 * session opened to resume operation. 2355 */ 2356 public static final int ERROR_LOST_STATE = 9; 2357 2358 /** @hide */ 2359 @IntDef({ 2360 ERROR_NO_KEY, 2361 ERROR_KEY_EXPIRED, 2362 ERROR_RESOURCE_BUSY, 2363 ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2364 ERROR_SESSION_NOT_OPENED, 2365 ERROR_UNSUPPORTED_OPERATION, 2366 ERROR_INSUFFICIENT_SECURITY, 2367 ERROR_FRAME_TOO_LARGE, 2368 ERROR_LOST_STATE 2369 }) 2370 @Retention(RetentionPolicy.SOURCE) 2371 public @interface CryptoErrorCode {} 2372 2373 /** 2374 * Retrieve the error code associated with a CryptoException 2375 */ 2376 @CryptoErrorCode 2377 public int getErrorCode() { 2378 return mErrorCode; 2379 } 2380 2381 private int mErrorCode; 2382 } 2383 2384 /** 2385 * After filling a range of the input buffer at the specified index 2386 * submit it to the component. Once an input buffer is queued to 2387 * the codec, it MUST NOT be used until it is later retrieved by 2388 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2389 * return value or a {@link Callback#onInputBufferAvailable} 2390 * callback. 2391 * <p> 2392 * Many decoders require the actual compressed data stream to be 2393 * preceded by "codec specific data", i.e. setup data used to initialize 2394 * the codec such as PPS/SPS in the case of AVC video or code tables 2395 * in the case of vorbis audio. 2396 * The class {@link android.media.MediaExtractor} provides codec 2397 * specific data as part of 2398 * the returned track format in entries named "csd-0", "csd-1" ... 2399 * <p> 2400 * These buffers can be submitted directly after {@link #start} or 2401 * {@link #flush} by specifying the flag {@link 2402 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2403 * codec with a {@link MediaFormat} containing these keys, they 2404 * will be automatically submitted by MediaCodec directly after 2405 * start. Therefore, the use of {@link 2406 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2407 * recommended only for advanced users. 2408 * <p> 2409 * To indicate that this is the final piece of input data (or rather that 2410 * no more input data follows unless the decoder is subsequently flushed) 2411 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2412 * <p class=note> 2413 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2414 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2415 * Surface output buffers, and the resulting frame timestamp was undefined. 2416 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2417 * Similarly, since frame timestamps can be used by the destination surface for rendering 2418 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2419 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2420 * SurfaceView specifics}).</strong> 2421 * 2422 * @param index The index of a client-owned input buffer previously returned 2423 * in a call to {@link #dequeueInputBuffer}. 2424 * @param offset The byte offset into the input buffer at which the data starts. 2425 * @param size The number of bytes of valid input data. 2426 * @param presentationTimeUs The presentation timestamp in microseconds for this 2427 * buffer. This is normally the media time at which this 2428 * buffer should be presented (rendered). When using an output 2429 * surface, this will be propagated as the {@link 2430 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2431 * conversion to nanoseconds). 2432 * @param flags A bitmask of flags 2433 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2434 * While not prohibited, most codecs do not use the 2435 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2436 * @throws IllegalStateException if not in the Executing state. 2437 * @throws MediaCodec.CodecException upon codec error. 2438 * @throws CryptoException if a crypto object has been specified in 2439 * {@link #configure} 2440 */ 2441 public final void queueInputBuffer( 2442 int index, 2443 int offset, int size, long presentationTimeUs, int flags) 2444 throws CryptoException { 2445 synchronized(mBufferLock) { 2446 invalidateByteBuffer(mCachedInputBuffers, index); 2447 mDequeuedInputBuffers.remove(index); 2448 } 2449 try { 2450 native_queueInputBuffer( 2451 index, offset, size, presentationTimeUs, flags); 2452 } catch (CryptoException | IllegalStateException e) { 2453 revalidateByteBuffer(mCachedInputBuffers, index); 2454 throw e; 2455 } 2456 } 2457 2458 private native final void native_queueInputBuffer( 2459 int index, 2460 int offset, int size, long presentationTimeUs, int flags) 2461 throws CryptoException; 2462 2463 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2464 public static final int CRYPTO_MODE_AES_CTR = 1; 2465 public static final int CRYPTO_MODE_AES_CBC = 2; 2466 2467 /** 2468 * Metadata describing the structure of an encrypted input sample. 2469 * <p> 2470 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2471 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2472 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2473 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2474 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2475 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2476 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2477 * <p> 2478 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2479 * "Common encryption in ISO base media file format files". 2480 * <p> 2481 * <h3>ISO-CENC Schemes</h3> 2482 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2483 * corresponding to each possible combination of an AES mode with the presence or absence of 2484 * patterned encryption. 2485 * 2486 * <table style="width: 0%"> 2487 * <thead> 2488 * <tr> 2489 * <th> </th> 2490 * <th>AES-CTR</th> 2491 * <th>AES-CBC</th> 2492 * </tr> 2493 * </thead> 2494 * <tbody> 2495 * <tr> 2496 * <th>Without Patterns</th> 2497 * <td>cenc</td> 2498 * <td>cbc1</td> 2499 * </tr><tr> 2500 * <th>With Patterns</th> 2501 * <td>cens</td> 2502 * <td>cbcs</td> 2503 * </tr> 2504 * </tbody> 2505 * </table> 2506 * 2507 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2508 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2509 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2510 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2511 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2512 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2513 * {@link #setPattern} is never called is all zeroes. 2514 * <p> 2515 * <h4>HLS SAMPLE-AES Audio</h4> 2516 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2517 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2518 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2519 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2520 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2521 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2522 * while still decrypting every block. 2523 */ 2524 public final static class CryptoInfo { 2525 /** 2526 * The number of subSamples that make up the buffer's contents. 2527 */ 2528 public int numSubSamples; 2529 /** 2530 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2531 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2532 */ 2533 public int[] numBytesOfClearData; 2534 /** 2535 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2536 * as clear and {@link #numBytesOfClearData} must be specified. 2537 */ 2538 public int[] numBytesOfEncryptedData; 2539 /** 2540 * A 16-byte key id 2541 */ 2542 public byte[] key; 2543 /** 2544 * A 16-byte initialization vector 2545 */ 2546 public byte[] iv; 2547 /** 2548 * The type of encryption that has been applied, 2549 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2550 * and {@link #CRYPTO_MODE_AES_CBC} 2551 */ 2552 public int mode; 2553 2554 /** 2555 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2556 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2557 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2558 */ 2559 public final static class Pattern { 2560 /** 2561 * Number of blocks to be encrypted in the pattern. If both this and 2562 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2563 */ 2564 private int mEncryptBlocks; 2565 2566 /** 2567 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2568 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2569 */ 2570 private int mSkipBlocks; 2571 2572 /** 2573 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2574 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2575 */ 2576 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2577 set(blocksToEncrypt, blocksToSkip); 2578 } 2579 2580 /** 2581 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2582 * parameters are zero, pattern encryption is inoperative. 2583 */ 2584 public void set(int blocksToEncrypt, int blocksToSkip) { 2585 mEncryptBlocks = blocksToEncrypt; 2586 mSkipBlocks = blocksToSkip; 2587 } 2588 2589 /** 2590 * Return the number of blocks to skip in a sample encryption pattern. 2591 */ 2592 public int getSkipBlocks() { 2593 return mSkipBlocks; 2594 } 2595 2596 /** 2597 * Return the number of blocks to encrypt in a sample encryption pattern. 2598 */ 2599 public int getEncryptBlocks() { 2600 return mEncryptBlocks; 2601 } 2602 }; 2603 2604 private final Pattern zeroPattern = new Pattern(0, 0); 2605 2606 /** 2607 * The pattern applicable to the protected data in each subsample. 2608 */ 2609 private Pattern pattern; 2610 2611 /** 2612 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2613 * a {@link MediaCodec.CryptoInfo} instance. 2614 */ 2615 public void set( 2616 int newNumSubSamples, 2617 @NonNull int[] newNumBytesOfClearData, 2618 @NonNull int[] newNumBytesOfEncryptedData, 2619 @NonNull byte[] newKey, 2620 @NonNull byte[] newIV, 2621 int newMode) { 2622 numSubSamples = newNumSubSamples; 2623 numBytesOfClearData = newNumBytesOfClearData; 2624 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2625 key = newKey; 2626 iv = newIV; 2627 mode = newMode; 2628 pattern = zeroPattern; 2629 } 2630 2631 /** 2632 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2633 * See {@link MediaCodec.CryptoInfo.Pattern}. 2634 */ 2635 public void setPattern(Pattern newPattern) { 2636 pattern = newPattern; 2637 } 2638 2639 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 2640 pattern = new Pattern(blocksToEncrypt, blocksToSkip); 2641 } 2642 2643 @Override 2644 public String toString() { 2645 StringBuilder builder = new StringBuilder(); 2646 builder.append(numSubSamples + " subsamples, key ["); 2647 String hexdigits = "0123456789abcdef"; 2648 for (int i = 0; i < key.length; i++) { 2649 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2650 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2651 } 2652 builder.append("], iv ["); 2653 for (int i = 0; i < key.length; i++) { 2654 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2655 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2656 } 2657 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2658 builder.append(Arrays.toString(numBytesOfClearData)); 2659 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2660 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2661 return builder.toString(); 2662 } 2663 }; 2664 2665 /** 2666 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2667 * potentially encrypted. 2668 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2669 * 2670 * @param index The index of a client-owned input buffer previously returned 2671 * in a call to {@link #dequeueInputBuffer}. 2672 * @param offset The byte offset into the input buffer at which the data starts. 2673 * @param info Metadata required to facilitate decryption, the object can be 2674 * reused immediately after this call returns. 2675 * @param presentationTimeUs The presentation timestamp in microseconds for this 2676 * buffer. This is normally the media time at which this 2677 * buffer should be presented (rendered). 2678 * @param flags A bitmask of flags 2679 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2680 * While not prohibited, most codecs do not use the 2681 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2682 * @throws IllegalStateException if not in the Executing state. 2683 * @throws MediaCodec.CodecException upon codec error. 2684 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2685 * An error code associated with the exception helps identify the 2686 * reason for the failure. 2687 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2688 public final void queueSecureInputBuffer( 2689 int index, 2690 int offset, 2691 @NonNull CryptoInfo info, 2692 long presentationTimeUs, 2693 int flags) throws CryptoException { 2694 synchronized(mBufferLock) { 2695 invalidateByteBuffer(mCachedInputBuffers, index); 2696 mDequeuedInputBuffers.remove(index); 2697 } 2698 try { 2699 native_queueSecureInputBuffer( 2700 index, offset, info, presentationTimeUs, flags); 2701 } catch (CryptoException | IllegalStateException e) { 2702 revalidateByteBuffer(mCachedInputBuffers, index); 2703 throw e; 2704 } 2705 } 2706 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2707 private native final void native_queueSecureInputBuffer( 2708 int index, 2709 int offset, 2710 @NonNull CryptoInfo info, 2711 long presentationTimeUs, 2712 int flags) throws CryptoException; 2713 2714 /** 2715 * Returns the index of an input buffer to be filled with valid data 2716 * or -1 if no such buffer is currently available. 2717 * This method will return immediately if timeoutUs == 0, wait indefinitely 2718 * for the availability of an input buffer if timeoutUs < 0 or wait up 2719 * to "timeoutUs" microseconds if timeoutUs > 0. 2720 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2721 * @throws IllegalStateException if not in the Executing state, 2722 * or codec is configured in asynchronous mode. 2723 * @throws MediaCodec.CodecException upon codec error. 2724 */ dequeueInputBuffer(long timeoutUs)2725 public final int dequeueInputBuffer(long timeoutUs) { 2726 int res = native_dequeueInputBuffer(timeoutUs); 2727 if (res >= 0) { 2728 synchronized(mBufferLock) { 2729 validateInputByteBuffer(mCachedInputBuffers, res); 2730 } 2731 } 2732 return res; 2733 } 2734 native_dequeueInputBuffer(long timeoutUs)2735 private native final int native_dequeueInputBuffer(long timeoutUs); 2736 2737 /** 2738 * If a non-negative timeout had been specified in the call 2739 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 2740 */ 2741 public static final int INFO_TRY_AGAIN_LATER = -1; 2742 2743 /** 2744 * The output format has changed, subsequent data will follow the new 2745 * format. {@link #getOutputFormat()} returns the new format. Note, that 2746 * you can also use the new {@link #getOutputFormat(int)} method to 2747 * get the format for a specific output buffer. This frees you from 2748 * having to track output format changes. 2749 */ 2750 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 2751 2752 /** 2753 * The output buffers have changed, the client must refer to the new 2754 * set of output buffers returned by {@link #getOutputBuffers} from 2755 * this point on. 2756 * 2757 * <p>Additionally, this event signals that the video scaling mode 2758 * may have been reset to the default.</p> 2759 * 2760 * @deprecated This return value can be ignored as {@link 2761 * #getOutputBuffers} has been deprecated. Client should 2762 * request a current buffer using on of the get-buffer or 2763 * get-image methods each time one has been dequeued. 2764 */ 2765 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 2766 2767 /** @hide */ 2768 @IntDef({ 2769 INFO_TRY_AGAIN_LATER, 2770 INFO_OUTPUT_FORMAT_CHANGED, 2771 INFO_OUTPUT_BUFFERS_CHANGED, 2772 }) 2773 @Retention(RetentionPolicy.SOURCE) 2774 public @interface OutputBufferInfo {} 2775 2776 /** 2777 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 2778 * Returns the index of an output buffer that has been successfully 2779 * decoded or one of the INFO_* constants. 2780 * @param info Will be filled with buffer meta data. 2781 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2782 * @throws IllegalStateException if not in the Executing state, 2783 * or codec is configured in asynchronous mode. 2784 * @throws MediaCodec.CodecException upon codec error. 2785 */ 2786 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2787 public final int dequeueOutputBuffer( 2788 @NonNull BufferInfo info, long timeoutUs) { 2789 int res = native_dequeueOutputBuffer(info, timeoutUs); 2790 synchronized(mBufferLock) { 2791 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 2792 cacheBuffers(false /* input */); 2793 } else if (res >= 0) { 2794 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 2795 if (mHasSurface) { 2796 mDequeuedOutputInfos.put(res, info.dup()); 2797 } 2798 } 2799 } 2800 return res; 2801 } 2802 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2803 private native final int native_dequeueOutputBuffer( 2804 @NonNull BufferInfo info, long timeoutUs); 2805 2806 /** 2807 * If you are done with a buffer, use this call to return the buffer to the codec 2808 * or to render it on the output surface. If you configured the codec with an 2809 * output surface, setting {@code render} to {@code true} will first send the buffer 2810 * to that output surface. The surface will release the buffer back to the codec once 2811 * it is no longer used/displayed. 2812 * 2813 * Once an output buffer is released to the codec, it MUST NOT 2814 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2815 * to a {@link #dequeueOutputBuffer} return value or a 2816 * {@link Callback#onOutputBufferAvailable} callback. 2817 * 2818 * @param index The index of a client-owned output buffer previously returned 2819 * from a call to {@link #dequeueOutputBuffer}. 2820 * @param render If a valid surface was specified when configuring the codec, 2821 * passing true renders this output buffer to the surface. 2822 * @throws IllegalStateException if not in the Executing state. 2823 * @throws MediaCodec.CodecException upon codec error. 2824 */ releaseOutputBuffer(int index, boolean render)2825 public final void releaseOutputBuffer(int index, boolean render) { 2826 BufferInfo info = null; 2827 synchronized(mBufferLock) { 2828 invalidateByteBuffer(mCachedOutputBuffers, index); 2829 mDequeuedOutputBuffers.remove(index); 2830 if (mHasSurface) { 2831 info = mDequeuedOutputInfos.remove(index); 2832 } 2833 } 2834 releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); 2835 } 2836 2837 /** 2838 * If you are done with a buffer, use this call to update its surface timestamp 2839 * and return it to the codec to render it on the output surface. If you 2840 * have not specified an output surface when configuring this video codec, 2841 * this call will simply return the buffer to the codec.<p> 2842 * 2843 * The timestamp may have special meaning depending on the destination surface. 2844 * 2845 * <table> 2846 * <tr><th>SurfaceView specifics</th></tr> 2847 * <tr><td> 2848 * If you render your buffer on a {@link android.view.SurfaceView}, 2849 * you can use the timestamp to render the buffer at a specific time (at the 2850 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 2851 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 2852 * Currently, this is set as within one (1) second. A few notes: 2853 * 2854 * <ul> 2855 * <li>the buffer will not be returned to the codec until the timestamp 2856 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 2857 * <li>buffers are processed sequentially, so you may block subsequent buffers to 2858 * be displayed on the {@link android.view.Surface}. This is important if you 2859 * want to react to user action, e.g. stop the video or seek. 2860 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 2861 * rendered at the same VSYNC, the last one will be shown, and the other ones 2862 * will be dropped. 2863 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 2864 * time, the {@link android.view.Surface} will ignore the timestamp, and 2865 * display the buffer at the earliest feasible time. In this mode it will not 2866 * drop frames. 2867 * <li>for best performance and quality, call this method when you are about 2868 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 2869 * about 33 msec. 2870 * </ul> 2871 * </td></tr> 2872 * </table> 2873 * 2874 * Once an output buffer is released to the codec, it MUST NOT 2875 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2876 * to a {@link #dequeueOutputBuffer} return value or a 2877 * {@link Callback#onOutputBufferAvailable} callback. 2878 * 2879 * @param index The index of a client-owned output buffer previously returned 2880 * from a call to {@link #dequeueOutputBuffer}. 2881 * @param renderTimestampNs The timestamp to associate with this buffer when 2882 * it is sent to the Surface. 2883 * @throws IllegalStateException if not in the Executing state. 2884 * @throws MediaCodec.CodecException upon codec error. 2885 */ releaseOutputBuffer(int index, long renderTimestampNs)2886 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 2887 BufferInfo info = null; 2888 synchronized(mBufferLock) { 2889 invalidateByteBuffer(mCachedOutputBuffers, index); 2890 mDequeuedOutputBuffers.remove(index); 2891 if (mHasSurface) { 2892 info = mDequeuedOutputInfos.remove(index); 2893 } 2894 } 2895 releaseOutputBuffer( 2896 index, true /* render */, true /* updatePTS */, renderTimestampNs); 2897 } 2898 2899 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)2900 private native final void releaseOutputBuffer( 2901 int index, boolean render, boolean updatePTS, long timeNs); 2902 2903 /** 2904 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 2905 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 2906 * encoders receiving input from a Surface created by {@link #createInputSurface}. 2907 * @throws IllegalStateException if not in the Executing state. 2908 * @throws MediaCodec.CodecException upon codec error. 2909 */ signalEndOfInputStream()2910 public native final void signalEndOfInputStream(); 2911 2912 /** 2913 * Call this after dequeueOutputBuffer signals a format change by returning 2914 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 2915 * You can also call this after {@link #configure} returns 2916 * successfully to get the output format initially configured 2917 * for the codec. Do this to determine what optional 2918 * configuration parameters were supported by the codec. 2919 * 2920 * @throws IllegalStateException if not in the Executing or 2921 * Configured state. 2922 * @throws MediaCodec.CodecException upon codec error. 2923 */ 2924 @NonNull getOutputFormat()2925 public final MediaFormat getOutputFormat() { 2926 return new MediaFormat(getFormatNative(false /* input */)); 2927 } 2928 2929 /** 2930 * Call this after {@link #configure} returns successfully to 2931 * get the input format accepted by the codec. Do this to 2932 * determine what optional configuration parameters were 2933 * supported by the codec. 2934 * 2935 * @throws IllegalStateException if not in the Executing or 2936 * Configured state. 2937 * @throws MediaCodec.CodecException upon codec error. 2938 */ 2939 @NonNull getInputFormat()2940 public final MediaFormat getInputFormat() { 2941 return new MediaFormat(getFormatNative(true /* input */)); 2942 } 2943 2944 /** 2945 * Returns the output format for a specific output buffer. 2946 * 2947 * @param index The index of a client-owned input buffer previously 2948 * returned from a call to {@link #dequeueInputBuffer}. 2949 * 2950 * @return the format for the output buffer, or null if the index 2951 * is not a dequeued output buffer. 2952 */ 2953 @NonNull getOutputFormat(int index)2954 public final MediaFormat getOutputFormat(int index) { 2955 return new MediaFormat(getOutputFormatNative(index)); 2956 } 2957 2958 @NonNull getFormatNative(boolean input)2959 private native final Map<String, Object> getFormatNative(boolean input); 2960 2961 @NonNull getOutputFormatNative(int index)2962 private native final Map<String, Object> getOutputFormatNative(int index); 2963 2964 // used to track dequeued buffers 2965 private static class BufferMap { 2966 // various returned representations of the codec buffer 2967 private static class CodecBuffer { 2968 private Image mImage; 2969 private ByteBuffer mByteBuffer; 2970 free()2971 public void free() { 2972 if (mByteBuffer != null) { 2973 // all of our ByteBuffers are direct 2974 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 2975 mByteBuffer = null; 2976 } 2977 if (mImage != null) { 2978 mImage.close(); 2979 mImage = null; 2980 } 2981 } 2982 setImage(@ullable Image image)2983 public void setImage(@Nullable Image image) { 2984 free(); 2985 mImage = image; 2986 } 2987 setByteBuffer(@ullable ByteBuffer buffer)2988 public void setByteBuffer(@Nullable ByteBuffer buffer) { 2989 free(); 2990 mByteBuffer = buffer; 2991 } 2992 } 2993 2994 private final Map<Integer, CodecBuffer> mMap = 2995 new HashMap<Integer, CodecBuffer>(); 2996 remove(int index)2997 public void remove(int index) { 2998 CodecBuffer buffer = mMap.get(index); 2999 if (buffer != null) { 3000 buffer.free(); 3001 mMap.remove(index); 3002 } 3003 } 3004 put(int index, @Nullable ByteBuffer newBuffer)3005 public void put(int index, @Nullable ByteBuffer newBuffer) { 3006 CodecBuffer buffer = mMap.get(index); 3007 if (buffer == null) { // likely 3008 buffer = new CodecBuffer(); 3009 mMap.put(index, buffer); 3010 } 3011 buffer.setByteBuffer(newBuffer); 3012 } 3013 put(int index, @Nullable Image newImage)3014 public void put(int index, @Nullable Image newImage) { 3015 CodecBuffer buffer = mMap.get(index); 3016 if (buffer == null) { // likely 3017 buffer = new CodecBuffer(); 3018 mMap.put(index, buffer); 3019 } 3020 buffer.setImage(newImage); 3021 } 3022 clear()3023 public void clear() { 3024 for (CodecBuffer buffer: mMap.values()) { 3025 buffer.free(); 3026 } 3027 mMap.clear(); 3028 } 3029 } 3030 3031 private ByteBuffer[] mCachedInputBuffers; 3032 private ByteBuffer[] mCachedOutputBuffers; 3033 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 3034 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 3035 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 3036 new HashMap<Integer, BufferInfo>(); 3037 final private Object mBufferLock; 3038 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3039 private final void invalidateByteBuffer( 3040 @Nullable ByteBuffer[] buffers, int index) { 3041 if (buffers != null && index >= 0 && index < buffers.length) { 3042 ByteBuffer buffer = buffers[index]; 3043 if (buffer != null) { 3044 buffer.setAccessible(false); 3045 } 3046 } 3047 } 3048 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)3049 private final void validateInputByteBuffer( 3050 @Nullable ByteBuffer[] buffers, int index) { 3051 if (buffers != null && index >= 0 && index < buffers.length) { 3052 ByteBuffer buffer = buffers[index]; 3053 if (buffer != null) { 3054 buffer.setAccessible(true); 3055 buffer.clear(); 3056 } 3057 } 3058 } 3059 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3060 private final void revalidateByteBuffer( 3061 @Nullable ByteBuffer[] buffers, int index) { 3062 synchronized(mBufferLock) { 3063 if (buffers != null && index >= 0 && index < buffers.length) { 3064 ByteBuffer buffer = buffers[index]; 3065 if (buffer != null) { 3066 buffer.setAccessible(true); 3067 } 3068 } 3069 } 3070 } 3071 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)3072 private final void validateOutputByteBuffer( 3073 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 3074 if (buffers != null && index >= 0 && index < buffers.length) { 3075 ByteBuffer buffer = buffers[index]; 3076 if (buffer != null) { 3077 buffer.setAccessible(true); 3078 buffer.limit(info.offset + info.size).position(info.offset); 3079 } 3080 } 3081 } 3082 invalidateByteBuffers(@ullable ByteBuffer[] buffers)3083 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 3084 if (buffers != null) { 3085 for (ByteBuffer buffer: buffers) { 3086 if (buffer != null) { 3087 buffer.setAccessible(false); 3088 } 3089 } 3090 } 3091 } 3092 freeByteBuffer(@ullable ByteBuffer buffer)3093 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 3094 if (buffer != null /* && buffer.isDirect() */) { 3095 // all of our ByteBuffers are direct 3096 java.nio.NioUtils.freeDirectBuffer(buffer); 3097 } 3098 } 3099 freeByteBuffers(@ullable ByteBuffer[] buffers)3100 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 3101 if (buffers != null) { 3102 for (ByteBuffer buffer: buffers) { 3103 freeByteBuffer(buffer); 3104 } 3105 } 3106 } 3107 freeAllTrackedBuffers()3108 private final void freeAllTrackedBuffers() { 3109 synchronized(mBufferLock) { 3110 freeByteBuffers(mCachedInputBuffers); 3111 freeByteBuffers(mCachedOutputBuffers); 3112 mCachedInputBuffers = null; 3113 mCachedOutputBuffers = null; 3114 mDequeuedInputBuffers.clear(); 3115 mDequeuedOutputBuffers.clear(); 3116 } 3117 } 3118 cacheBuffers(boolean input)3119 private final void cacheBuffers(boolean input) { 3120 ByteBuffer[] buffers = null; 3121 try { 3122 buffers = getBuffers(input); 3123 invalidateByteBuffers(buffers); 3124 } catch (IllegalStateException e) { 3125 // we don't get buffers in async mode 3126 } 3127 if (input) { 3128 mCachedInputBuffers = buffers; 3129 } else { 3130 mCachedOutputBuffers = buffers; 3131 } 3132 } 3133 3134 /** 3135 * Retrieve the set of input buffers. Call this after start() 3136 * returns. After calling this method, any ByteBuffers 3137 * previously returned by an earlier call to this method MUST no 3138 * longer be used. 3139 * 3140 * @deprecated Use the new {@link #getInputBuffer} method instead 3141 * each time an input buffer is dequeued. 3142 * 3143 * <b>Note:</b> As of API 21, dequeued input buffers are 3144 * automatically {@link java.nio.Buffer#clear cleared}. 3145 * 3146 * <em>Do not use this method if using an input surface.</em> 3147 * 3148 * @throws IllegalStateException if not in the Executing state, 3149 * or codec is configured in asynchronous mode. 3150 * @throws MediaCodec.CodecException upon codec error. 3151 */ 3152 @NonNull getInputBuffers()3153 public ByteBuffer[] getInputBuffers() { 3154 if (mCachedInputBuffers == null) { 3155 throw new IllegalStateException(); 3156 } 3157 // FIXME: check codec status 3158 return mCachedInputBuffers; 3159 } 3160 3161 /** 3162 * Retrieve the set of output buffers. Call this after start() 3163 * returns and whenever dequeueOutputBuffer signals an output 3164 * buffer change by returning {@link 3165 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 3166 * ByteBuffers previously returned by an earlier call to this 3167 * method MUST no longer be used. 3168 * 3169 * @deprecated Use the new {@link #getOutputBuffer} method instead 3170 * each time an output buffer is dequeued. This method is not 3171 * supported if codec is configured in asynchronous mode. 3172 * 3173 * <b>Note:</b> As of API 21, the position and limit of output 3174 * buffers that are dequeued will be set to the valid data 3175 * range. 3176 * 3177 * <em>Do not use this method if using an output surface.</em> 3178 * 3179 * @throws IllegalStateException if not in the Executing state, 3180 * or codec is configured in asynchronous mode. 3181 * @throws MediaCodec.CodecException upon codec error. 3182 */ 3183 @NonNull getOutputBuffers()3184 public ByteBuffer[] getOutputBuffers() { 3185 if (mCachedOutputBuffers == null) { 3186 throw new IllegalStateException(); 3187 } 3188 // FIXME: check codec status 3189 return mCachedOutputBuffers; 3190 } 3191 3192 /** 3193 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 3194 * object for a dequeued input buffer index to contain the input data. 3195 * 3196 * After calling this method any ByteBuffer or Image object 3197 * previously returned for the same input index MUST no longer 3198 * be used. 3199 * 3200 * @param index The index of a client-owned input buffer previously 3201 * returned from a call to {@link #dequeueInputBuffer}, 3202 * or received via an onInputBufferAvailable callback. 3203 * 3204 * @return the input buffer, or null if the index is not a dequeued 3205 * input buffer, or if the codec is configured for surface input. 3206 * 3207 * @throws IllegalStateException if not in the Executing state. 3208 * @throws MediaCodec.CodecException upon codec error. 3209 */ 3210 @Nullable getInputBuffer(int index)3211 public ByteBuffer getInputBuffer(int index) { 3212 ByteBuffer newBuffer = getBuffer(true /* input */, index); 3213 synchronized(mBufferLock) { 3214 invalidateByteBuffer(mCachedInputBuffers, index); 3215 mDequeuedInputBuffers.put(index, newBuffer); 3216 } 3217 return newBuffer; 3218 } 3219 3220 /** 3221 * Returns a writable Image object for a dequeued input buffer 3222 * index to contain the raw input video frame. 3223 * 3224 * After calling this method any ByteBuffer or Image object 3225 * previously returned for the same input index MUST no longer 3226 * be used. 3227 * 3228 * @param index The index of a client-owned input buffer previously 3229 * returned from a call to {@link #dequeueInputBuffer}, 3230 * or received via an onInputBufferAvailable callback. 3231 * 3232 * @return the input image, or null if the index is not a 3233 * dequeued input buffer, or not a ByteBuffer that contains a 3234 * raw image. 3235 * 3236 * @throws IllegalStateException if not in the Executing state. 3237 * @throws MediaCodec.CodecException upon codec error. 3238 */ 3239 @Nullable getInputImage(int index)3240 public Image getInputImage(int index) { 3241 Image newImage = getImage(true /* input */, index); 3242 synchronized(mBufferLock) { 3243 invalidateByteBuffer(mCachedInputBuffers, index); 3244 mDequeuedInputBuffers.put(index, newImage); 3245 } 3246 return newImage; 3247 } 3248 3249 /** 3250 * Returns a read-only ByteBuffer for a dequeued output buffer 3251 * index. The position and limit of the returned buffer are set 3252 * to the valid output data. 3253 * 3254 * After calling this method, any ByteBuffer or Image object 3255 * previously returned for the same output index MUST no longer 3256 * be used. 3257 * 3258 * @param index The index of a client-owned output buffer previously 3259 * returned from a call to {@link #dequeueOutputBuffer}, 3260 * or received via an onOutputBufferAvailable callback. 3261 * 3262 * @return the output buffer, or null if the index is not a dequeued 3263 * output buffer, or the codec is configured with an output surface. 3264 * 3265 * @throws IllegalStateException if not in the Executing state. 3266 * @throws MediaCodec.CodecException upon codec error. 3267 */ 3268 @Nullable getOutputBuffer(int index)3269 public ByteBuffer getOutputBuffer(int index) { 3270 ByteBuffer newBuffer = getBuffer(false /* input */, index); 3271 synchronized(mBufferLock) { 3272 invalidateByteBuffer(mCachedOutputBuffers, index); 3273 mDequeuedOutputBuffers.put(index, newBuffer); 3274 } 3275 return newBuffer; 3276 } 3277 3278 /** 3279 * Returns a read-only Image object for a dequeued output buffer 3280 * index that contains the raw video frame. 3281 * 3282 * After calling this method, any ByteBuffer or Image object previously 3283 * returned for the same output index MUST no longer be used. 3284 * 3285 * @param index The index of a client-owned output buffer previously 3286 * returned from a call to {@link #dequeueOutputBuffer}, 3287 * or received via an onOutputBufferAvailable callback. 3288 * 3289 * @return the output image, or null if the index is not a 3290 * dequeued output buffer, not a raw video frame, or if the codec 3291 * was configured with an output surface. 3292 * 3293 * @throws IllegalStateException if not in the Executing state. 3294 * @throws MediaCodec.CodecException upon codec error. 3295 */ 3296 @Nullable getOutputImage(int index)3297 public Image getOutputImage(int index) { 3298 Image newImage = getImage(false /* input */, index); 3299 synchronized(mBufferLock) { 3300 invalidateByteBuffer(mCachedOutputBuffers, index); 3301 mDequeuedOutputBuffers.put(index, newImage); 3302 } 3303 return newImage; 3304 } 3305 3306 /** 3307 * The content is scaled to the surface dimensions 3308 */ 3309 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 3310 3311 /** 3312 * The content is scaled, maintaining its aspect ratio, the whole 3313 * surface area is used, content may be cropped. 3314 * <p class=note> 3315 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 3316 * configure the pixel aspect ratio for a {@link Surface}. 3317 * <p class=note> 3318 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 3319 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 3320 */ 3321 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 3322 3323 /** @hide */ 3324 @IntDef({ 3325 VIDEO_SCALING_MODE_SCALE_TO_FIT, 3326 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 3327 }) 3328 @Retention(RetentionPolicy.SOURCE) 3329 public @interface VideoScalingMode {} 3330 3331 /** 3332 * If a surface has been specified in a previous call to {@link #configure} 3333 * specifies the scaling mode to use. The default is "scale to fit". 3334 * <p class=note> 3335 * The scaling mode may be reset to the <strong>default</strong> each time an 3336 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 3337 * must call this method after every buffer change event (and before the first output buffer is 3338 * released for rendering) to ensure consistent scaling mode. 3339 * <p class=note> 3340 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 3341 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 3342 * 3343 * @throws IllegalArgumentException if mode is not recognized. 3344 * @throws IllegalStateException if in the Released state. 3345 */ setVideoScalingMode(@ideoScalingMode int mode)3346 public native final void setVideoScalingMode(@VideoScalingMode int mode); 3347 3348 /** 3349 * Sets the audio presentation. 3350 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 3351 */ setAudioPresentation(@onNull AudioPresentation presentation)3352 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 3353 if (presentation == null) { 3354 throw new NullPointerException("audio presentation is null"); 3355 } 3356 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 3357 } 3358 native_setAudioPresentation(int presentationId, int programId)3359 private native void native_setAudioPresentation(int presentationId, int programId); 3360 3361 /** 3362 * Retrieve the codec name. 3363 * 3364 * If the codec was created by createDecoderByType or createEncoderByType, what component is 3365 * chosen is not known beforehand. This method returns the name of the codec that was 3366 * selected by the platform. 3367 * 3368 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 3369 * names) for the same underlying codec, any of which can be used to instantiate the same 3370 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 3371 * name used to create the codec in this case. 3372 * 3373 * @throws IllegalStateException if in the Released state. 3374 */ 3375 @NonNull getName()3376 public final String getName() { 3377 // get canonical name to handle exception 3378 String canonicalName = getCanonicalName(); 3379 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 3380 } 3381 3382 /** 3383 * Retrieve the underlying codec name. 3384 * 3385 * This method is similar to {@link #getName}, except that it returns the underlying component 3386 * name even if an alias was used to create this MediaCodec object by name, 3387 * 3388 * @throws IllegalStateException if in the Released state. 3389 */ 3390 @NonNull getCanonicalName()3391 public native final String getCanonicalName(); 3392 3393 /** 3394 * Return Metrics data about the current codec instance. 3395 * 3396 * @return a {@link PersistableBundle} containing the set of attributes and values 3397 * available for the media being handled by this instance of MediaCodec 3398 * The attributes are descibed in {@link MetricsConstants}. 3399 * 3400 * Additional vendor-specific fields may also be present in 3401 * the return value. 3402 */ getMetrics()3403 public PersistableBundle getMetrics() { 3404 PersistableBundle bundle = native_getMetrics(); 3405 return bundle; 3406 } 3407 native_getMetrics()3408 private native PersistableBundle native_getMetrics(); 3409 3410 /** 3411 * Change a video encoder's target bitrate on the fly. The value is an 3412 * Integer object containing the new bitrate in bps. 3413 * 3414 * @see #setParameters(Bundle) 3415 */ 3416 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 3417 3418 /** 3419 * Temporarily suspend/resume encoding of input data. While suspended 3420 * input data is effectively discarded instead of being fed into the 3421 * encoder. This parameter really only makes sense to use with an encoder 3422 * in "surface-input" mode, as the client code has no control over the 3423 * input-side of the encoder in that case. 3424 * The value is an Integer object containing the value 1 to suspend 3425 * or the value 0 to resume. 3426 * 3427 * @see #setParameters(Bundle) 3428 */ 3429 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 3430 3431 /** 3432 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 3433 * optionally use this key to specify the timestamp (in micro-second) 3434 * at which the suspend/resume operation takes effect. 3435 * 3436 * Note that the specified timestamp must be greater than or equal to the 3437 * timestamp of any previously queued suspend/resume operations. 3438 * 3439 * The value is a long int, indicating the timestamp to suspend/resume. 3440 * 3441 * @see #setParameters(Bundle) 3442 */ 3443 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 3444 3445 /** 3446 * Specify an offset (in micro-second) to be added on top of the timestamps 3447 * onward. A typical use case is to apply an adjust to the timestamps after 3448 * a period of pause by the user. 3449 * 3450 * This parameter can only be used on an encoder in "surface-input" mode. 3451 * 3452 * The value is a long int, indicating the timestamp offset to be applied. 3453 * 3454 * @see #setParameters(Bundle) 3455 */ 3456 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 3457 3458 /** 3459 * Request that the encoder produce a sync frame "soon". 3460 * Provide an Integer with the value 0. 3461 * 3462 * @see #setParameters(Bundle) 3463 */ 3464 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 3465 3466 /** 3467 * Set the HDR10+ metadata on the next queued input frame. 3468 * 3469 * Provide a byte array of data that's conforming to the 3470 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 3471 *<p> 3472 * For decoders: 3473 *<p> 3474 * When a decoder is configured for one of the HDR10+ profiles that uses 3475 * out-of-band metadata (such as {@link 3476 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 3477 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 3478 * parameter sets the HDR10+ metadata on the next input buffer queued 3479 * to the decoder. A decoder supporting these profiles must propagate 3480 * the metadata to the format of the output buffer corresponding to this 3481 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 3482 * The metadata should be applied to that output buffer and the buffers 3483 * following it (in display order), until the next output buffer (in 3484 * display order) upon which an HDR10+ metadata is set. 3485 *<p> 3486 * This parameter shouldn't be set if the decoder is not configured for 3487 * an HDR10+ profile that uses out-of-band metadata. In particular, 3488 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 3489 * where the metadata is embedded in the input buffers, for example 3490 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 3491 *<p> 3492 * For encoders: 3493 *<p> 3494 * When an encoder is configured for one of the HDR10+ profiles and the 3495 * operates in byte buffer input mode (instead of surface input mode), 3496 * this parameter sets the HDR10+ metadata on the next input buffer queued 3497 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 3498 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 3499 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 3500 * the metadata must be propagated to the format of the output buffer 3501 * corresponding to this particular input buffer (under key {@link 3502 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 3503 * in-band metadata (such as {@link 3504 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 3505 * metadata info must be embedded in the corresponding output buffer itself. 3506 *<p> 3507 * This parameter shouldn't be set if the encoder is not configured for 3508 * an HDR10+ profile, or if it's operating in surface input mode. 3509 *<p> 3510 * 3511 * @see MediaFormat#KEY_HDR10_PLUS_INFO 3512 */ 3513 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 3514 3515 /** 3516 * Communicate additional parameter changes to the component instance. 3517 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 3518 * 3519 * @param params The bundle of parameters to set. 3520 * @throws IllegalStateException if in the Released state. 3521 */ setParameters(@ullable Bundle params)3522 public final void setParameters(@Nullable Bundle params) { 3523 if (params == null) { 3524 return; 3525 } 3526 3527 String[] keys = new String[params.size()]; 3528 Object[] values = new Object[params.size()]; 3529 3530 int i = 0; 3531 for (final String key: params.keySet()) { 3532 keys[i] = key; 3533 Object value = params.get(key); 3534 3535 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 3536 if (value instanceof byte[]) { 3537 values[i] = ByteBuffer.wrap((byte[])value); 3538 } else { 3539 values[i] = value; 3540 } 3541 ++i; 3542 } 3543 3544 setParameters(keys, values); 3545 } 3546 3547 /** 3548 * Sets an asynchronous callback for actionable MediaCodec events. 3549 * 3550 * If the client intends to use the component in asynchronous mode, 3551 * a valid callback should be provided before {@link #configure} is called. 3552 * 3553 * When asynchronous callback is enabled, the client should not call 3554 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 3555 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 3556 * <p> 3557 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 3558 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 3559 * even if an input surface was created. 3560 * 3561 * @param cb The callback that will run. Use {@code null} to clear a previously 3562 * set callback (before {@link #configure configure} is called and run 3563 * in synchronous mode). 3564 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 3565 * callbacks are done on the default thread (the caller's thread or the 3566 * main thread.) 3567 */ setCallback(@ullable Callback cb, @Nullable Handler handler)3568 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 3569 if (cb != null) { 3570 synchronized (mListenerLock) { 3571 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 3572 // NOTE: there are no callbacks on the handler at this time, but check anyways 3573 // even if we were to extend this to be callable dynamically, it must 3574 // be called when codec is flushed, so no messages are pending. 3575 if (newHandler != mCallbackHandler) { 3576 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3577 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3578 mCallbackHandler = newHandler; 3579 } 3580 } 3581 } else if (mCallbackHandler != null) { 3582 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3583 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3584 } 3585 3586 if (mCallbackHandler != null) { 3587 // set java callback on main handler 3588 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 3589 mCallbackHandler.sendMessage(msg); 3590 3591 // set native handler here, don't post to handler because 3592 // it may cause the callback to be delayed and set in a wrong state. 3593 // Note that native codec may start sending events to the callback 3594 // handler after this returns. 3595 native_setCallback(cb); 3596 } 3597 } 3598 3599 /** 3600 * Sets an asynchronous callback for actionable MediaCodec events on the default 3601 * looper. 3602 * <p> 3603 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 3604 * @param cb The callback that will run. Use {@code null} to clear a previously 3605 * set callback (before {@link #configure configure} is called and run 3606 * in synchronous mode). 3607 * @see #setCallback(Callback, Handler) 3608 */ setCallback(@ullable Callback cb)3609 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 3610 setCallback(cb, null /* handler */); 3611 } 3612 3613 /** 3614 * Listener to be called when an output frame has rendered on the output surface 3615 * 3616 * @see MediaCodec#setOnFrameRenderedListener 3617 */ 3618 public interface OnFrameRenderedListener { 3619 3620 /** 3621 * Called when an output frame has rendered on the output surface. 3622 * <p> 3623 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3624 * render timing samples, and can be significantly delayed and batched. Some frames may have 3625 * been rendered even if there was no callback generated. 3626 * 3627 * @param codec the MediaCodec instance 3628 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 3629 * This is usually the same as specified in {@link #queueInputBuffer}; however, 3630 * some codecs may alter the media time by applying some time-based transformation, 3631 * such as frame rate conversion. In that case, presentation time corresponds 3632 * to the actual output frame rendered. 3633 * @param nanoTime The system time when the frame was rendered. 3634 * 3635 * @see System#nanoTime 3636 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)3637 public void onFrameRendered( 3638 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 3639 } 3640 3641 /** 3642 * Registers a callback to be invoked when an output frame is rendered on the output surface. 3643 * <p> 3644 * This method can be called in any codec state, but will only have an effect in the 3645 * Executing state for codecs that render buffers to the output surface. 3646 * <p> 3647 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3648 * render timing samples, and can be significantly delayed and batched. Some frames may have 3649 * been rendered even if there was no callback generated. 3650 * 3651 * @param listener the callback that will be run 3652 * @param handler the callback will be run on the handler's thread. If {@code null}, 3653 * the callback will be run on the default thread, which is the looper 3654 * from which the codec was created, or a new thread if there was none. 3655 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)3656 public void setOnFrameRenderedListener( 3657 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 3658 synchronized (mListenerLock) { 3659 mOnFrameRenderedListener = listener; 3660 if (listener != null) { 3661 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 3662 if (newHandler != mOnFrameRenderedHandler) { 3663 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3664 } 3665 mOnFrameRenderedHandler = newHandler; 3666 } else if (mOnFrameRenderedHandler != null) { 3667 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3668 } 3669 native_enableOnFrameRenderedListener(listener != null); 3670 } 3671 } 3672 native_enableOnFrameRenderedListener(boolean enable)3673 private native void native_enableOnFrameRenderedListener(boolean enable); 3674 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)3675 private EventHandler getEventHandlerOn( 3676 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 3677 if (handler == null) { 3678 return mEventHandler; 3679 } else { 3680 Looper looper = handler.getLooper(); 3681 if (lastHandler.getLooper() == looper) { 3682 return lastHandler; 3683 } else { 3684 return new EventHandler(this, looper); 3685 } 3686 } 3687 } 3688 3689 /** 3690 * MediaCodec callback interface. Used to notify the user asynchronously 3691 * of various MediaCodec events. 3692 */ 3693 public static abstract class Callback { 3694 /** 3695 * Called when an input buffer becomes available. 3696 * 3697 * @param codec The MediaCodec object. 3698 * @param index The index of the available input buffer. 3699 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)3700 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 3701 3702 /** 3703 * Called when an output buffer becomes available. 3704 * 3705 * @param codec The MediaCodec object. 3706 * @param index The index of the available output buffer. 3707 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 3708 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)3709 public abstract void onOutputBufferAvailable( 3710 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 3711 3712 /** 3713 * Called when the MediaCodec encountered an error 3714 * 3715 * @param codec The MediaCodec object. 3716 * @param e The {@link MediaCodec.CodecException} object describing the error. 3717 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)3718 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 3719 3720 /** 3721 * Called when the output format has changed 3722 * 3723 * @param codec The MediaCodec object. 3724 * @param format The new output format. 3725 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)3726 public abstract void onOutputFormatChanged( 3727 @NonNull MediaCodec codec, @NonNull MediaFormat format); 3728 } 3729 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)3730 private void postEventFromNative( 3731 int what, int arg1, int arg2, @Nullable Object obj) { 3732 synchronized (mListenerLock) { 3733 EventHandler handler = mEventHandler; 3734 if (what == EVENT_CALLBACK) { 3735 handler = mCallbackHandler; 3736 } else if (what == EVENT_FRAME_RENDERED) { 3737 handler = mOnFrameRenderedHandler; 3738 } 3739 if (handler != null) { 3740 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 3741 handler.sendMessage(msg); 3742 } 3743 } 3744 } 3745 3746 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)3747 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 3748 3749 /** 3750 * Get the codec info. If the codec was created by createDecoderByType 3751 * or createEncoderByType, what component is chosen is not known beforehand, 3752 * and thus the caller does not have the MediaCodecInfo. 3753 * @throws IllegalStateException if in the Released state. 3754 */ 3755 @NonNull getCodecInfo()3756 public MediaCodecInfo getCodecInfo() { 3757 // Get the codec name first. If the codec is already released, 3758 // IllegalStateException will be thrown here. 3759 String name = getName(); 3760 synchronized (mCodecInfoLock) { 3761 if (mCodecInfo == null) { 3762 // Get the codec info for this codec itself first. Only initialize 3763 // the full codec list if this somehow fails because it can be slow. 3764 mCodecInfo = getOwnCodecInfo(); 3765 if (mCodecInfo == null) { 3766 mCodecInfo = MediaCodecList.getInfoFor(name); 3767 } 3768 } 3769 return mCodecInfo; 3770 } 3771 } 3772 3773 @NonNull getOwnCodecInfo()3774 private native final MediaCodecInfo getOwnCodecInfo(); 3775 3776 @NonNull 3777 @UnsupportedAppUsage getBuffers(boolean input)3778 private native final ByteBuffer[] getBuffers(boolean input); 3779 3780 @Nullable getBuffer(boolean input, int index)3781 private native final ByteBuffer getBuffer(boolean input, int index); 3782 3783 @Nullable getImage(boolean input, int index)3784 private native final Image getImage(boolean input, int index); 3785 native_init()3786 private static native final void native_init(); 3787 native_setup( @onNull String name, boolean nameIsType, boolean encoder)3788 private native final void native_setup( 3789 @NonNull String name, boolean nameIsType, boolean encoder); 3790 native_finalize()3791 private native final void native_finalize(); 3792 3793 static { 3794 System.loadLibrary("media_jni"); native_init()3795 native_init(); 3796 } 3797 3798 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 3799 private long mNativeContext = 0; 3800 private final Lock mNativeContextLock = new ReentrantLock(); 3801 lockAndGetContext()3802 private final long lockAndGetContext() { 3803 mNativeContextLock.lock(); 3804 return mNativeContext; 3805 } 3806 setAndUnlockContext(long context)3807 private final void setAndUnlockContext(long context) { 3808 mNativeContext = context; 3809 mNativeContextLock.unlock(); 3810 } 3811 3812 /** @hide */ 3813 public static class MediaImage extends Image { 3814 private final boolean mIsReadOnly; 3815 private final int mWidth; 3816 private final int mHeight; 3817 private final int mFormat; 3818 private long mTimestamp; 3819 private final Plane[] mPlanes; 3820 private final ByteBuffer mBuffer; 3821 private final ByteBuffer mInfo; 3822 private final int mXOffset; 3823 private final int mYOffset; 3824 3825 private final static int TYPE_YUV = 1; 3826 3827 private final int mTransform = 0; //Default no transform 3828 private final int mScalingMode = 0; //Default frozen scaling mode 3829 3830 @Override getFormat()3831 public int getFormat() { 3832 throwISEIfImageIsInvalid(); 3833 return mFormat; 3834 } 3835 3836 @Override getHeight()3837 public int getHeight() { 3838 throwISEIfImageIsInvalid(); 3839 return mHeight; 3840 } 3841 3842 @Override getWidth()3843 public int getWidth() { 3844 throwISEIfImageIsInvalid(); 3845 return mWidth; 3846 } 3847 3848 @Override getTransform()3849 public int getTransform() { 3850 throwISEIfImageIsInvalid(); 3851 return mTransform; 3852 } 3853 3854 @Override getScalingMode()3855 public int getScalingMode() { 3856 throwISEIfImageIsInvalid(); 3857 return mScalingMode; 3858 } 3859 3860 @Override getTimestamp()3861 public long getTimestamp() { 3862 throwISEIfImageIsInvalid(); 3863 return mTimestamp; 3864 } 3865 3866 @Override 3867 @NonNull getPlanes()3868 public Plane[] getPlanes() { 3869 throwISEIfImageIsInvalid(); 3870 return Arrays.copyOf(mPlanes, mPlanes.length); 3871 } 3872 3873 @Override close()3874 public void close() { 3875 if (mIsImageValid) { 3876 java.nio.NioUtils.freeDirectBuffer(mBuffer); 3877 mIsImageValid = false; 3878 } 3879 } 3880 3881 /** 3882 * Set the crop rectangle associated with this frame. 3883 * <p> 3884 * The crop rectangle specifies the region of valid pixels in the image, 3885 * using coordinates in the largest-resolution plane. 3886 */ 3887 @Override setCropRect(@ullable Rect cropRect)3888 public void setCropRect(@Nullable Rect cropRect) { 3889 if (mIsReadOnly) { 3890 throw new ReadOnlyBufferException(); 3891 } 3892 super.setCropRect(cropRect); 3893 } 3894 3895 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)3896 public MediaImage( 3897 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 3898 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 3899 mFormat = ImageFormat.YUV_420_888; 3900 mTimestamp = timestamp; 3901 mIsImageValid = true; 3902 mIsReadOnly = buffer.isReadOnly(); 3903 mBuffer = buffer.duplicate(); 3904 3905 // save offsets and info 3906 mXOffset = xOffset; 3907 mYOffset = yOffset; 3908 mInfo = info; 3909 3910 // read media-info. See MediaImage2 3911 if (info.remaining() == 104) { 3912 int type = info.getInt(); 3913 if (type != TYPE_YUV) { 3914 throw new UnsupportedOperationException("unsupported type: " + type); 3915 } 3916 int numPlanes = info.getInt(); 3917 if (numPlanes != 3) { 3918 throw new RuntimeException("unexpected number of planes: " + numPlanes); 3919 } 3920 mWidth = info.getInt(); 3921 mHeight = info.getInt(); 3922 if (mWidth < 1 || mHeight < 1) { 3923 throw new UnsupportedOperationException( 3924 "unsupported size: " + mWidth + "x" + mHeight); 3925 } 3926 int bitDepth = info.getInt(); 3927 if (bitDepth != 8) { 3928 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 3929 } 3930 int bitDepthAllocated = info.getInt(); 3931 if (bitDepthAllocated != 8) { 3932 throw new UnsupportedOperationException( 3933 "unsupported allocated bit depth: " + bitDepthAllocated); 3934 } 3935 mPlanes = new MediaPlane[numPlanes]; 3936 for (int ix = 0; ix < numPlanes; ix++) { 3937 int planeOffset = info.getInt(); 3938 int colInc = info.getInt(); 3939 int rowInc = info.getInt(); 3940 int horiz = info.getInt(); 3941 int vert = info.getInt(); 3942 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 3943 throw new UnsupportedOperationException("unexpected subsampling: " 3944 + horiz + "x" + vert + " on plane " + ix); 3945 } 3946 if (colInc < 1 || rowInc < 1) { 3947 throw new UnsupportedOperationException("unexpected strides: " 3948 + colInc + " pixel, " + rowInc + " row on plane " + ix); 3949 } 3950 3951 buffer.clear(); 3952 buffer.position(mBuffer.position() + planeOffset 3953 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 3954 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 3955 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 3956 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 3957 } 3958 } else { 3959 throw new UnsupportedOperationException( 3960 "unsupported info length: " + info.remaining()); 3961 } 3962 3963 if (cropRect == null) { 3964 cropRect = new Rect(0, 0, mWidth, mHeight); 3965 } 3966 cropRect.offset(-xOffset, -yOffset); 3967 super.setCropRect(cropRect); 3968 } 3969 3970 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)3971 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 3972 mData = buffer; 3973 mRowInc = rowInc; 3974 mColInc = colInc; 3975 } 3976 3977 @Override getRowStride()3978 public int getRowStride() { 3979 throwISEIfImageIsInvalid(); 3980 return mRowInc; 3981 } 3982 3983 @Override getPixelStride()3984 public int getPixelStride() { 3985 throwISEIfImageIsInvalid(); 3986 return mColInc; 3987 } 3988 3989 @Override 3990 @NonNull getBuffer()3991 public ByteBuffer getBuffer() { 3992 throwISEIfImageIsInvalid(); 3993 return mData; 3994 } 3995 3996 private final int mRowInc; 3997 private final int mColInc; 3998 private final ByteBuffer mData; 3999 } 4000 } 4001 4002 public final static class MetricsConstants 4003 { MetricsConstants()4004 private MetricsConstants() {} 4005 4006 /** 4007 * Key to extract the codec being used 4008 * from the {@link MediaCodec#getMetrics} return value. 4009 * The value is a String. 4010 */ 4011 public static final String CODEC = "android.media.mediacodec.codec"; 4012 4013 /** 4014 * Key to extract the MIME type 4015 * from the {@link MediaCodec#getMetrics} return value. 4016 * The value is a String. 4017 */ 4018 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 4019 4020 /** 4021 * Key to extract what the codec mode 4022 * from the {@link MediaCodec#getMetrics} return value. 4023 * The value is a String. Values will be one of the constants 4024 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 4025 */ 4026 public static final String MODE = "android.media.mediacodec.mode"; 4027 4028 /** 4029 * The value returned for the key {@link #MODE} when the 4030 * codec is a audio codec. 4031 */ 4032 public static final String MODE_AUDIO = "audio"; 4033 4034 /** 4035 * The value returned for the key {@link #MODE} when the 4036 * codec is a video codec. 4037 */ 4038 public static final String MODE_VIDEO = "video"; 4039 4040 /** 4041 * Key to extract the flag indicating whether the codec is running 4042 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 4043 * The value is an integer. 4044 * A 0 indicates decoder; 1 indicates encoder. 4045 */ 4046 public static final String ENCODER = "android.media.mediacodec.encoder"; 4047 4048 /** 4049 * Key to extract the flag indicating whether the codec is running 4050 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 4051 * The value is an integer. 4052 */ 4053 public static final String SECURE = "android.media.mediacodec.secure"; 4054 4055 /** 4056 * Key to extract the width (in pixels) of the video track 4057 * from the {@link MediaCodec#getMetrics} return value. 4058 * The value is an integer. 4059 */ 4060 public static final String WIDTH = "android.media.mediacodec.width"; 4061 4062 /** 4063 * Key to extract the height (in pixels) of the video track 4064 * from the {@link MediaCodec#getMetrics} return value. 4065 * The value is an integer. 4066 */ 4067 public static final String HEIGHT = "android.media.mediacodec.height"; 4068 4069 /** 4070 * Key to extract the rotation (in degrees) to properly orient the video 4071 * from the {@link MediaCodec#getMetrics} return. 4072 * The value is a integer. 4073 */ 4074 public static final String ROTATION = "android.media.mediacodec.rotation"; 4075 4076 } 4077 } 4078
