1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.compat.annotation.UnsupportedAppUsage; 23 import android.graphics.ImageFormat; 24 import android.graphics.Rect; 25 import android.graphics.SurfaceTexture; 26 import android.hardware.HardwareBuffer; 27 import android.media.MediaCodecInfo.CodecCapabilities; 28 import android.os.Build; 29 import android.os.Bundle; 30 import android.os.Handler; 31 import android.os.IHwBinder; 32 import android.os.Looper; 33 import android.os.Message; 34 import android.os.PersistableBundle; 35 import android.view.Surface; 36 37 import java.io.IOException; 38 import java.lang.annotation.Retention; 39 import java.lang.annotation.RetentionPolicy; 40 import java.nio.ByteBuffer; 41 import java.nio.ByteOrder; 42 import java.nio.ReadOnlyBufferException; 43 import java.util.ArrayList; 44 import java.util.Arrays; 45 import java.util.Collections; 46 import java.util.HashMap; 47 import java.util.HashSet; 48 import java.util.List; 49 import java.util.Map; 50 import java.util.Objects; 51 import java.util.Set; 52 import java.util.concurrent.BlockingQueue; 53 import java.util.concurrent.LinkedBlockingQueue; 54 import java.util.concurrent.locks.Lock; 55 import java.util.concurrent.locks.ReentrantLock; 56 57 /** 58 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 59 It is part of the Android low-level multimedia support infrastructure (normally used together 60 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 61 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 62 <p> 63 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 64 data="../../../images/media/mediacodec_buffers.svg"><img 65 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 66 alt="MediaCodec buffer flow diagram"></object></center> 67 <p> 68 In broad terms, a codec processes input data to generate output data. It processes data 69 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 70 (or receive) an empty input buffer, fill it up with data and send it to the codec for 71 processing. The codec uses up the data and transforms it into one of its empty output buffers. 72 Finally, you request (or receive) a filled output buffer, consume its contents and release it 73 back to the codec. 74 75 <h3 id=qualityFloor><a name="qualityFloor">Minimum Quality Floor for Video Encoding</h3> 76 <p> 77 Beginning with {@link android.os.Build.VERSION_CODES#S}, Android's Video MediaCodecs enforce a 78 minimum quality floor. The intent is to eliminate poor quality video encodings. This quality 79 floor is applied when the codec is in Variable Bitrate (VBR) mode; it is not applied when 80 the codec is in Constant Bitrate (CBR) mode. The quality floor enforcement is also restricted 81 to a particular size range; this size range is currently for video resolutions 82 larger than 320x240 up through 1920x1080. 83 84 <p> 85 When this quality floor is in effect, the codec and supporting framework code will work to 86 ensure that the generated video is of at least a "fair" or "good" quality. The metric 87 used to choose these targets is the VMAF (Video Multi-method Assessment Function) with a 88 target score of 70 for selected test sequences. 89 90 <p> 91 The typical effect is that 92 some videos will generate a higher bitrate than originally configured. This will be most 93 notable for videos which were configured with very low bitrates; the codec will use a bitrate 94 that is determined to be more likely to generate an "fair" or "good" quality video. Another 95 situation is where a video includes very complicated content (lots of motion and detail); 96 in such configurations, the codec will use extra bitrate as needed to avoid losing all of 97 the content's finer detail. 98 99 <p> 100 This quality floor will not impact content captured at high bitrates (a high bitrate should 101 already provide the codec with sufficient capacity to encode all of the detail). 102 The quality floor does not operate on CBR encodings. 103 The quality floor currently does not operate on resolutions of 320x240 or lower, nor on 104 videos with resolution above 1920x1080. 105 106 <h3>Data Types</h3> 107 <p> 108 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 109 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 110 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 111 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 112 You normally cannot access the raw video data when using a Surface, but you can use the 113 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 114 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 115 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 116 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 117 OutputImage(int)}. 118 119 <h4>Compressed Buffers</h4> 120 <p> 121 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 122 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 123 compressed video frame. For audio data this is normally a single access unit (an encoded audio 124 segment typically containing a few milliseconds of audio as dictated by the format type), but 125 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 126 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 127 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 128 129 <h4>Raw Audio Buffers</h4> 130 <p> 131 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 132 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 133 in native byte order. 134 Raw audio buffers in the float PCM encoding are only possible 135 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 136 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 137 {@link #configure configure(…)} 138 and confirmed by {@link #getOutputFormat} for decoders 139 or {@link #getInputFormat} for encoders. 140 A sample method to check for float PCM in the MediaFormat is as follows: 141 142 <pre class=prettyprint> 143 static boolean isPcmFloat(MediaFormat format) { 144 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 145 == AudioFormat.ENCODING_PCM_FLOAT; 146 }</pre> 147 148 In order to extract, in a short array, 149 one channel of a buffer containing 16 bit signed integer audio data, 150 the following code may be used: 151 152 <pre class=prettyprint> 153 // Assumes the buffer PCM encoding is 16 bit. 154 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 155 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 156 MediaFormat format = codec.getOutputFormat(bufferId); 157 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 158 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 159 if (channelIx < 0 || channelIx >= numChannels) { 160 return null; 161 } 162 short[] res = new short[samples.remaining() / numChannels]; 163 for (int i = 0; i < res.length; ++i) { 164 res[i] = samples.get(i * numChannels + channelIx); 165 } 166 return res; 167 }</pre> 168 169 <h4>Raw Video Buffers</h4> 170 <p> 171 In ByteBuffer mode video buffers are laid out according to their {@linkplain 172 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 173 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 174 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 175 Video codecs may support three kinds of color formats: 176 <ul> 177 <li><strong>native raw video format:</strong> This is marked by {@link 178 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 179 <li><strong>flexible YUV buffers</strong> (such as {@link 180 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 181 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 182 OutputImage(int)}.</li> 183 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 184 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 185 For color formats that are equivalent to a flexible format, you can still use {@link 186 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 187 </ul> 188 <p> 189 All video codecs support flexible YUV 4:2:0 buffers since {@link 190 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 191 192 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 193 <p> 194 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 195 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 196 values to understand the layout of the raw output buffers. 197 <p class=note> 198 Note that on some devices the slice-height is advertised as 0. This could mean either that the 199 slice-height is the same as the frame height, or that the slice-height is the frame height 200 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 201 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 202 plane in planar formats is also not specified or defined, though usually it is half of the slice 203 height. 204 <p> 205 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 206 video frames; however, for most encondings the video (picture) only occupies a portion of the 207 video frame. This is represented by the 'crop rectangle'. 208 <p> 209 You need to use the following keys to get the crop rectangle of raw output images from the 210 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 211 entire video frame.The crop rectangle is understood in the context of the output frame 212 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 213 <table style="width: 0%"> 214 <thead> 215 <tr> 216 <th>Format Key</th> 217 <th>Type</th> 218 <th>Description</th> 219 </tr> 220 </thead> 221 <tbody> 222 <tr> 223 <td>{@code "crop-left"}</td> 224 <td>Integer</td> 225 <td>The left-coordinate (x) of the crop rectangle</td> 226 </tr><tr> 227 <td>{@code "crop-top"}</td> 228 <td>Integer</td> 229 <td>The top-coordinate (y) of the crop rectangle</td> 230 </tr><tr> 231 <td>{@code "crop-right"}</td> 232 <td>Integer</td> 233 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 234 </tr><tr> 235 <td>{@code "crop-bottom"}</td> 236 <td>Integer</td> 237 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 238 </tr><tr> 239 <td colspan=3> 240 The right and bottom coordinates can be understood as the coordinates of the right-most 241 valid column/bottom-most valid row of the cropped output image. 242 </td> 243 </tr> 244 </tbody> 245 </table> 246 <p> 247 The size of the video frame (before rotation) can be calculated as such: 248 <pre class=prettyprint> 249 MediaFormat format = decoder.getOutputFormat(…); 250 int width = format.getInteger(MediaFormat.KEY_WIDTH); 251 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 252 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 253 } 254 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 255 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 256 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 257 } 258 </pre> 259 <p class=note> 260 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 261 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 262 most devices it pointed to the top-left pixel of the entire frame. 263 264 <h3>States</h3> 265 <p> 266 During its life a codec conceptually exists in one of three states: Stopped, Executing or 267 Released. The Stopped collective state is actually the conglomeration of three states: 268 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 269 three sub-states: Flushed, Running and End-of-Stream. 270 <p> 271 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 272 data="../../../images/media/mediacodec_states.svg"><img 273 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 274 alt="MediaCodec state diagram"></object></center> 275 <p> 276 When you create a codec using one of the factory methods, the codec is in the Uninitialized 277 state. First, you need to configure it via {@link #configure configure(…)}, which brings 278 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 279 state you can process data through the buffer queue manipulation described above. 280 <p> 281 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 282 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 283 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 284 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 285 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 286 codec no longer accepts further input buffers, but still generates output buffers until the 287 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 288 while in the Executing state using {@link #flush}. 289 <p> 290 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 291 again. When you are done using a codec, you must release it by calling {@link #release}. 292 <p> 293 On rare occasions the codec may encounter an error and move to the Error state. This is 294 communicated using an invalid return value from a queuing operation, or sometimes via an 295 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 296 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 297 terminal Released state. 298 299 <h3>Creation</h3> 300 <p> 301 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 302 decoding a file or a stream, you can get the desired format from {@link 303 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 304 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 305 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 306 name of a codec that can handle that specific media format. Finally, create the codec using 307 {@link #createByCodecName}. 308 <p class=note> 309 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 310 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 311 MediaFormat#KEY_FRAME_RATE frame rate}. Use 312 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 313 to clear any existing frame rate setting in the format. 314 <p> 315 You can also create the preferred codec for a specific MIME type using {@link 316 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 317 This, however, cannot be used to inject features, and may create a codec that cannot handle the 318 specific desired media format. 319 320 <h4>Creating secure decoders</h4> 321 <p> 322 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 323 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 324 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 325 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 326 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 327 <p> 328 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 329 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 330 331 <h3>Initialization</h3> 332 <p> 333 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 334 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 335 specific media format. This is when you can specify the output {@link Surface} for video 336 producers – codecs that generate raw video data (e.g. video decoders). This is also when 337 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 338 some codecs can operate in multiple modes, you must specify whether you want it to work as a 339 decoder or an encoder. 340 <p> 341 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 342 output format in the Configured state. You can use this to verify the resulting configuration, 343 e.g. color formats, before starting the codec. 344 <p> 345 If you want to process raw input video buffers natively with a video consumer – a codec 346 that processes raw video input, such as a video encoder – create a destination Surface for 347 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 348 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 349 surface} by calling {@link #setInputSurface}. 350 351 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 352 <p> 353 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 354 to be prefixed by a number of buffers containing setup data, or codec specific data. When 355 processing such compressed formats, this data must be submitted to the codec after {@link 356 #start} and before any frame data. Such data must be marked using the flag {@link 357 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 358 <p> 359 Codec-specific data can also be included in the format passed to {@link #configure configure} in 360 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 361 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 362 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 363 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 364 specific data, you can choose to submit it using the specified number of buffers in the correct 365 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 366 codec-specific data and submit it as a single codec-config buffer. 367 <p> 368 Android uses the following codec-specific data buffers. These are also required to be set in 369 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 370 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 371 {@code "\x00\x00\x00\x01"}. 372 <p> 373 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 374 <table> 375 <thead> 376 <th>Format</th> 377 <th>CSD buffer #0</th> 378 <th>CSD buffer #1</th> 379 <th>CSD buffer #2</th> 380 </thead> 381 <tbody class=mid> 382 <tr> 383 <td>AAC</td> 384 <td>Decoder-specific information from ESDS<sup>*</sup></td> 385 <td class=NA>Not Used</td> 386 <td class=NA>Not Used</td> 387 </tr> 388 <tr> 389 <td>VORBIS</td> 390 <td>Identification header</td> 391 <td>Setup header</td> 392 <td class=NA>Not Used</td> 393 </tr> 394 <tr> 395 <td>OPUS</td> 396 <td>Identification header</td> 397 <td>Pre-skip in nanosecs<br> 398 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 399 This overrides the pre-skip value in the identification header.</td> 400 <td>Seek Pre-roll in nanosecs<br> 401 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 402 </tr> 403 <tr> 404 <td>FLAC</td> 405 <td>"fLaC", the FLAC stream marker in ASCII,<br> 406 followed by the STREAMINFO block (the mandatory metadata block),<br> 407 optionally followed by any number of other metadata blocks</td> 408 <td class=NA>Not Used</td> 409 <td class=NA>Not Used</td> 410 </tr> 411 <tr> 412 <td>MPEG-4</td> 413 <td>Decoder-specific information from ESDS<sup>*</sup></td> 414 <td class=NA>Not Used</td> 415 <td class=NA>Not Used</td> 416 </tr> 417 <tr> 418 <td>H.264 AVC</td> 419 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 420 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 421 <td class=NA>Not Used</td> 422 </tr> 423 <tr> 424 <td>H.265 HEVC</td> 425 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 426 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 427 PPS (Picture Parameter Sets<sup>*</sup>)</td> 428 <td class=NA>Not Used</td> 429 <td class=NA>Not Used</td> 430 </tr> 431 <tr> 432 <td>VP9</td> 433 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 434 (optional)</td> 435 <td class=NA>Not Used</td> 436 <td class=NA>Not Used</td> 437 </tr> 438 </tbody> 439 </table> 440 441 <p class=note> 442 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 443 after start, before any output buffer or output format change has been returned, as the codec 444 specific data may be lost during the flush. You must resubmit the data using buffers marked with 445 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 446 <p> 447 Encoders (or codecs that generate compressed data) will create and return the codec specific data 448 before any valid output buffer in output buffers marked with the {@linkplain 449 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 450 meaningful timestamps. 451 452 <h3>Data Processing</h3> 453 <p> 454 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 455 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 456 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 457 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 458 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 459 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 460 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 461 <p> 462 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 463 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 464 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 465 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 466 <p> 467 The codec in turn will return a read-only output buffer via the {@link 468 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 469 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 470 output buffer has been processed, call one of the {@link #releaseOutputBuffer 471 releaseOutputBuffer} methods to return the buffer to the codec. 472 <p> 473 While you are not required to resubmit/release buffers immediately to the codec, holding onto 474 input and/or output buffers may stall the codec, and this behavior is device dependent. 475 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 476 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 477 hold onto to available buffers as little as possible. 478 <p> 479 Depending on the API version, you can process data in three ways: 480 <table> 481 <thead> 482 <tr> 483 <th>Processing Mode</th> 484 <th>API version <= 20<br>Jelly Bean/KitKat</th> 485 <th>API version >= 21<br>Lollipop and later</th> 486 </tr> 487 </thead> 488 <tbody> 489 <tr> 490 <td>Synchronous API using buffer arrays</td> 491 <td>Supported</td> 492 <td>Deprecated</td> 493 </tr> 494 <tr> 495 <td>Synchronous API using buffers</td> 496 <td class=NA>Not Available</td> 497 <td>Supported</td> 498 </tr> 499 <tr> 500 <td>Asynchronous API using buffers</td> 501 <td class=NA>Not Available</td> 502 <td>Supported</td> 503 </tr> 504 </tbody> 505 </table> 506 507 <h4>Asynchronous Processing using Buffers</h4> 508 <p> 509 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 510 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 511 mode changes the state transitions slightly, because you must call {@link #start} after {@link 512 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 513 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 514 sub-state and start passing available input buffers via the callback. 515 <p> 516 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 517 data="../../../images/media/mediacodec_async_states.svg"><img 518 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 519 alt="MediaCodec state diagram for asynchronous operation"></object></center> 520 <p> 521 MediaCodec is typically used like this in asynchronous mode: 522 <pre class=prettyprint> 523 MediaCodec codec = MediaCodec.createByCodecName(name); 524 MediaFormat mOutputFormat; // member variable 525 codec.setCallback(new MediaCodec.Callback() { 526 {@literal @Override} 527 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 528 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 529 // fill inputBuffer with valid data 530 … 531 codec.queueInputBuffer(inputBufferId, …); 532 } 533 534 {@literal @Override} 535 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 536 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 537 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 538 // bufferFormat is equivalent to mOutputFormat 539 // outputBuffer is ready to be processed or rendered. 540 … 541 codec.releaseOutputBuffer(outputBufferId, …); 542 } 543 544 {@literal @Override} 545 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 546 // Subsequent data will conform to new format. 547 // Can ignore if using getOutputFormat(outputBufferId) 548 mOutputFormat = format; // option B 549 } 550 551 {@literal @Override} 552 void onError(…) { 553 … 554 } 555 }); 556 codec.configure(format, …); 557 mOutputFormat = codec.getOutputFormat(); // option B 558 codec.start(); 559 // wait for processing to complete 560 codec.stop(); 561 codec.release();</pre> 562 563 <h4>Synchronous Processing using Buffers</h4> 564 <p> 565 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 566 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 567 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 568 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 569 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 570 getInput}/{@link #getOutputBuffers OutputBuffers()}. 571 572 <p class=note> 573 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 574 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 575 #start} or after having dequeued an output buffer ID with the value of {@link 576 #INFO_OUTPUT_FORMAT_CHANGED}. 577 <p> 578 MediaCodec is typically used like this in synchronous mode: 579 <pre> 580 MediaCodec codec = MediaCodec.createByCodecName(name); 581 codec.configure(format, …); 582 MediaFormat outputFormat = codec.getOutputFormat(); // option B 583 codec.start(); 584 for (;;) { 585 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 586 if (inputBufferId >= 0) { 587 ByteBuffer inputBuffer = codec.getInputBuffer(…); 588 // fill inputBuffer with valid data 589 … 590 codec.queueInputBuffer(inputBufferId, …); 591 } 592 int outputBufferId = codec.dequeueOutputBuffer(…); 593 if (outputBufferId >= 0) { 594 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 595 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 596 // bufferFormat is identical to outputFormat 597 // outputBuffer is ready to be processed or rendered. 598 … 599 codec.releaseOutputBuffer(outputBufferId, …); 600 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 601 // Subsequent data will conform to new format. 602 // Can ignore if using getOutputFormat(outputBufferId) 603 outputFormat = codec.getOutputFormat(); // option B 604 } 605 } 606 codec.stop(); 607 codec.release();</pre> 608 609 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 610 <p> 611 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 612 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 613 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 614 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 615 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 616 between the size of the arrays and the number of input and output buffers used by the system, 617 although the array size provides an upper bound. 618 <pre> 619 MediaCodec codec = MediaCodec.createByCodecName(name); 620 codec.configure(format, …); 621 codec.start(); 622 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 623 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 624 for (;;) { 625 int inputBufferId = codec.dequeueInputBuffer(…); 626 if (inputBufferId >= 0) { 627 // fill inputBuffers[inputBufferId] with valid data 628 … 629 codec.queueInputBuffer(inputBufferId, …); 630 } 631 int outputBufferId = codec.dequeueOutputBuffer(…); 632 if (outputBufferId >= 0) { 633 // outputBuffers[outputBufferId] is ready to be processed or rendered. 634 … 635 codec.releaseOutputBuffer(outputBufferId, …); 636 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 637 outputBuffers = codec.getOutputBuffers(); 638 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 639 // Subsequent data will conform to new format. 640 MediaFormat format = codec.getOutputFormat(); 641 } 642 } 643 codec.stop(); 644 codec.release();</pre> 645 646 <h4>End-of-stream Handling</h4> 647 <p> 648 When you reach the end of the input data, you must signal it to the codec by specifying the 649 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 650 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 651 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 652 be ignored. 653 <p> 654 The codec will continue to return output buffers until it eventually signals the end of the 655 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 656 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 657 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 658 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 659 <p> 660 Do not submit additional input buffers after signaling the end of the input stream, unless the 661 codec has been flushed, or stopped and restarted. 662 663 <h4>Using an Output Surface</h4> 664 <p> 665 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 666 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 667 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 668 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 669 null}-s. 670 <p> 671 When using an output Surface, you can select whether or not to render each output buffer on the 672 surface. You have three choices: 673 <ul> 674 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 675 releaseOutputBuffer(bufferId, false)}.</li> 676 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 677 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 678 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 679 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 680 </ul> 681 <p> 682 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 683 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 684 It was not defined prior to that. 685 <p> 686 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 687 dynamically using {@link #setOutputSurface setOutputSurface}. 688 <p> 689 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 690 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 691 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 692 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 693 was undefined, with the exception that View surfaces (SurfaceView or TextureView) always dropped 694 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 695 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 696 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 697 setting the key {@link MediaFormat#KEY_ALLOW_FRAME_DROP} to {@code 0} 698 in their configure format. 699 700 <h4>Transformations When Rendering onto Surface</h4> 701 702 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 703 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 704 mode} will be automatically applied with one exception: 705 <p class=note> 706 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 707 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 708 and simple way to identify software decoders, or if they apply the rotation other than by trying 709 it out. 710 <p> 711 There are also some caveats. 712 <p class=note> 713 Note that the pixel aspect ratio is not considered when displaying the output onto the 714 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 715 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 716 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 717 square pixels (pixel aspect ratio or 1:1). 718 <p class=note> 719 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 720 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 721 by 90 or 270 degrees. 722 <p class=note> 723 When setting the video scaling mode, note that it must be reset after each time the output 724 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 725 do this after each time the output format changes. 726 727 <h4>Using an Input Surface</h4> 728 <p> 729 When using an input Surface, there are no accessible input buffers, as buffers are automatically 730 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 731 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 732 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 733 <p> 734 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 735 submitting data to the codec immediately after this call. 736 <p> 737 738 <h3>Seeking & Adaptive Playback Support</h3> 739 <p> 740 Video decoders (and in general codecs that consume compressed video data) behave differently 741 regarding seek and format change whether or not they support and are configured for adaptive 742 playback. You can check if a decoder supports {@linkplain 743 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 744 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 745 playback support for video decoders is only activated if you configure the codec to decode onto a 746 {@link Surface}. 747 748 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 749 <p> 750 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 751 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 752 completely on its own (for most codecs this means an I-frame), and no frames that are to be 753 displayed after a key frame refer to frames before the key frame. 754 <p> 755 The following table summarizes suitable key frames for various video formats. 756 <table> 757 <thead> 758 <tr> 759 <th>Format</th> 760 <th>Suitable key frame</th> 761 </tr> 762 </thead> 763 <tbody class=mid> 764 <tr> 765 <td>VP9/VP8</td> 766 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 767 <i>(There is no specific name for such key frame.)</i></td> 768 </tr> 769 <tr> 770 <td>H.265 HEVC</td> 771 <td>IDR or CRA</td> 772 </tr> 773 <tr> 774 <td>H.264 AVC</td> 775 <td>IDR</td> 776 </tr> 777 <tr> 778 <td>MPEG-4<br>H.263<br>MPEG-2</td> 779 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 780 <i>(There is no specific name for such key frame.)</td> 781 </tr> 782 </tbody> 783 </table> 784 785 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 786 Surface)</h4> 787 <p> 788 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 789 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 790 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 791 before you call {@code flush}. It is important that the input data after a flush starts at a 792 suitable stream boundary/key frame. 793 <p class=note> 794 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 795 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 796 #configure configure(…)} - {@link #start} cycle is necessary. 797 798 <p class=note> 799 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 800 generally, before the first output buffer or output format change is received – you 801 will need to resubmit the codec-specific-data to the codec. See the <a 802 href="#CSD">codec-specific-data section</a> for more info. 803 804 <h4>For decoders that support and are configured for adaptive playback</h4> 805 <p> 806 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 807 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 808 discontinuity must start at a suitable stream boundary/key frame. 809 <p> 810 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 811 picture size or configuration mid-stream. To do this you must package the entire new 812 codec-specific configuration data together with the key frame into a single buffer (including 813 any start codes), and submit it as a <strong>regular</strong> input buffer. 814 <p> 815 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 816 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 817 onOutputFormatChanged} callback just after the picture-size change takes place and before any 818 frames with the new size have been returned. 819 <p class=note> 820 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 821 {@link #flush} shortly after you have changed the picture size. If you have not received 822 confirmation of the picture size change, you will need to repeat the request for the new picture 823 size. 824 825 <h3>Error handling</h3> 826 <p> 827 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 828 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 829 which you must catch or declare to pass up. MediaCodec methods throw {@code 830 IllegalStateException} when the method is called from a codec state that does not allow it; this 831 is typically due to incorrect application API usage. Methods involving secure buffers may throw 832 {@link CryptoException}, which has further error information obtainable from {@link 833 CryptoException#getErrorCode}. 834 <p> 835 Internal codec errors result in a {@link CodecException}, which may be due to media content 836 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 837 correctly using the API. The recommended action when receiving a {@code CodecException} 838 can be determined by calling {@link CodecException#isRecoverable} and {@link 839 CodecException#isTransient}: 840 <ul> 841 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 842 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 843 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 844 temporarily unavailable and the method may be retried at a later time.</li> 845 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 846 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 847 reset} or {@linkplain #release released}.</li> 848 </ul> 849 <p> 850 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 851 852 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 853 <p> 854 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 855 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 856 857 <style> 858 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 859 .api > tr > th { vertical-align: bottom; } 860 .api > tr > td { vertical-align: middle; } 861 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 862 .fn { text-align: left; } 863 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 864 .deg45 { 865 white-space: nowrap; background: none; border: none; vertical-align: bottom; 866 width: 30px; height: 83px; 867 } 868 .deg45 > div { 869 transform: skew(-45deg, 0deg) translate(1px, -67px); 870 transform-origin: bottom left 0; 871 width: 30px; height: 20px; 872 } 873 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 874 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 875 </style> 876 877 <table align="right" style="width: 0%"> 878 <thead> 879 <tr><th>Symbol</th><th>Meaning</th></tr> 880 </thead> 881 <tbody class=sml> 882 <tr><td>●</td><td>Supported</td></tr> 883 <tr><td>⁕</td><td>Semantics changed</td></tr> 884 <tr><td>○</td><td>Experimental support</td></tr> 885 <tr><td>[ ]</td><td>Deprecated</td></tr> 886 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 887 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 888 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 889 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 890 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 891 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 892 </tbody> 893 </table> 894 895 <table style="width: 100%;"> 896 <thead class=api> 897 <tr> 898 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 899 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 900 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 901 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 902 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 903 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 904 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 905 <th></th> 906 <th colspan="8">SDK Version</th> 907 </tr> 908 <tr> 909 <th colspan="7">State</th> 910 <th>Method</th> 911 <th>16</th> 912 <th>17</th> 913 <th>18</th> 914 <th>19</th> 915 <th>20</th> 916 <th>21</th> 917 <th>22</th> 918 <th>23</th> 919 </tr> 920 </thead> 921 <tbody class=api> 922 <tr> 923 <td></td> 924 <td></td> 925 <td></td> 926 <td></td> 927 <td></td> 928 <td></td> 929 <td></td> 930 <td class=fn>{@link #createByCodecName createByCodecName}</td> 931 <td>●</td> 932 <td>●</td> 933 <td>●</td> 934 <td>●</td> 935 <td>●</td> 936 <td>●</td> 937 <td>●</td> 938 <td>●</td> 939 </tr> 940 <tr> 941 <td></td> 942 <td></td> 943 <td></td> 944 <td></td> 945 <td></td> 946 <td></td> 947 <td></td> 948 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 949 <td>●</td> 950 <td>●</td> 951 <td>●</td> 952 <td>●</td> 953 <td>●</td> 954 <td>●</td> 955 <td>●</td> 956 <td>●</td> 957 </tr> 958 <tr> 959 <td></td> 960 <td></td> 961 <td></td> 962 <td></td> 963 <td></td> 964 <td></td> 965 <td></td> 966 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 967 <td>●</td> 968 <td>●</td> 969 <td>●</td> 970 <td>●</td> 971 <td>●</td> 972 <td>●</td> 973 <td>●</td> 974 <td>●</td> 975 </tr> 976 <tr> 977 <td></td> 978 <td></td> 979 <td></td> 980 <td></td> 981 <td></td> 982 <td></td> 983 <td></td> 984 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 985 <td></td> 986 <td></td> 987 <td></td> 988 <td></td> 989 <td></td> 990 <td></td> 991 <td></td> 992 <td>●</td> 993 </tr> 994 <tr> 995 <td>16+</td> 996 <td>-</td> 997 <td>-</td> 998 <td>-</td> 999 <td>-</td> 1000 <td>-</td> 1001 <td>-</td> 1002 <td class=fn>{@link #configure configure}</td> 1003 <td>●</td> 1004 <td>●</td> 1005 <td>●</td> 1006 <td>●</td> 1007 <td>●</td> 1008 <td>⁕</td> 1009 <td>●</td> 1010 <td>●</td> 1011 </tr> 1012 <tr> 1013 <td>-</td> 1014 <td>18+</td> 1015 <td>-</td> 1016 <td>-</td> 1017 <td>-</td> 1018 <td>-</td> 1019 <td>-</td> 1020 <td class=fn>{@link #createInputSurface createInputSurface}</td> 1021 <td></td> 1022 <td></td> 1023 <td>⎋</td> 1024 <td>⎋</td> 1025 <td>⎋</td> 1026 <td>⎋</td> 1027 <td>⎋</td> 1028 <td>⎋</td> 1029 </tr> 1030 <tr> 1031 <td>-</td> 1032 <td>-</td> 1033 <td>16+</td> 1034 <td>16+</td> 1035 <td>(16+)</td> 1036 <td>-</td> 1037 <td>-</td> 1038 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 1039 <td>●</td> 1040 <td>●</td> 1041 <td>▧</td> 1042 <td>▧</td> 1043 <td>▧</td> 1044 <td>⁕▧↩</td> 1045 <td>▧↩</td> 1046 <td>▧↩</td> 1047 </tr> 1048 <tr> 1049 <td>-</td> 1050 <td>-</td> 1051 <td>16+</td> 1052 <td>16+</td> 1053 <td>16+</td> 1054 <td>-</td> 1055 <td>-</td> 1056 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1057 <td>●</td> 1058 <td>●</td> 1059 <td>●</td> 1060 <td>●</td> 1061 <td>●</td> 1062 <td>⁕↩</td> 1063 <td>↩</td> 1064 <td>↩</td> 1065 </tr> 1066 <tr> 1067 <td>-</td> 1068 <td>-</td> 1069 <td>16+</td> 1070 <td>16+</td> 1071 <td>16+</td> 1072 <td>-</td> 1073 <td>-</td> 1074 <td class=fn>{@link #flush flush}</td> 1075 <td>●</td> 1076 <td>●</td> 1077 <td>●</td> 1078 <td>●</td> 1079 <td>●</td> 1080 <td>●</td> 1081 <td>●</td> 1082 <td>●</td> 1083 </tr> 1084 <tr> 1085 <td>18+</td> 1086 <td>18+</td> 1087 <td>18+</td> 1088 <td>18+</td> 1089 <td>18+</td> 1090 <td>18+</td> 1091 <td>-</td> 1092 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1093 <td></td> 1094 <td></td> 1095 <td>●</td> 1096 <td>●</td> 1097 <td>●</td> 1098 <td>●</td> 1099 <td>●</td> 1100 <td>●</td> 1101 </tr> 1102 <tr> 1103 <td>-</td> 1104 <td>-</td> 1105 <td>(21+)</td> 1106 <td>21+</td> 1107 <td>(21+)</td> 1108 <td>-</td> 1109 <td>-</td> 1110 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1111 <td></td> 1112 <td></td> 1113 <td></td> 1114 <td></td> 1115 <td></td> 1116 <td>●</td> 1117 <td>●</td> 1118 <td>●</td> 1119 </tr> 1120 <tr> 1121 <td>-</td> 1122 <td>-</td> 1123 <td>16+</td> 1124 <td>(16+)</td> 1125 <td>(16+)</td> 1126 <td>-</td> 1127 <td>-</td> 1128 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1129 <td>●</td> 1130 <td>●</td> 1131 <td>●</td> 1132 <td>●</td> 1133 <td>●</td> 1134 <td>[⁕↩]</td> 1135 <td>[↩]</td> 1136 <td>[↩]</td> 1137 </tr> 1138 <tr> 1139 <td>-</td> 1140 <td>21+</td> 1141 <td>(21+)</td> 1142 <td>(21+)</td> 1143 <td>(21+)</td> 1144 <td>-</td> 1145 <td>-</td> 1146 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1147 <td></td> 1148 <td></td> 1149 <td></td> 1150 <td></td> 1151 <td></td> 1152 <td>●</td> 1153 <td>●</td> 1154 <td>●</td> 1155 </tr> 1156 <tr> 1157 <td>-</td> 1158 <td>-</td> 1159 <td>(21+)</td> 1160 <td>21+</td> 1161 <td>(21+)</td> 1162 <td>-</td> 1163 <td>-</td> 1164 <td class=fn>{@link #getInputImage getInputImage}</td> 1165 <td></td> 1166 <td></td> 1167 <td></td> 1168 <td></td> 1169 <td></td> 1170 <td>○</td> 1171 <td>●</td> 1172 <td>●</td> 1173 </tr> 1174 <tr> 1175 <td>18+</td> 1176 <td>18+</td> 1177 <td>18+</td> 1178 <td>18+</td> 1179 <td>18+</td> 1180 <td>18+</td> 1181 <td>-</td> 1182 <td class=fn>{@link #getName getName}</td> 1183 <td></td> 1184 <td></td> 1185 <td>●</td> 1186 <td>●</td> 1187 <td>●</td> 1188 <td>●</td> 1189 <td>●</td> 1190 <td>●</td> 1191 </tr> 1192 <tr> 1193 <td>-</td> 1194 <td>-</td> 1195 <td>(21+)</td> 1196 <td>21+</td> 1197 <td>21+</td> 1198 <td>-</td> 1199 <td>-</td> 1200 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1201 <td></td> 1202 <td></td> 1203 <td></td> 1204 <td></td> 1205 <td></td> 1206 <td>●</td> 1207 <td>●</td> 1208 <td>●</td> 1209 </tr> 1210 <tr> 1211 <td>-</td> 1212 <td>-</td> 1213 <td>16+</td> 1214 <td>16+</td> 1215 <td>16+</td> 1216 <td>-</td> 1217 <td>-</td> 1218 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1219 <td>●</td> 1220 <td>●</td> 1221 <td>●</td> 1222 <td>●</td> 1223 <td>●</td> 1224 <td>[⁕↩]</td> 1225 <td>[↩]</td> 1226 <td>[↩]</td> 1227 </tr> 1228 <tr> 1229 <td>-</td> 1230 <td>21+</td> 1231 <td>16+</td> 1232 <td>16+</td> 1233 <td>16+</td> 1234 <td>-</td> 1235 <td>-</td> 1236 <td class=fn>{@link #getOutputFormat()}</td> 1237 <td>●</td> 1238 <td>●</td> 1239 <td>●</td> 1240 <td>●</td> 1241 <td>●</td> 1242 <td>●</td> 1243 <td>●</td> 1244 <td>●</td> 1245 </tr> 1246 <tr> 1247 <td>-</td> 1248 <td>-</td> 1249 <td>(21+)</td> 1250 <td>21+</td> 1251 <td>21+</td> 1252 <td>-</td> 1253 <td>-</td> 1254 <td class=fn>{@link #getOutputFormat(int)}</td> 1255 <td></td> 1256 <td></td> 1257 <td></td> 1258 <td></td> 1259 <td></td> 1260 <td>●</td> 1261 <td>●</td> 1262 <td>●</td> 1263 </tr> 1264 <tr> 1265 <td>-</td> 1266 <td>-</td> 1267 <td>(21+)</td> 1268 <td>21+</td> 1269 <td>21+</td> 1270 <td>-</td> 1271 <td>-</td> 1272 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1273 <td></td> 1274 <td></td> 1275 <td></td> 1276 <td></td> 1277 <td></td> 1278 <td>○</td> 1279 <td>●</td> 1280 <td>●</td> 1281 </tr> 1282 <tr> 1283 <td>-</td> 1284 <td>-</td> 1285 <td>-</td> 1286 <td>16+</td> 1287 <td>(16+)</td> 1288 <td>-</td> 1289 <td>-</td> 1290 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1291 <td>●</td> 1292 <td>●</td> 1293 <td>●</td> 1294 <td>●</td> 1295 <td>●</td> 1296 <td>⁕</td> 1297 <td>●</td> 1298 <td>●</td> 1299 </tr> 1300 <tr> 1301 <td>-</td> 1302 <td>-</td> 1303 <td>-</td> 1304 <td>16+</td> 1305 <td>(16+)</td> 1306 <td>-</td> 1307 <td>-</td> 1308 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1309 <td>●</td> 1310 <td>●</td> 1311 <td>●</td> 1312 <td>●</td> 1313 <td>●</td> 1314 <td>⁕</td> 1315 <td>●</td> 1316 <td>●</td> 1317 </tr> 1318 <tr> 1319 <td>16+</td> 1320 <td>16+</td> 1321 <td>16+</td> 1322 <td>16+</td> 1323 <td>16+</td> 1324 <td>16+</td> 1325 <td>16+</td> 1326 <td class=fn>{@link #release release}</td> 1327 <td>●</td> 1328 <td>●</td> 1329 <td>●</td> 1330 <td>●</td> 1331 <td>●</td> 1332 <td>●</td> 1333 <td>●</td> 1334 <td>●</td> 1335 </tr> 1336 <tr> 1337 <td>-</td> 1338 <td>-</td> 1339 <td>-</td> 1340 <td>16+</td> 1341 <td>16+</td> 1342 <td>-</td> 1343 <td>-</td> 1344 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1345 <td>●</td> 1346 <td>●</td> 1347 <td>●</td> 1348 <td>●</td> 1349 <td>●</td> 1350 <td>⁕</td> 1351 <td>●</td> 1352 <td>⁕</td> 1353 </tr> 1354 <tr> 1355 <td>-</td> 1356 <td>-</td> 1357 <td>-</td> 1358 <td>21+</td> 1359 <td>21+</td> 1360 <td>-</td> 1361 <td>-</td> 1362 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1363 <td></td> 1364 <td></td> 1365 <td></td> 1366 <td></td> 1367 <td></td> 1368 <td>⎆</td> 1369 <td>⎆</td> 1370 <td>⎆</td> 1371 </tr> 1372 <tr> 1373 <td>21+</td> 1374 <td>21+</td> 1375 <td>21+</td> 1376 <td>21+</td> 1377 <td>21+</td> 1378 <td>21+</td> 1379 <td>-</td> 1380 <td class=fn>{@link #reset reset}</td> 1381 <td></td> 1382 <td></td> 1383 <td></td> 1384 <td></td> 1385 <td></td> 1386 <td>●</td> 1387 <td>●</td> 1388 <td>●</td> 1389 </tr> 1390 <tr> 1391 <td>21+</td> 1392 <td>-</td> 1393 <td>-</td> 1394 <td>-</td> 1395 <td>-</td> 1396 <td>-</td> 1397 <td>-</td> 1398 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1399 <td></td> 1400 <td></td> 1401 <td></td> 1402 <td></td> 1403 <td></td> 1404 <td>●</td> 1405 <td>●</td> 1406 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1407 </tr> 1408 <tr> 1409 <td>-</td> 1410 <td>23+</td> 1411 <td>-</td> 1412 <td>-</td> 1413 <td>-</td> 1414 <td>-</td> 1415 <td>-</td> 1416 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1417 <td></td> 1418 <td></td> 1419 <td></td> 1420 <td></td> 1421 <td></td> 1422 <td></td> 1423 <td></td> 1424 <td>⎋</td> 1425 </tr> 1426 <tr> 1427 <td>23+</td> 1428 <td>23+</td> 1429 <td>23+</td> 1430 <td>23+</td> 1431 <td>23+</td> 1432 <td>(23+)</td> 1433 <td>(23+)</td> 1434 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1435 <td></td> 1436 <td></td> 1437 <td></td> 1438 <td></td> 1439 <td></td> 1440 <td></td> 1441 <td></td> 1442 <td>○ ⎆</td> 1443 </tr> 1444 <tr> 1445 <td>-</td> 1446 <td>23+</td> 1447 <td>23+</td> 1448 <td>23+</td> 1449 <td>23+</td> 1450 <td>-</td> 1451 <td>-</td> 1452 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1453 <td></td> 1454 <td></td> 1455 <td></td> 1456 <td></td> 1457 <td></td> 1458 <td></td> 1459 <td></td> 1460 <td>⎆</td> 1461 </tr> 1462 <tr> 1463 <td>19+</td> 1464 <td>19+</td> 1465 <td>19+</td> 1466 <td>19+</td> 1467 <td>19+</td> 1468 <td>(19+)</td> 1469 <td>-</td> 1470 <td class=fn>{@link #setParameters setParameters}</td> 1471 <td></td> 1472 <td></td> 1473 <td></td> 1474 <td>●</td> 1475 <td>●</td> 1476 <td>●</td> 1477 <td>●</td> 1478 <td>●</td> 1479 </tr> 1480 <tr> 1481 <td>-</td> 1482 <td>(16+)</td> 1483 <td>(16+)</td> 1484 <td>16+</td> 1485 <td>(16+)</td> 1486 <td>(16+)</td> 1487 <td>-</td> 1488 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1489 <td>⎆</td> 1490 <td>⎆</td> 1491 <td>⎆</td> 1492 <td>⎆</td> 1493 <td>⎆</td> 1494 <td>⎆</td> 1495 <td>⎆</td> 1496 <td>⎆</td> 1497 </tr> 1498 <tr> 1499 <td>(29+)</td> 1500 <td>29+</td> 1501 <td>29+</td> 1502 <td>29+</td> 1503 <td>(29+)</td> 1504 <td>(29+)</td> 1505 <td>-</td> 1506 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1507 <td></td> 1508 <td></td> 1509 <td></td> 1510 <td></td> 1511 <td></td> 1512 <td></td> 1513 <td></td> 1514 <td></td> 1515 </tr> 1516 <tr> 1517 <td>-</td> 1518 <td>-</td> 1519 <td>18+</td> 1520 <td>18+</td> 1521 <td>-</td> 1522 <td>-</td> 1523 <td>-</td> 1524 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1525 <td></td> 1526 <td></td> 1527 <td>⎋</td> 1528 <td>⎋</td> 1529 <td>⎋</td> 1530 <td>⎋</td> 1531 <td>⎋</td> 1532 <td>⎋</td> 1533 </tr> 1534 <tr> 1535 <td>-</td> 1536 <td>16+</td> 1537 <td>21+(⇄)</td> 1538 <td>-</td> 1539 <td>-</td> 1540 <td>-</td> 1541 <td>-</td> 1542 <td class=fn>{@link #start start}</td> 1543 <td>●</td> 1544 <td>●</td> 1545 <td>●</td> 1546 <td>●</td> 1547 <td>●</td> 1548 <td>⁕</td> 1549 <td>●</td> 1550 <td>●</td> 1551 </tr> 1552 <tr> 1553 <td>-</td> 1554 <td>-</td> 1555 <td>16+</td> 1556 <td>16+</td> 1557 <td>16+</td> 1558 <td>-</td> 1559 <td>-</td> 1560 <td class=fn>{@link #stop stop}</td> 1561 <td>●</td> 1562 <td>●</td> 1563 <td>●</td> 1564 <td>●</td> 1565 <td>●</td> 1566 <td>●</td> 1567 <td>●</td> 1568 <td>●</td> 1569 </tr> 1570 </tbody> 1571 </table> 1572 */ 1573 final public class MediaCodec { 1574 1575 /** 1576 * Per buffer metadata includes an offset and size specifying 1577 * the range of valid data in the associated codec (output) buffer. 1578 */ 1579 public final static class BufferInfo { 1580 /** 1581 * Update the buffer metadata information. 1582 * 1583 * @param newOffset the start-offset of the data in the buffer. 1584 * @param newSize the amount of data (in bytes) in the buffer. 1585 * @param newTimeUs the presentation timestamp in microseconds. 1586 * @param newFlags buffer flags associated with the buffer. This 1587 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1588 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1589 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1590 public void set( 1591 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1592 offset = newOffset; 1593 size = newSize; 1594 presentationTimeUs = newTimeUs; 1595 flags = newFlags; 1596 } 1597 1598 /** 1599 * The start-offset of the data in the buffer. 1600 */ 1601 public int offset; 1602 1603 /** 1604 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1605 * the buffer has no data in it and can be discarded. The only 1606 * use of a 0-size buffer is to carry the end-of-stream marker. 1607 */ 1608 public int size; 1609 1610 /** 1611 * The presentation timestamp in microseconds for the buffer. 1612 * This is derived from the presentation timestamp passed in 1613 * with the corresponding input buffer. This should be ignored for 1614 * a 0-sized buffer. 1615 */ 1616 public long presentationTimeUs; 1617 1618 /** 1619 * Buffer flags associated with the buffer. A combination of 1620 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1621 * 1622 * <p>Encoded buffers that are key frames are marked with 1623 * {@link #BUFFER_FLAG_KEY_FRAME}. 1624 * 1625 * <p>The last output buffer corresponding to the input buffer 1626 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1627 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1628 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1629 * marker. 1630 */ 1631 @BufferFlag 1632 public int flags; 1633 1634 /** @hide */ 1635 @NonNull dup()1636 public BufferInfo dup() { 1637 BufferInfo copy = new BufferInfo(); 1638 copy.set(offset, size, presentationTimeUs, flags); 1639 return copy; 1640 } 1641 }; 1642 1643 // The follow flag constants MUST stay in sync with their equivalents 1644 // in MediaCodec.h ! 1645 1646 /** 1647 * This indicates that the (encoded) buffer marked as such contains 1648 * the data for a key frame. 1649 * 1650 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1651 */ 1652 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1653 1654 /** 1655 * This indicates that the (encoded) buffer marked as such contains 1656 * the data for a key frame. 1657 */ 1658 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1659 1660 /** 1661 * This indicated that the buffer marked as such contains codec 1662 * initialization / codec specific data instead of media data. 1663 */ 1664 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1665 1666 /** 1667 * This signals the end of stream, i.e. no buffers will be available 1668 * after this, unless of course, {@link #flush} follows. 1669 */ 1670 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1671 1672 /** 1673 * This indicates that the buffer only contains part of a frame, 1674 * and the decoder should batch the data until a buffer without 1675 * this flag appears before decoding the frame. 1676 */ 1677 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1678 1679 /** 1680 * This indicates that the buffer contains non-media data for the 1681 * muxer to process. 1682 * 1683 * All muxer data should start with a FOURCC header that determines the type of data. 1684 * 1685 * For example, when it contains Exif data sent to a MediaMuxer track of 1686 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1687 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1688 * 1689 * @hide 1690 */ 1691 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1692 1693 /** @hide */ 1694 @IntDef( 1695 flag = true, 1696 value = { 1697 BUFFER_FLAG_SYNC_FRAME, 1698 BUFFER_FLAG_KEY_FRAME, 1699 BUFFER_FLAG_CODEC_CONFIG, 1700 BUFFER_FLAG_END_OF_STREAM, 1701 BUFFER_FLAG_PARTIAL_FRAME, 1702 BUFFER_FLAG_MUXER_DATA, 1703 }) 1704 @Retention(RetentionPolicy.SOURCE) 1705 public @interface BufferFlag {} 1706 1707 private EventHandler mEventHandler; 1708 private EventHandler mOnFirstTunnelFrameReadyHandler; 1709 private EventHandler mOnFrameRenderedHandler; 1710 private EventHandler mCallbackHandler; 1711 private Callback mCallback; 1712 private OnFirstTunnelFrameReadyListener mOnFirstTunnelFrameReadyListener; 1713 private OnFrameRenderedListener mOnFrameRenderedListener; 1714 private final Object mListenerLock = new Object(); 1715 private MediaCodecInfo mCodecInfo; 1716 private final Object mCodecInfoLock = new Object(); 1717 private MediaCrypto mCrypto; 1718 1719 private static final int EVENT_CALLBACK = 1; 1720 private static final int EVENT_SET_CALLBACK = 2; 1721 private static final int EVENT_FRAME_RENDERED = 3; 1722 private static final int EVENT_FIRST_TUNNEL_FRAME_READY = 4; 1723 1724 private static final int CB_INPUT_AVAILABLE = 1; 1725 private static final int CB_OUTPUT_AVAILABLE = 2; 1726 private static final int CB_ERROR = 3; 1727 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1728 1729 1730 private class EventHandler extends Handler { 1731 private MediaCodec mCodec; 1732 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1733 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1734 super(looper); 1735 mCodec = codec; 1736 } 1737 1738 @Override handleMessage(@onNull Message msg)1739 public void handleMessage(@NonNull Message msg) { 1740 switch (msg.what) { 1741 case EVENT_CALLBACK: 1742 { 1743 handleCallback(msg); 1744 break; 1745 } 1746 case EVENT_SET_CALLBACK: 1747 { 1748 mCallback = (MediaCodec.Callback) msg.obj; 1749 break; 1750 } 1751 case EVENT_FRAME_RENDERED: 1752 Map<String, Object> map = (Map<String, Object>)msg.obj; 1753 for (int i = 0; ; ++i) { 1754 Object mediaTimeUs = map.get(i + "-media-time-us"); 1755 Object systemNano = map.get(i + "-system-nano"); 1756 OnFrameRenderedListener onFrameRenderedListener; 1757 synchronized (mListenerLock) { 1758 onFrameRenderedListener = mOnFrameRenderedListener; 1759 } 1760 if (mediaTimeUs == null || systemNano == null 1761 || onFrameRenderedListener == null) { 1762 break; 1763 } 1764 onFrameRenderedListener.onFrameRendered( 1765 mCodec, (long)mediaTimeUs, (long)systemNano); 1766 } 1767 break; 1768 case EVENT_FIRST_TUNNEL_FRAME_READY: 1769 OnFirstTunnelFrameReadyListener onFirstTunnelFrameReadyListener; 1770 synchronized (mListenerLock) { 1771 onFirstTunnelFrameReadyListener = mOnFirstTunnelFrameReadyListener; 1772 } 1773 if (onFirstTunnelFrameReadyListener == null) { 1774 break; 1775 } 1776 onFirstTunnelFrameReadyListener.onFirstTunnelFrameReady(mCodec); 1777 break; 1778 default: 1779 { 1780 break; 1781 } 1782 } 1783 } 1784 handleCallback(@onNull Message msg)1785 private void handleCallback(@NonNull Message msg) { 1786 if (mCallback == null) { 1787 return; 1788 } 1789 1790 switch (msg.arg1) { 1791 case CB_INPUT_AVAILABLE: 1792 { 1793 int index = msg.arg2; 1794 synchronized(mBufferLock) { 1795 switch (mBufferMode) { 1796 case BUFFER_MODE_LEGACY: 1797 validateInputByteBuffer(mCachedInputBuffers, index); 1798 break; 1799 case BUFFER_MODE_BLOCK: 1800 while (mQueueRequests.size() <= index) { 1801 mQueueRequests.add(null); 1802 } 1803 QueueRequest request = mQueueRequests.get(index); 1804 if (request == null) { 1805 request = new QueueRequest(mCodec, index); 1806 mQueueRequests.set(index, request); 1807 } 1808 request.setAccessible(true); 1809 break; 1810 default: 1811 throw new IllegalStateException( 1812 "Unrecognized buffer mode: " + mBufferMode); 1813 } 1814 } 1815 mCallback.onInputBufferAvailable(mCodec, index); 1816 break; 1817 } 1818 1819 case CB_OUTPUT_AVAILABLE: 1820 { 1821 int index = msg.arg2; 1822 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1823 synchronized(mBufferLock) { 1824 switch (mBufferMode) { 1825 case BUFFER_MODE_LEGACY: 1826 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1827 break; 1828 case BUFFER_MODE_BLOCK: 1829 while (mOutputFrames.size() <= index) { 1830 mOutputFrames.add(null); 1831 } 1832 OutputFrame frame = mOutputFrames.get(index); 1833 if (frame == null) { 1834 frame = new OutputFrame(index); 1835 mOutputFrames.set(index, frame); 1836 } 1837 frame.setBufferInfo(info); 1838 frame.setAccessible(true); 1839 break; 1840 default: 1841 throw new IllegalStateException( 1842 "Unrecognized buffer mode: " + mBufferMode); 1843 } 1844 } 1845 mCallback.onOutputBufferAvailable( 1846 mCodec, index, info); 1847 break; 1848 } 1849 1850 case CB_ERROR: 1851 { 1852 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1853 break; 1854 } 1855 1856 case CB_OUTPUT_FORMAT_CHANGE: 1857 { 1858 mCallback.onOutputFormatChanged(mCodec, 1859 new MediaFormat((Map<String, Object>) msg.obj)); 1860 break; 1861 } 1862 1863 default: 1864 { 1865 break; 1866 } 1867 } 1868 } 1869 } 1870 1871 private boolean mHasSurface = false; 1872 1873 /** 1874 * Instantiate the preferred decoder supporting input data of the given mime type. 1875 * 1876 * The following is a partial list of defined mime types and their semantics: 1877 * <ul> 1878 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1879 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1880 * <li>"video/avc" - H.264/AVC video 1881 * <li>"video/hevc" - H.265/HEVC video 1882 * <li>"video/mp4v-es" - MPEG4 video 1883 * <li>"video/3gpp" - H.263 video 1884 * <li>"audio/3gpp" - AMR narrowband audio 1885 * <li>"audio/amr-wb" - AMR wideband audio 1886 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1887 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1888 * <li>"audio/vorbis" - vorbis audio 1889 * <li>"audio/g711-alaw" - G.711 alaw audio 1890 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1891 * </ul> 1892 * 1893 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1894 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1895 * given format. 1896 * 1897 * @param type The mime type of the input data. 1898 * @throws IOException if the codec cannot be created. 1899 * @throws IllegalArgumentException if type is not a valid mime type. 1900 * @throws NullPointerException if type is null. 1901 */ 1902 @NonNull createDecoderByType(@onNull String type)1903 public static MediaCodec createDecoderByType(@NonNull String type) 1904 throws IOException { 1905 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1906 } 1907 1908 /** 1909 * Instantiate the preferred encoder supporting output data of the given mime type. 1910 * 1911 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1912 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1913 * given format. 1914 * 1915 * @param type The desired mime type of the output data. 1916 * @throws IOException if the codec cannot be created. 1917 * @throws IllegalArgumentException if type is not a valid mime type. 1918 * @throws NullPointerException if type is null. 1919 */ 1920 @NonNull createEncoderByType(@onNull String type)1921 public static MediaCodec createEncoderByType(@NonNull String type) 1922 throws IOException { 1923 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1924 } 1925 1926 /** 1927 * If you know the exact name of the component you want to instantiate 1928 * use this method to instantiate it. Use with caution. 1929 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1930 * @param name The name of the codec to be instantiated. 1931 * @throws IOException if the codec cannot be created. 1932 * @throws IllegalArgumentException if name is not valid. 1933 * @throws NullPointerException if name is null. 1934 */ 1935 @NonNull createByCodecName(@onNull String name)1936 public static MediaCodec createByCodecName(@NonNull String name) 1937 throws IOException { 1938 return new MediaCodec( 1939 name, false /* nameIsType */, false /* unused */); 1940 } 1941 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1942 private MediaCodec( 1943 @NonNull String name, boolean nameIsType, boolean encoder) { 1944 Looper looper; 1945 if ((looper = Looper.myLooper()) != null) { 1946 mEventHandler = new EventHandler(this, looper); 1947 } else if ((looper = Looper.getMainLooper()) != null) { 1948 mEventHandler = new EventHandler(this, looper); 1949 } else { 1950 mEventHandler = null; 1951 } 1952 mCallbackHandler = mEventHandler; 1953 mOnFirstTunnelFrameReadyHandler = mEventHandler; 1954 mOnFrameRenderedHandler = mEventHandler; 1955 1956 mBufferLock = new Object(); 1957 1958 // save name used at creation 1959 mNameAtCreation = nameIsType ? null : name; 1960 1961 native_setup(name, nameIsType, encoder); 1962 } 1963 1964 private String mNameAtCreation; 1965 1966 @Override finalize()1967 protected void finalize() { 1968 native_finalize(); 1969 mCrypto = null; 1970 } 1971 1972 /** 1973 * Returns the codec to its initial (Uninitialized) state. 1974 * 1975 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1976 * error has occured to reset the codec to its initial state after creation. 1977 * 1978 * @throws CodecException if an unrecoverable error has occured and the codec 1979 * could not be reset. 1980 * @throws IllegalStateException if in the Released state. 1981 */ reset()1982 public final void reset() { 1983 freeAllTrackedBuffers(); // free buffers first 1984 native_reset(); 1985 mCrypto = null; 1986 } 1987 native_reset()1988 private native final void native_reset(); 1989 1990 /** 1991 * Free up resources used by the codec instance. 1992 * 1993 * Make sure you call this when you're done to free up any opened 1994 * component instance instead of relying on the garbage collector 1995 * to do this for you at some point in the future. 1996 */ release()1997 public final void release() { 1998 freeAllTrackedBuffers(); // free buffers first 1999 native_release(); 2000 mCrypto = null; 2001 } 2002 native_release()2003 private native final void native_release(); 2004 2005 /** 2006 * If this codec is to be used as an encoder, pass this flag. 2007 */ 2008 public static final int CONFIGURE_FLAG_ENCODE = 1; 2009 2010 /** 2011 * If this codec is to be used with {@link LinearBlock} and/or {@link 2012 * HardwareBuffer}, pass this flag. 2013 * <p> 2014 * When this flag is set, the following APIs throw {@link IncompatibleWithBlockModelException}. 2015 * <ul> 2016 * <li>{@link #getInputBuffer} 2017 * <li>{@link #getInputImage} 2018 * <li>{@link #getInputBuffers} 2019 * <li>{@link #getOutputBuffer} 2020 * <li>{@link #getOutputImage} 2021 * <li>{@link #getOutputBuffers} 2022 * <li>{@link #queueInputBuffer} 2023 * <li>{@link #queueSecureInputBuffer} 2024 * <li>{@link #dequeueInputBuffer} 2025 * <li>{@link #dequeueOutputBuffer} 2026 * </ul> 2027 */ 2028 public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; 2029 2030 /** @hide */ 2031 @IntDef( 2032 flag = true, 2033 value = { 2034 CONFIGURE_FLAG_ENCODE, 2035 CONFIGURE_FLAG_USE_BLOCK_MODEL, 2036 }) 2037 @Retention(RetentionPolicy.SOURCE) 2038 public @interface ConfigureFlag {} 2039 2040 /** 2041 * Thrown when the codec is configured for block model and an incompatible API is called. 2042 */ 2043 public class IncompatibleWithBlockModelException extends RuntimeException { IncompatibleWithBlockModelException()2044 IncompatibleWithBlockModelException() { } 2045 IncompatibleWithBlockModelException(String message)2046 IncompatibleWithBlockModelException(String message) { 2047 super(message); 2048 } 2049 IncompatibleWithBlockModelException(String message, Throwable cause)2050 IncompatibleWithBlockModelException(String message, Throwable cause) { 2051 super(message, cause); 2052 } 2053 IncompatibleWithBlockModelException(Throwable cause)2054 IncompatibleWithBlockModelException(Throwable cause) { 2055 super(cause); 2056 } 2057 } 2058 2059 /** 2060 * Configures a component. 2061 * 2062 * @param format The format of the input data (decoder) or the desired 2063 * format of the output data (encoder). Passing {@code null} 2064 * as {@code format} is equivalent to passing an 2065 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2066 * @param surface Specify a surface on which to render the output of this 2067 * decoder. Pass {@code null} as {@code surface} if the 2068 * codec does not generate raw video output (e.g. not a video 2069 * decoder) and/or if you want to configure the codec for 2070 * {@link ByteBuffer} output. 2071 * @param crypto Specify a crypto object to facilitate secure decryption 2072 * of the media data. Pass {@code null} as {@code crypto} for 2073 * non-secure codecs. 2074 * Please note that {@link MediaCodec} does NOT take ownership 2075 * of the {@link MediaCrypto} object; it is the application's 2076 * responsibility to properly cleanup the {@link MediaCrypto} object 2077 * when not in use. 2078 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2079 * component as an encoder. 2080 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2081 * or the format is unacceptable (e.g. missing a mandatory key), 2082 * or the flags are not set properly 2083 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2084 * @throws IllegalStateException if not in the Uninitialized state. 2085 * @throws CryptoException upon DRM error. 2086 * @throws CodecException upon codec error. 2087 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)2088 public void configure( 2089 @Nullable MediaFormat format, 2090 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2091 @ConfigureFlag int flags) { 2092 configure(format, surface, crypto, null, flags); 2093 } 2094 2095 /** 2096 * Configure a component to be used with a descrambler. 2097 * @param format The format of the input data (decoder) or the desired 2098 * format of the output data (encoder). Passing {@code null} 2099 * as {@code format} is equivalent to passing an 2100 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2101 * @param surface Specify a surface on which to render the output of this 2102 * decoder. Pass {@code null} as {@code surface} if the 2103 * codec does not generate raw video output (e.g. not a video 2104 * decoder) and/or if you want to configure the codec for 2105 * {@link ByteBuffer} output. 2106 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2107 * component as an encoder. 2108 * @param descrambler Specify a descrambler object to facilitate secure 2109 * descrambling of the media data, or null for non-secure codecs. 2110 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2111 * or the format is unacceptable (e.g. missing a mandatory key), 2112 * or the flags are not set properly 2113 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2114 * @throws IllegalStateException if not in the Uninitialized state. 2115 * @throws CryptoException upon DRM error. 2116 * @throws CodecException upon codec error. 2117 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)2118 public void configure( 2119 @Nullable MediaFormat format, @Nullable Surface surface, 2120 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 2121 configure(format, surface, null, 2122 descrambler != null ? descrambler.getBinder() : null, flags); 2123 } 2124 2125 private static final int BUFFER_MODE_INVALID = -1; 2126 private static final int BUFFER_MODE_LEGACY = 0; 2127 private static final int BUFFER_MODE_BLOCK = 1; 2128 private int mBufferMode = BUFFER_MODE_INVALID; 2129 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2130 private void configure( 2131 @Nullable MediaFormat format, @Nullable Surface surface, 2132 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 2133 @ConfigureFlag int flags) { 2134 if (crypto != null && descramblerBinder != null) { 2135 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 2136 } 2137 2138 String[] keys = null; 2139 Object[] values = null; 2140 2141 if (format != null) { 2142 Map<String, Object> formatMap = format.getMap(); 2143 keys = new String[formatMap.size()]; 2144 values = new Object[formatMap.size()]; 2145 2146 int i = 0; 2147 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2148 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2149 int sessionId = 0; 2150 try { 2151 sessionId = (Integer)entry.getValue(); 2152 } 2153 catch (Exception e) { 2154 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2155 } 2156 keys[i] = "audio-hw-sync"; 2157 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2158 } else { 2159 keys[i] = entry.getKey(); 2160 values[i] = entry.getValue(); 2161 } 2162 ++i; 2163 } 2164 } 2165 2166 mHasSurface = surface != null; 2167 mCrypto = crypto; 2168 synchronized (mBufferLock) { 2169 if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { 2170 mBufferMode = BUFFER_MODE_BLOCK; 2171 } else { 2172 mBufferMode = BUFFER_MODE_LEGACY; 2173 } 2174 } 2175 2176 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2177 } 2178 2179 /** 2180 * Dynamically sets the output surface of a codec. 2181 * <p> 2182 * This can only be used if the codec was configured with an output surface. The 2183 * new output surface should have a compatible usage type to the original output surface. 2184 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2185 * to ImageReader (software readable) output. 2186 * @param surface the output surface to use. It must not be {@code null}. 2187 * @throws IllegalStateException if the codec does not support setting the output 2188 * surface in the current state. 2189 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2190 */ setOutputSurface(@onNull Surface surface)2191 public void setOutputSurface(@NonNull Surface surface) { 2192 if (!mHasSurface) { 2193 throw new IllegalStateException("codec was not configured for an output surface"); 2194 } 2195 native_setSurface(surface); 2196 } 2197 native_setSurface(@onNull Surface surface)2198 private native void native_setSurface(@NonNull Surface surface); 2199 2200 /** 2201 * Create a persistent input surface that can be used with codecs that normally have an input 2202 * surface, such as video encoders. A persistent input can be reused by subsequent 2203 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2204 * most one codec or recorder instance concurrently. 2205 * <p> 2206 * The application is responsible for calling release() on the Surface when done. 2207 * 2208 * @return an input surface that can be used with {@link #setInputSurface}. 2209 */ 2210 @NonNull createPersistentInputSurface()2211 public static Surface createPersistentInputSurface() { 2212 return native_createPersistentInputSurface(); 2213 } 2214 2215 static class PersistentSurface extends Surface { 2216 @SuppressWarnings("unused") PersistentSurface()2217 PersistentSurface() {} // used by native 2218 2219 @Override release()2220 public void release() { 2221 native_releasePersistentInputSurface(this); 2222 super.release(); 2223 } 2224 2225 private long mPersistentObject; 2226 }; 2227 2228 /** 2229 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2230 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2231 * lieu of {@link #createInputSurface}. 2232 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2233 * @throws IllegalStateException if not in the Configured state or does not require an input 2234 * surface. 2235 * @throws IllegalArgumentException if the surface was not created by 2236 * {@link #createPersistentInputSurface}. 2237 */ setInputSurface(@onNull Surface surface)2238 public void setInputSurface(@NonNull Surface surface) { 2239 if (!(surface instanceof PersistentSurface)) { 2240 throw new IllegalArgumentException("not a PersistentSurface"); 2241 } 2242 native_setInputSurface(surface); 2243 } 2244 2245 @NonNull native_createPersistentInputSurface()2246 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2247 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2248 private native final void native_setInputSurface(@NonNull Surface surface); 2249 native_setCallback(@ullable Callback cb)2250 private native final void native_setCallback(@Nullable Callback cb); 2251 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2252 private native final void native_configure( 2253 @Nullable String[] keys, @Nullable Object[] values, 2254 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2255 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2256 2257 /** 2258 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2259 * may only be called after {@link #configure} and before {@link #start}. 2260 * <p> 2261 * The application is responsible for calling release() on the Surface when 2262 * done. 2263 * <p> 2264 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2265 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2266 * unexpected results. 2267 * @throws IllegalStateException if not in the Configured state. 2268 */ 2269 @NonNull createInputSurface()2270 public native final Surface createInputSurface(); 2271 2272 /** 2273 * After successfully configuring the component, call {@code start}. 2274 * <p> 2275 * Call {@code start} also if the codec is configured in asynchronous mode, 2276 * and it has just been flushed, to resume requesting input buffers. 2277 * @throws IllegalStateException if not in the Configured state 2278 * or just after {@link #flush} for a codec that is configured 2279 * in asynchronous mode. 2280 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2281 * for start may be attributed to future method calls. 2282 */ start()2283 public final void start() { 2284 native_start(); 2285 synchronized(mBufferLock) { 2286 cacheBuffers(true /* input */); 2287 cacheBuffers(false /* input */); 2288 } 2289 } native_start()2290 private native final void native_start(); 2291 2292 /** 2293 * Finish the decode/encode session, note that the codec instance 2294 * remains active and ready to be {@link #start}ed again. 2295 * To ensure that it is available to other client call {@link #release} 2296 * and don't just rely on garbage collection to eventually do this for you. 2297 * @throws IllegalStateException if in the Released state. 2298 */ stop()2299 public final void stop() { 2300 native_stop(); 2301 freeAllTrackedBuffers(); 2302 2303 synchronized (mListenerLock) { 2304 if (mCallbackHandler != null) { 2305 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2306 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2307 } 2308 if (mOnFirstTunnelFrameReadyHandler != null) { 2309 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 2310 } 2311 if (mOnFrameRenderedHandler != null) { 2312 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2313 } 2314 } 2315 } 2316 native_stop()2317 private native final void native_stop(); 2318 2319 /** 2320 * Flush both input and output ports of the component. 2321 * <p> 2322 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2323 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2324 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2325 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2326 * invalid, and all buffers are owned by the codec. 2327 * <p> 2328 * If the codec is configured in asynchronous mode, call {@link #start} 2329 * after {@code flush} has returned to resume codec operations. The codec 2330 * will not request input buffers until this has happened. 2331 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2332 * callbacks that were not handled prior to calling {@code flush}. 2333 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2334 * should be discarded.</strong> 2335 * <p> 2336 * If the codec is configured in synchronous mode, codec will resume 2337 * automatically if it is configured with an input surface. Otherwise, it 2338 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2339 * 2340 * @throws IllegalStateException if not in the Executing state. 2341 * @throws MediaCodec.CodecException upon codec error. 2342 */ flush()2343 public final void flush() { 2344 synchronized(mBufferLock) { 2345 invalidateByteBuffers(mCachedInputBuffers); 2346 invalidateByteBuffers(mCachedOutputBuffers); 2347 mDequeuedInputBuffers.clear(); 2348 mDequeuedOutputBuffers.clear(); 2349 } 2350 native_flush(); 2351 } 2352 native_flush()2353 private native final void native_flush(); 2354 2355 /** 2356 * Thrown when an internal codec error occurs. 2357 */ 2358 public final static class CodecException extends IllegalStateException { 2359 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2360 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2361 super(detailMessage); 2362 mErrorCode = errorCode; 2363 mActionCode = actionCode; 2364 2365 // TODO get this from codec 2366 final String sign = errorCode < 0 ? "neg_" : ""; 2367 mDiagnosticInfo = 2368 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2369 } 2370 2371 /** 2372 * Returns true if the codec exception is a transient issue, 2373 * perhaps due to resource constraints, and that the method 2374 * (or encoding/decoding) may be retried at a later time. 2375 */ 2376 public boolean isTransient() { 2377 return mActionCode == ACTION_TRANSIENT; 2378 } 2379 2380 /** 2381 * Returns true if the codec cannot proceed further, 2382 * but can be recovered by stopping, configuring, 2383 * and starting again. 2384 */ 2385 public boolean isRecoverable() { 2386 return mActionCode == ACTION_RECOVERABLE; 2387 } 2388 2389 /** 2390 * Retrieve the error code associated with a CodecException 2391 */ 2392 public int getErrorCode() { 2393 return mErrorCode; 2394 } 2395 2396 /** 2397 * Retrieve a developer-readable diagnostic information string 2398 * associated with the exception. Do not show this to end-users, 2399 * since this string will not be localized or generally 2400 * comprehensible to end-users. 2401 */ 2402 public @NonNull String getDiagnosticInfo() { 2403 return mDiagnosticInfo; 2404 } 2405 2406 /** 2407 * This indicates required resource was not able to be allocated. 2408 */ 2409 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2410 2411 /** 2412 * This indicates the resource manager reclaimed the media resource used by the codec. 2413 * <p> 2414 * With this exception, the codec must be released, as it has moved to terminal state. 2415 */ 2416 public static final int ERROR_RECLAIMED = 1101; 2417 2418 /** @hide */ 2419 @IntDef({ 2420 ERROR_INSUFFICIENT_RESOURCE, 2421 ERROR_RECLAIMED, 2422 }) 2423 @Retention(RetentionPolicy.SOURCE) 2424 public @interface ReasonCode {} 2425 2426 /* Must be in sync with android_media_MediaCodec.cpp */ 2427 private final static int ACTION_TRANSIENT = 1; 2428 private final static int ACTION_RECOVERABLE = 2; 2429 2430 private final String mDiagnosticInfo; 2431 private final int mErrorCode; 2432 private final int mActionCode; 2433 } 2434 2435 /** 2436 * Thrown when a crypto error occurs while queueing a secure input buffer. 2437 */ 2438 public final static class CryptoException extends RuntimeException { 2439 public CryptoException(int errorCode, @Nullable String detailMessage) { 2440 super(detailMessage); 2441 mErrorCode = errorCode; 2442 } 2443 2444 /** 2445 * This indicates that the requested key was not found when trying to 2446 * perform a decrypt operation. The operation can be retried after adding 2447 * the correct decryption key. 2448 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_NO_KEY}. 2449 */ 2450 public static final int ERROR_NO_KEY = MediaDrm.ErrorCodes.ERROR_NO_KEY; 2451 2452 /** 2453 * This indicates that the key used for decryption is no longer 2454 * valid due to license term expiration. The operation can be retried 2455 * after updating the expired keys. 2456 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_KEY_EXPIRED}. 2457 */ 2458 public static final int ERROR_KEY_EXPIRED = MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED; 2459 2460 /** 2461 * This indicates that a required crypto resource was not able to be 2462 * allocated while attempting the requested operation. The operation 2463 * can be retried if the app is able to release resources. 2464 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_RESOURCE_BUSY} 2465 */ 2466 public static final int ERROR_RESOURCE_BUSY = MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY; 2467 2468 /** 2469 * This indicates that the output protection levels supported by the 2470 * device are not sufficient to meet the requirements set by the 2471 * content owner in the license policy. 2472 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_OUTPUT_PROTECTION} 2473 */ 2474 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 2475 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION; 2476 2477 /** 2478 * This indicates that decryption was attempted on a session that is 2479 * not opened, which could be due to a failure to open the session, 2480 * closing the session prematurely, or the session being reclaimed 2481 * by the resource manager. 2482 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_SESSION_NOT_OPENED} 2483 */ 2484 public static final int ERROR_SESSION_NOT_OPENED = 2485 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED; 2486 2487 /** 2488 * This indicates that an operation was attempted that could not be 2489 * supported by the crypto system of the device in its current 2490 * configuration. It may occur when the license policy requires 2491 * device security features that aren't supported by the device, 2492 * or due to an internal error in the crypto system that prevents 2493 * the specified security policy from being met. 2494 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_UNSUPPORTED_OPERATION} 2495 */ 2496 public static final int ERROR_UNSUPPORTED_OPERATION = 2497 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION; 2498 2499 /** 2500 * This indicates that the security level of the device is not 2501 * sufficient to meet the requirements set by the content owner 2502 * in the license policy. 2503 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_SECURITY} 2504 */ 2505 public static final int ERROR_INSUFFICIENT_SECURITY = 2506 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY; 2507 2508 /** 2509 * This indicates that the video frame being decrypted exceeds 2510 * the size of the device's protected output buffers. When 2511 * encountering this error the app should try playing content 2512 * of a lower resolution. 2513 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_FRAME_TOO_LARGE} 2514 */ 2515 public static final int ERROR_FRAME_TOO_LARGE = MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE; 2516 2517 /** 2518 * This error indicates that session state has been 2519 * invalidated. It can occur on devices that are not capable 2520 * of retaining crypto session state across device 2521 * suspend/resume. The session must be closed and a new 2522 * session opened to resume operation. 2523 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_LOST_STATE} 2524 */ 2525 public static final int ERROR_LOST_STATE = MediaDrm.ErrorCodes.ERROR_LOST_STATE; 2526 2527 /** @hide */ 2528 @IntDef({ 2529 MediaDrm.ErrorCodes.ERROR_NO_KEY, 2530 MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED, 2531 MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY, 2532 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2533 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED, 2534 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION, 2535 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY, 2536 MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE, 2537 MediaDrm.ErrorCodes.ERROR_LOST_STATE, 2538 MediaDrm.ErrorCodes.ERROR_GENERIC_OEM, 2539 MediaDrm.ErrorCodes.ERROR_GENERIC_PLUGIN, 2540 MediaDrm.ErrorCodes.ERROR_LICENSE_PARSE, 2541 MediaDrm.ErrorCodes.ERROR_MEDIA_FRAMEWORK, 2542 MediaDrm.ErrorCodes.ERROR_ZERO_SUBSAMPLES 2543 }) 2544 @Retention(RetentionPolicy.SOURCE) 2545 public @interface CryptoErrorCode {} 2546 2547 /** 2548 * Returns error code associated with this {@link CryptoException}. 2549 * <p> 2550 * Please refer to {@link MediaDrm.ErrorCodes} for the general error 2551 * handling strategy and details about each possible return value. 2552 * 2553 * @return an error code defined in {@link MediaDrm.ErrorCodes}. 2554 */ 2555 @CryptoErrorCode 2556 public int getErrorCode() { 2557 return mErrorCode; 2558 } 2559 2560 private int mErrorCode; 2561 } 2562 2563 /** 2564 * After filling a range of the input buffer at the specified index 2565 * submit it to the component. Once an input buffer is queued to 2566 * the codec, it MUST NOT be used until it is later retrieved by 2567 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2568 * return value or a {@link Callback#onInputBufferAvailable} 2569 * callback. 2570 * <p> 2571 * Many decoders require the actual compressed data stream to be 2572 * preceded by "codec specific data", i.e. setup data used to initialize 2573 * the codec such as PPS/SPS in the case of AVC video or code tables 2574 * in the case of vorbis audio. 2575 * The class {@link android.media.MediaExtractor} provides codec 2576 * specific data as part of 2577 * the returned track format in entries named "csd-0", "csd-1" ... 2578 * <p> 2579 * These buffers can be submitted directly after {@link #start} or 2580 * {@link #flush} by specifying the flag {@link 2581 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2582 * codec with a {@link MediaFormat} containing these keys, they 2583 * will be automatically submitted by MediaCodec directly after 2584 * start. Therefore, the use of {@link 2585 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2586 * recommended only for advanced users. 2587 * <p> 2588 * To indicate that this is the final piece of input data (or rather that 2589 * no more input data follows unless the decoder is subsequently flushed) 2590 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2591 * <p class=note> 2592 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2593 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2594 * Surface output buffers, and the resulting frame timestamp was undefined. 2595 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2596 * Similarly, since frame timestamps can be used by the destination surface for rendering 2597 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2598 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2599 * SurfaceView specifics}).</strong> 2600 * 2601 * @param index The index of a client-owned input buffer previously returned 2602 * in a call to {@link #dequeueInputBuffer}. 2603 * @param offset The byte offset into the input buffer at which the data starts. 2604 * @param size The number of bytes of valid input data. 2605 * @param presentationTimeUs The presentation timestamp in microseconds for this 2606 * buffer. This is normally the media time at which this 2607 * buffer should be presented (rendered). When using an output 2608 * surface, this will be propagated as the {@link 2609 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2610 * conversion to nanoseconds). 2611 * @param flags A bitmask of flags 2612 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2613 * While not prohibited, most codecs do not use the 2614 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2615 * @throws IllegalStateException if not in the Executing state. 2616 * @throws MediaCodec.CodecException upon codec error. 2617 * @throws CryptoException if a crypto object has been specified in 2618 * {@link #configure} 2619 */ 2620 public final void queueInputBuffer( 2621 int index, 2622 int offset, int size, long presentationTimeUs, int flags) 2623 throws CryptoException { 2624 synchronized(mBufferLock) { 2625 if (mBufferMode == BUFFER_MODE_BLOCK) { 2626 throw new IncompatibleWithBlockModelException("queueInputBuffer() " 2627 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2628 + "Please use getQueueRequest() to queue buffers"); 2629 } 2630 invalidateByteBuffer(mCachedInputBuffers, index); 2631 mDequeuedInputBuffers.remove(index); 2632 } 2633 try { 2634 native_queueInputBuffer( 2635 index, offset, size, presentationTimeUs, flags); 2636 } catch (CryptoException | IllegalStateException e) { 2637 revalidateByteBuffer(mCachedInputBuffers, index); 2638 throw e; 2639 } 2640 } 2641 2642 private native final void native_queueInputBuffer( 2643 int index, 2644 int offset, int size, long presentationTimeUs, int flags) 2645 throws CryptoException; 2646 2647 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2648 public static final int CRYPTO_MODE_AES_CTR = 1; 2649 public static final int CRYPTO_MODE_AES_CBC = 2; 2650 2651 /** 2652 * Metadata describing the structure of an encrypted input sample. 2653 * <p> 2654 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2655 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2656 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2657 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2658 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2659 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2660 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2661 * <p> 2662 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2663 * "Common encryption in ISO base media file format files". 2664 * <p> 2665 * <h3>ISO-CENC Schemes</h3> 2666 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2667 * corresponding to each possible combination of an AES mode with the presence or absence of 2668 * patterned encryption. 2669 * 2670 * <table style="width: 0%"> 2671 * <thead> 2672 * <tr> 2673 * <th> </th> 2674 * <th>AES-CTR</th> 2675 * <th>AES-CBC</th> 2676 * </tr> 2677 * </thead> 2678 * <tbody> 2679 * <tr> 2680 * <th>Without Patterns</th> 2681 * <td>cenc</td> 2682 * <td>cbc1</td> 2683 * </tr><tr> 2684 * <th>With Patterns</th> 2685 * <td>cens</td> 2686 * <td>cbcs</td> 2687 * </tr> 2688 * </tbody> 2689 * </table> 2690 * 2691 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2692 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2693 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2694 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2695 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2696 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2697 * {@link #setPattern} is never called is all zeroes. 2698 * <p> 2699 * <h4>HLS SAMPLE-AES Audio</h4> 2700 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2701 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2702 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2703 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2704 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2705 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2706 * while still decrypting every block. 2707 */ 2708 public final static class CryptoInfo { 2709 /** 2710 * The number of subSamples that make up the buffer's contents. 2711 */ 2712 public int numSubSamples; 2713 /** 2714 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2715 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2716 */ 2717 public int[] numBytesOfClearData; 2718 /** 2719 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2720 * as clear and {@link #numBytesOfClearData} must be specified. 2721 */ 2722 public int[] numBytesOfEncryptedData; 2723 /** 2724 * A 16-byte key id 2725 */ 2726 public byte[] key; 2727 /** 2728 * A 16-byte initialization vector 2729 */ 2730 public byte[] iv; 2731 /** 2732 * The type of encryption that has been applied, 2733 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2734 * and {@link #CRYPTO_MODE_AES_CBC} 2735 */ 2736 public int mode; 2737 2738 /** 2739 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2740 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2741 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2742 */ 2743 public final static class Pattern { 2744 /** 2745 * Number of blocks to be encrypted in the pattern. If both this and 2746 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2747 */ 2748 private int mEncryptBlocks; 2749 2750 /** 2751 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2752 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2753 */ 2754 private int mSkipBlocks; 2755 2756 /** 2757 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2758 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2759 */ 2760 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2761 set(blocksToEncrypt, blocksToSkip); 2762 } 2763 2764 /** 2765 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2766 * parameters are zero, pattern encryption is inoperative. 2767 */ 2768 public void set(int blocksToEncrypt, int blocksToSkip) { 2769 mEncryptBlocks = blocksToEncrypt; 2770 mSkipBlocks = blocksToSkip; 2771 } 2772 2773 /** 2774 * Return the number of blocks to skip in a sample encryption pattern. 2775 */ 2776 public int getSkipBlocks() { 2777 return mSkipBlocks; 2778 } 2779 2780 /** 2781 * Return the number of blocks to encrypt in a sample encryption pattern. 2782 */ 2783 public int getEncryptBlocks() { 2784 return mEncryptBlocks; 2785 } 2786 }; 2787 2788 private static final Pattern ZERO_PATTERN = new Pattern(0, 0); 2789 2790 /** 2791 * The pattern applicable to the protected data in each subsample. 2792 */ 2793 private Pattern mPattern = ZERO_PATTERN; 2794 2795 /** 2796 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2797 * a {@link MediaCodec.CryptoInfo} instance. 2798 */ 2799 public void set( 2800 int newNumSubSamples, 2801 @NonNull int[] newNumBytesOfClearData, 2802 @NonNull int[] newNumBytesOfEncryptedData, 2803 @NonNull byte[] newKey, 2804 @NonNull byte[] newIV, 2805 int newMode) { 2806 numSubSamples = newNumSubSamples; 2807 numBytesOfClearData = newNumBytesOfClearData; 2808 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2809 key = newKey; 2810 iv = newIV; 2811 mode = newMode; 2812 mPattern = ZERO_PATTERN; 2813 } 2814 2815 /** 2816 * Returns the {@link Pattern encryption pattern}. 2817 */ 2818 public @NonNull Pattern getPattern() { 2819 return new Pattern(mPattern.getEncryptBlocks(), mPattern.getSkipBlocks()); 2820 } 2821 2822 /** 2823 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2824 * See {@link Pattern}. 2825 */ 2826 public void setPattern(Pattern newPattern) { 2827 if (newPattern == null) { 2828 newPattern = ZERO_PATTERN; 2829 } 2830 setPattern(newPattern.getEncryptBlocks(), newPattern.getSkipBlocks()); 2831 } 2832 2833 // Accessed from android_media_MediaExtractor.cpp. 2834 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 2835 mPattern = new Pattern(blocksToEncrypt, blocksToSkip); 2836 } 2837 2838 @Override 2839 public String toString() { 2840 StringBuilder builder = new StringBuilder(); 2841 builder.append(numSubSamples + " subsamples, key ["); 2842 String hexdigits = "0123456789abcdef"; 2843 for (int i = 0; i < key.length; i++) { 2844 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2845 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2846 } 2847 builder.append("], iv ["); 2848 for (int i = 0; i < iv.length; i++) { 2849 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2850 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2851 } 2852 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2853 builder.append(Arrays.toString(numBytesOfClearData)); 2854 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2855 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2856 builder.append(", pattern (encrypt: "); builder.append(mPattern.mEncryptBlocks)2857 builder.append(mPattern.mEncryptBlocks); 2858 builder.append(", skip: "); builder.append(mPattern.mSkipBlocks)2859 builder.append(mPattern.mSkipBlocks); 2860 builder.append(")"); 2861 return builder.toString(); 2862 } 2863 }; 2864 2865 /** 2866 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2867 * potentially encrypted. 2868 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2869 * 2870 * @param index The index of a client-owned input buffer previously returned 2871 * in a call to {@link #dequeueInputBuffer}. 2872 * @param offset The byte offset into the input buffer at which the data starts. 2873 * @param info Metadata required to facilitate decryption, the object can be 2874 * reused immediately after this call returns. 2875 * @param presentationTimeUs The presentation timestamp in microseconds for this 2876 * buffer. This is normally the media time at which this 2877 * buffer should be presented (rendered). 2878 * @param flags A bitmask of flags 2879 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2880 * While not prohibited, most codecs do not use the 2881 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2882 * @throws IllegalStateException if not in the Executing state. 2883 * @throws MediaCodec.CodecException upon codec error. 2884 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2885 * An error code associated with the exception helps identify the 2886 * reason for the failure. 2887 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2888 public final void queueSecureInputBuffer( 2889 int index, 2890 int offset, 2891 @NonNull CryptoInfo info, 2892 long presentationTimeUs, 2893 int flags) throws CryptoException { 2894 synchronized(mBufferLock) { 2895 if (mBufferMode == BUFFER_MODE_BLOCK) { 2896 throw new IncompatibleWithBlockModelException("queueSecureInputBuffer() " 2897 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2898 + "Please use getQueueRequest() to queue buffers"); 2899 } 2900 invalidateByteBuffer(mCachedInputBuffers, index); 2901 mDequeuedInputBuffers.remove(index); 2902 } 2903 try { 2904 native_queueSecureInputBuffer( 2905 index, offset, info, presentationTimeUs, flags); 2906 } catch (CryptoException | IllegalStateException e) { 2907 revalidateByteBuffer(mCachedInputBuffers, index); 2908 throw e; 2909 } 2910 } 2911 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2912 private native final void native_queueSecureInputBuffer( 2913 int index, 2914 int offset, 2915 @NonNull CryptoInfo info, 2916 long presentationTimeUs, 2917 int flags) throws CryptoException; 2918 2919 /** 2920 * Returns the index of an input buffer to be filled with valid data 2921 * or -1 if no such buffer is currently available. 2922 * This method will return immediately if timeoutUs == 0, wait indefinitely 2923 * for the availability of an input buffer if timeoutUs < 0 or wait up 2924 * to "timeoutUs" microseconds if timeoutUs > 0. 2925 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2926 * @throws IllegalStateException if not in the Executing state, 2927 * or codec is configured in asynchronous mode. 2928 * @throws MediaCodec.CodecException upon codec error. 2929 */ dequeueInputBuffer(long timeoutUs)2930 public final int dequeueInputBuffer(long timeoutUs) { 2931 synchronized (mBufferLock) { 2932 if (mBufferMode == BUFFER_MODE_BLOCK) { 2933 throw new IncompatibleWithBlockModelException("dequeueInputBuffer() " 2934 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2935 + "Please use MediaCodec.Callback objectes to get input buffer slots."); 2936 } 2937 } 2938 int res = native_dequeueInputBuffer(timeoutUs); 2939 if (res >= 0) { 2940 synchronized(mBufferLock) { 2941 validateInputByteBuffer(mCachedInputBuffers, res); 2942 } 2943 } 2944 return res; 2945 } 2946 native_dequeueInputBuffer(long timeoutUs)2947 private native final int native_dequeueInputBuffer(long timeoutUs); 2948 2949 /** 2950 * Section of memory that represents a linear block. Applications may 2951 * acquire a block via {@link LinearBlock#obtain} and queue all or part 2952 * of the block as an input buffer to a codec, or get a block allocated by 2953 * codec as an output buffer from {@link OutputFrame}. 2954 * 2955 * {@see QueueRequest#setLinearBlock} 2956 * {@see QueueRequest#setEncryptedLinearBlock} 2957 * {@see OutputFrame#getLinearBlock} 2958 */ 2959 public static final class LinearBlock { 2960 // No public constructors. LinearBlock()2961 private LinearBlock() {} 2962 2963 /** 2964 * Returns true if the buffer is mappable. 2965 * @throws IllegalStateException if invalid 2966 */ isMappable()2967 public boolean isMappable() { 2968 synchronized (mLock) { 2969 if (!mValid) { 2970 throw new IllegalStateException("The linear block is invalid"); 2971 } 2972 return mMappable; 2973 } 2974 } 2975 2976 /** 2977 * Map the memory and return the mapped region. 2978 * <p> 2979 * The returned memory region becomes inaccessible after 2980 * {@link #recycle}, or the buffer is queued to the codecs and not 2981 * returned to the client yet. 2982 * 2983 * @return mapped memory region as {@link ByteBuffer} object 2984 * @throws IllegalStateException if not mappable or invalid 2985 */ map()2986 public @NonNull ByteBuffer map() { 2987 synchronized (mLock) { 2988 if (!mValid) { 2989 throw new IllegalStateException("The linear block is invalid"); 2990 } 2991 if (!mMappable) { 2992 throw new IllegalStateException("The linear block is not mappable"); 2993 } 2994 if (mMapped == null) { 2995 mMapped = native_map(); 2996 } 2997 return mMapped; 2998 } 2999 } 3000 native_map()3001 private native ByteBuffer native_map(); 3002 3003 /** 3004 * Mark this block as ready to be recycled by the framework once it is 3005 * no longer in use. All operations to this object after 3006 * this call will cause exceptions, as well as attempt to access the 3007 * previously mapped memory region. Caller should clear all references 3008 * to this object after this call. 3009 * <p> 3010 * To avoid excessive memory consumption, it is recommended that callers 3011 * recycle buffers as soon as they no longer need the buffers 3012 * 3013 * @throws IllegalStateException if invalid 3014 */ recycle()3015 public void recycle() { 3016 synchronized (mLock) { 3017 if (!mValid) { 3018 throw new IllegalStateException("The linear block is invalid"); 3019 } 3020 if (mMapped != null) { 3021 mMapped.setAccessible(false); 3022 mMapped = null; 3023 } 3024 native_recycle(); 3025 mValid = false; 3026 mNativeContext = 0; 3027 } 3028 sPool.offer(this); 3029 } 3030 native_recycle()3031 private native void native_recycle(); 3032 native_obtain(int capacity, String[] codecNames)3033 private native void native_obtain(int capacity, String[] codecNames); 3034 3035 @Override finalize()3036 protected void finalize() { 3037 native_recycle(); 3038 } 3039 3040 /** 3041 * Returns true if it is possible to allocate a linear block that can be 3042 * passed to all listed codecs as input buffers without copying the 3043 * content. 3044 * <p> 3045 * Note that even if this function returns true, {@link #obtain} may 3046 * still throw due to invalid arguments or allocation failure. 3047 * 3048 * @param codecNames list of codecs that the client wants to use a 3049 * linear block without copying. Null entries are 3050 * ignored. 3051 */ isCodecCopyFreeCompatible(@onNull String[] codecNames)3052 public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { 3053 return native_checkCompatible(codecNames); 3054 } 3055 native_checkCompatible(@onNull String[] codecNames)3056 private static native boolean native_checkCompatible(@NonNull String[] codecNames); 3057 3058 /** 3059 * Obtain a linear block object no smaller than {@code capacity}. 3060 * If {@link #isCodecCopyFreeCompatible} with the same 3061 * {@code codecNames} returned true, the returned 3062 * {@link LinearBlock} object can be queued to the listed codecs without 3063 * copying. The returned {@link LinearBlock} object is always 3064 * read/write mappable. 3065 * 3066 * @param capacity requested capacity of the linear block in bytes 3067 * @param codecNames list of codecs that the client wants to use this 3068 * linear block without copying. Null entries are 3069 * ignored. 3070 * @return a linear block object. 3071 * @throws IllegalArgumentException if the capacity is invalid or 3072 * codecNames contains invalid name 3073 * @throws IOException if an error occurred while allocating a buffer 3074 */ obtain( int capacity, @NonNull String[] codecNames)3075 public static @Nullable LinearBlock obtain( 3076 int capacity, @NonNull String[] codecNames) { 3077 LinearBlock buffer = sPool.poll(); 3078 if (buffer == null) { 3079 buffer = new LinearBlock(); 3080 } 3081 synchronized (buffer.mLock) { 3082 buffer.native_obtain(capacity, codecNames); 3083 } 3084 return buffer; 3085 } 3086 3087 // Called from native setInternalStateLocked(long context, boolean isMappable)3088 private void setInternalStateLocked(long context, boolean isMappable) { 3089 mNativeContext = context; 3090 mMappable = isMappable; 3091 mValid = (context != 0); 3092 } 3093 3094 private static final BlockingQueue<LinearBlock> sPool = 3095 new LinkedBlockingQueue<>(); 3096 3097 private final Object mLock = new Object(); 3098 private boolean mValid = false; 3099 private boolean mMappable = false; 3100 private ByteBuffer mMapped = null; 3101 private long mNativeContext = 0; 3102 } 3103 3104 /** 3105 * Map a {@link HardwareBuffer} object into {@link Image}, so that the content of the buffer is 3106 * accessible. Depending on the usage and pixel format of the hardware buffer, it may not be 3107 * mappable; this method returns null in that case. 3108 * 3109 * @param hardwareBuffer {@link HardwareBuffer} to map. 3110 * @return Mapped {@link Image} object, or null if the buffer is not mappable. 3111 */ mapHardwareBuffer(@onNull HardwareBuffer hardwareBuffer)3112 public static @Nullable Image mapHardwareBuffer(@NonNull HardwareBuffer hardwareBuffer) { 3113 return native_mapHardwareBuffer(hardwareBuffer); 3114 } 3115 native_mapHardwareBuffer( @onNull HardwareBuffer hardwareBuffer)3116 private static native @Nullable Image native_mapHardwareBuffer( 3117 @NonNull HardwareBuffer hardwareBuffer); 3118 native_closeMediaImage(long context)3119 private static native void native_closeMediaImage(long context); 3120 3121 /** 3122 * Builder-like class for queue requests. Use this class to prepare a 3123 * queue request and send it. 3124 */ 3125 public final class QueueRequest { 3126 // No public constructor QueueRequest(@onNull MediaCodec codec, int index)3127 private QueueRequest(@NonNull MediaCodec codec, int index) { 3128 mCodec = codec; 3129 mIndex = index; 3130 } 3131 3132 /** 3133 * Set a linear block to this queue request. Exactly one buffer must be 3134 * set for a queue request before calling {@link #queue}. It is possible 3135 * to use the same {@link LinearBlock} object for multiple queue 3136 * requests. The behavior is undefined if the range of the buffer 3137 * overlaps for multiple requests, or the application writes into the 3138 * region being processed by the codec. 3139 * 3140 * @param block The linear block object 3141 * @param offset The byte offset into the input buffer at which the data starts. 3142 * @param size The number of bytes of valid input data. 3143 * @return this object 3144 * @throws IllegalStateException if a buffer is already set 3145 */ setLinearBlock( @onNull LinearBlock block, int offset, int size)3146 public @NonNull QueueRequest setLinearBlock( 3147 @NonNull LinearBlock block, 3148 int offset, 3149 int size) { 3150 if (!isAccessible()) { 3151 throw new IllegalStateException("The request is stale"); 3152 } 3153 if (mLinearBlock != null || mHardwareBuffer != null) { 3154 throw new IllegalStateException("Cannot set block twice"); 3155 } 3156 mLinearBlock = block; 3157 mOffset = offset; 3158 mSize = size; 3159 mCryptoInfo = null; 3160 return this; 3161 } 3162 3163 /** 3164 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3165 * set for a queue request before calling {@link #queue}. It is possible 3166 * to use the same {@link LinearBlock} object for multiple queue 3167 * requests. The behavior is undefined if the range of the buffer 3168 * overlaps for multiple requests, or the application writes into the 3169 * region being processed by the codec. 3170 * 3171 * @param block The linear block object 3172 * @param offset The byte offset into the input buffer at which the data starts. 3173 * @param size The number of bytes of valid input data. 3174 * @param cryptoInfo Metadata describing the structure of the encrypted input sample. 3175 * @return this object 3176 * @throws IllegalStateException if a buffer is already set 3177 */ setEncryptedLinearBlock( @onNull LinearBlock block, int offset, int size, @NonNull MediaCodec.CryptoInfo cryptoInfo)3178 public @NonNull QueueRequest setEncryptedLinearBlock( 3179 @NonNull LinearBlock block, 3180 int offset, 3181 int size, 3182 @NonNull MediaCodec.CryptoInfo cryptoInfo) { 3183 Objects.requireNonNull(cryptoInfo); 3184 if (!isAccessible()) { 3185 throw new IllegalStateException("The request is stale"); 3186 } 3187 if (mLinearBlock != null || mHardwareBuffer != null) { 3188 throw new IllegalStateException("Cannot set block twice"); 3189 } 3190 mLinearBlock = block; 3191 mOffset = offset; 3192 mSize = size; 3193 mCryptoInfo = cryptoInfo; 3194 return this; 3195 } 3196 3197 /** 3198 * Set a harware graphic buffer to this queue request. Exactly one buffer must 3199 * be set for a queue request before calling {@link #queue}. 3200 * <p> 3201 * Note: buffers should have format {@link HardwareBuffer#YCBCR_420_888}, 3202 * a single layer, and an appropriate usage ({@link HardwareBuffer#USAGE_CPU_READ_OFTEN} 3203 * for software codecs and {@link HardwareBuffer#USAGE_VIDEO_ENCODE} for hardware) 3204 * for codecs to recognize. Codecs may throw exception if the buffer is not recognizable. 3205 * 3206 * @param buffer The hardware graphic buffer object 3207 * @return this object 3208 * @throws IllegalStateException if a buffer is already set 3209 */ setHardwareBuffer( @onNull HardwareBuffer buffer)3210 public @NonNull QueueRequest setHardwareBuffer( 3211 @NonNull HardwareBuffer buffer) { 3212 if (!isAccessible()) { 3213 throw new IllegalStateException("The request is stale"); 3214 } 3215 if (mLinearBlock != null || mHardwareBuffer != null) { 3216 throw new IllegalStateException("Cannot set block twice"); 3217 } 3218 mHardwareBuffer = buffer; 3219 return this; 3220 } 3221 3222 /** 3223 * Set timestamp to this queue request. 3224 * 3225 * @param presentationTimeUs The presentation timestamp in microseconds for this 3226 * buffer. This is normally the media time at which this 3227 * buffer should be presented (rendered). When using an output 3228 * surface, this will be propagated as the {@link 3229 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3230 * conversion to nanoseconds). 3231 * @return this object 3232 */ setPresentationTimeUs(long presentationTimeUs)3233 public @NonNull QueueRequest setPresentationTimeUs(long presentationTimeUs) { 3234 if (!isAccessible()) { 3235 throw new IllegalStateException("The request is stale"); 3236 } 3237 mPresentationTimeUs = presentationTimeUs; 3238 return this; 3239 } 3240 3241 /** 3242 * Set flags to this queue request. 3243 * 3244 * @param flags A bitmask of flags 3245 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3246 * While not prohibited, most codecs do not use the 3247 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3248 * @return this object 3249 */ setFlags(@ufferFlag int flags)3250 public @NonNull QueueRequest setFlags(@BufferFlag int flags) { 3251 if (!isAccessible()) { 3252 throw new IllegalStateException("The request is stale"); 3253 } 3254 mFlags = flags; 3255 return this; 3256 } 3257 3258 /** 3259 * Add an integer parameter. 3260 * See {@link MediaFormat} for an exhaustive list of supported keys with 3261 * values of type int, that can also be set with {@link MediaFormat#setInteger}. 3262 * 3263 * If there was {@link MediaCodec#setParameters} 3264 * call with the same key which is not processed by the codec yet, the 3265 * value set from this method will override the unprocessed value. 3266 * 3267 * @return this object 3268 */ setIntegerParameter( @onNull String key, int value)3269 public @NonNull QueueRequest setIntegerParameter( 3270 @NonNull String key, int value) { 3271 if (!isAccessible()) { 3272 throw new IllegalStateException("The request is stale"); 3273 } 3274 mTuningKeys.add(key); 3275 mTuningValues.add(Integer.valueOf(value)); 3276 return this; 3277 } 3278 3279 /** 3280 * Add a long parameter. 3281 * See {@link MediaFormat} for an exhaustive list of supported keys with 3282 * values of type long, that can also be set with {@link MediaFormat#setLong}. 3283 * 3284 * If there was {@link MediaCodec#setParameters} 3285 * call with the same key which is not processed by the codec yet, the 3286 * value set from this method will override the unprocessed value. 3287 * 3288 * @return this object 3289 */ setLongParameter( @onNull String key, long value)3290 public @NonNull QueueRequest setLongParameter( 3291 @NonNull String key, long value) { 3292 if (!isAccessible()) { 3293 throw new IllegalStateException("The request is stale"); 3294 } 3295 mTuningKeys.add(key); 3296 mTuningValues.add(Long.valueOf(value)); 3297 return this; 3298 } 3299 3300 /** 3301 * Add a float parameter. 3302 * See {@link MediaFormat} for an exhaustive list of supported keys with 3303 * values of type float, that can also be set with {@link MediaFormat#setFloat}. 3304 * 3305 * If there was {@link MediaCodec#setParameters} 3306 * call with the same key which is not processed by the codec yet, the 3307 * value set from this method will override the unprocessed value. 3308 * 3309 * @return this object 3310 */ setFloatParameter( @onNull String key, float value)3311 public @NonNull QueueRequest setFloatParameter( 3312 @NonNull String key, float value) { 3313 if (!isAccessible()) { 3314 throw new IllegalStateException("The request is stale"); 3315 } 3316 mTuningKeys.add(key); 3317 mTuningValues.add(Float.valueOf(value)); 3318 return this; 3319 } 3320 3321 /** 3322 * Add a {@link ByteBuffer} parameter. 3323 * See {@link MediaFormat} for an exhaustive list of supported keys with 3324 * values of byte buffer, that can also be set with {@link MediaFormat#setByteBuffer}. 3325 * 3326 * If there was {@link MediaCodec#setParameters} 3327 * call with the same key which is not processed by the codec yet, the 3328 * value set from this method will override the unprocessed value. 3329 * 3330 * @return this object 3331 */ setByteBufferParameter( @onNull String key, @NonNull ByteBuffer value)3332 public @NonNull QueueRequest setByteBufferParameter( 3333 @NonNull String key, @NonNull ByteBuffer value) { 3334 if (!isAccessible()) { 3335 throw new IllegalStateException("The request is stale"); 3336 } 3337 mTuningKeys.add(key); 3338 mTuningValues.add(value); 3339 return this; 3340 } 3341 3342 /** 3343 * Add a string parameter. 3344 * See {@link MediaFormat} for an exhaustive list of supported keys with 3345 * values of type string, that can also be set with {@link MediaFormat#setString}. 3346 * 3347 * If there was {@link MediaCodec#setParameters} 3348 * call with the same key which is not processed by the codec yet, the 3349 * value set from this method will override the unprocessed value. 3350 * 3351 * @return this object 3352 */ setStringParameter( @onNull String key, @NonNull String value)3353 public @NonNull QueueRequest setStringParameter( 3354 @NonNull String key, @NonNull String value) { 3355 if (!isAccessible()) { 3356 throw new IllegalStateException("The request is stale"); 3357 } 3358 mTuningKeys.add(key); 3359 mTuningValues.add(value); 3360 return this; 3361 } 3362 3363 /** 3364 * Finish building a queue request and queue the buffers with tunings. 3365 */ queue()3366 public void queue() { 3367 if (!isAccessible()) { 3368 throw new IllegalStateException("The request is stale"); 3369 } 3370 if (mLinearBlock == null && mHardwareBuffer == null) { 3371 throw new IllegalStateException("No block is set"); 3372 } 3373 setAccessible(false); 3374 if (mLinearBlock != null) { 3375 mCodec.native_queueLinearBlock( 3376 mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo, 3377 mPresentationTimeUs, mFlags, 3378 mTuningKeys, mTuningValues); 3379 } else if (mHardwareBuffer != null) { 3380 mCodec.native_queueHardwareBuffer( 3381 mIndex, mHardwareBuffer, mPresentationTimeUs, mFlags, 3382 mTuningKeys, mTuningValues); 3383 } 3384 clear(); 3385 } 3386 clear()3387 @NonNull QueueRequest clear() { 3388 mLinearBlock = null; 3389 mOffset = 0; 3390 mSize = 0; 3391 mCryptoInfo = null; 3392 mHardwareBuffer = null; 3393 mPresentationTimeUs = 0; 3394 mFlags = 0; 3395 mTuningKeys.clear(); 3396 mTuningValues.clear(); 3397 return this; 3398 } 3399 isAccessible()3400 boolean isAccessible() { 3401 return mAccessible; 3402 } 3403 setAccessible(boolean accessible)3404 @NonNull QueueRequest setAccessible(boolean accessible) { 3405 mAccessible = accessible; 3406 return this; 3407 } 3408 3409 private final MediaCodec mCodec; 3410 private final int mIndex; 3411 private LinearBlock mLinearBlock = null; 3412 private int mOffset = 0; 3413 private int mSize = 0; 3414 private MediaCodec.CryptoInfo mCryptoInfo = null; 3415 private HardwareBuffer mHardwareBuffer = null; 3416 private long mPresentationTimeUs = 0; 3417 private @BufferFlag int mFlags = 0; 3418 private final ArrayList<String> mTuningKeys = new ArrayList<>(); 3419 private final ArrayList<Object> mTuningValues = new ArrayList<>(); 3420 3421 private boolean mAccessible = false; 3422 } 3423 native_queueLinearBlock( int index, @NonNull LinearBlock block, int offset, int size, @Nullable CryptoInfo cryptoInfo, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3424 private native void native_queueLinearBlock( 3425 int index, 3426 @NonNull LinearBlock block, 3427 int offset, 3428 int size, 3429 @Nullable CryptoInfo cryptoInfo, 3430 long presentationTimeUs, 3431 int flags, 3432 @NonNull ArrayList<String> keys, 3433 @NonNull ArrayList<Object> values); 3434 native_queueHardwareBuffer( int index, @NonNull HardwareBuffer buffer, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3435 private native void native_queueHardwareBuffer( 3436 int index, 3437 @NonNull HardwareBuffer buffer, 3438 long presentationTimeUs, 3439 int flags, 3440 @NonNull ArrayList<String> keys, 3441 @NonNull ArrayList<Object> values); 3442 3443 private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); 3444 3445 /** 3446 * Return a {@link QueueRequest} object for an input slot index. 3447 * 3448 * @param index input slot index from 3449 * {@link Callback#onInputBufferAvailable} 3450 * @return queue request object 3451 * @throws IllegalStateException if not using block model 3452 * @throws IllegalArgumentException if the input slot is not available or 3453 * the index is out of range 3454 */ getQueueRequest(int index)3455 public @NonNull QueueRequest getQueueRequest(int index) { 3456 synchronized (mBufferLock) { 3457 if (mBufferMode != BUFFER_MODE_BLOCK) { 3458 throw new IllegalStateException("The codec is not configured for block model"); 3459 } 3460 if (index < 0 || index >= mQueueRequests.size()) { 3461 throw new IndexOutOfBoundsException("Expected range of index: [0," 3462 + (mQueueRequests.size() - 1) + "]; actual: " + index); 3463 } 3464 QueueRequest request = mQueueRequests.get(index); 3465 if (request == null) { 3466 throw new IllegalArgumentException("Unavailable index: " + index); 3467 } 3468 if (!request.isAccessible()) { 3469 throw new IllegalArgumentException( 3470 "The request is stale at index " + index); 3471 } 3472 return request.clear(); 3473 } 3474 } 3475 3476 /** 3477 * If a non-negative timeout had been specified in the call 3478 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 3479 */ 3480 public static final int INFO_TRY_AGAIN_LATER = -1; 3481 3482 /** 3483 * The output format has changed, subsequent data will follow the new 3484 * format. {@link #getOutputFormat()} returns the new format. Note, that 3485 * you can also use the new {@link #getOutputFormat(int)} method to 3486 * get the format for a specific output buffer. This frees you from 3487 * having to track output format changes. 3488 */ 3489 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 3490 3491 /** 3492 * The output buffers have changed, the client must refer to the new 3493 * set of output buffers returned by {@link #getOutputBuffers} from 3494 * this point on. 3495 * 3496 * <p>Additionally, this event signals that the video scaling mode 3497 * may have been reset to the default.</p> 3498 * 3499 * @deprecated This return value can be ignored as {@link 3500 * #getOutputBuffers} has been deprecated. Client should 3501 * request a current buffer using on of the get-buffer or 3502 * get-image methods each time one has been dequeued. 3503 */ 3504 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 3505 3506 /** @hide */ 3507 @IntDef({ 3508 INFO_TRY_AGAIN_LATER, 3509 INFO_OUTPUT_FORMAT_CHANGED, 3510 INFO_OUTPUT_BUFFERS_CHANGED, 3511 }) 3512 @Retention(RetentionPolicy.SOURCE) 3513 public @interface OutputBufferInfo {} 3514 3515 /** 3516 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 3517 * Returns the index of an output buffer that has been successfully 3518 * decoded or one of the INFO_* constants. 3519 * @param info Will be filled with buffer meta data. 3520 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3521 * @throws IllegalStateException if not in the Executing state, 3522 * or codec is configured in asynchronous mode. 3523 * @throws MediaCodec.CodecException upon codec error. 3524 */ 3525 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3526 public final int dequeueOutputBuffer( 3527 @NonNull BufferInfo info, long timeoutUs) { 3528 synchronized (mBufferLock) { 3529 if (mBufferMode == BUFFER_MODE_BLOCK) { 3530 throw new IncompatibleWithBlockModelException("dequeueOutputBuffer() " 3531 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3532 + "Please use MediaCodec.Callback objects to get output buffer slots."); 3533 } 3534 } 3535 int res = native_dequeueOutputBuffer(info, timeoutUs); 3536 synchronized (mBufferLock) { 3537 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 3538 cacheBuffers(false /* input */); 3539 } else if (res >= 0) { 3540 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 3541 if (mHasSurface) { 3542 mDequeuedOutputInfos.put(res, info.dup()); 3543 } 3544 } 3545 } 3546 return res; 3547 } 3548 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3549 private native final int native_dequeueOutputBuffer( 3550 @NonNull BufferInfo info, long timeoutUs); 3551 3552 /** 3553 * If you are done with a buffer, use this call to return the buffer to the codec 3554 * or to render it on the output surface. If you configured the codec with an 3555 * output surface, setting {@code render} to {@code true} will first send the buffer 3556 * to that output surface. The surface will release the buffer back to the codec once 3557 * it is no longer used/displayed. 3558 * 3559 * Once an output buffer is released to the codec, it MUST NOT 3560 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3561 * to a {@link #dequeueOutputBuffer} return value or a 3562 * {@link Callback#onOutputBufferAvailable} callback. 3563 * 3564 * @param index The index of a client-owned output buffer previously returned 3565 * from a call to {@link #dequeueOutputBuffer}. 3566 * @param render If a valid surface was specified when configuring the codec, 3567 * passing true renders this output buffer to the surface. 3568 * @throws IllegalStateException if not in the Executing state. 3569 * @throws MediaCodec.CodecException upon codec error. 3570 */ releaseOutputBuffer(int index, boolean render)3571 public final void releaseOutputBuffer(int index, boolean render) { 3572 releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); 3573 } 3574 3575 /** 3576 * If you are done with a buffer, use this call to update its surface timestamp 3577 * and return it to the codec to render it on the output surface. If you 3578 * have not specified an output surface when configuring this video codec, 3579 * this call will simply return the buffer to the codec.<p> 3580 * 3581 * The timestamp may have special meaning depending on the destination surface. 3582 * 3583 * <table> 3584 * <tr><th>SurfaceView specifics</th></tr> 3585 * <tr><td> 3586 * If you render your buffer on a {@link android.view.SurfaceView}, 3587 * you can use the timestamp to render the buffer at a specific time (at the 3588 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 3589 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 3590 * Currently, this is set as within one (1) second. A few notes: 3591 * 3592 * <ul> 3593 * <li>the buffer will not be returned to the codec until the timestamp 3594 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 3595 * <li>buffers are processed sequentially, so you may block subsequent buffers to 3596 * be displayed on the {@link android.view.Surface}. This is important if you 3597 * want to react to user action, e.g. stop the video or seek. 3598 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 3599 * rendered at the same VSYNC, the last one will be shown, and the other ones 3600 * will be dropped. 3601 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 3602 * time, the {@link android.view.Surface} will ignore the timestamp, and 3603 * display the buffer at the earliest feasible time. In this mode it will not 3604 * drop frames. 3605 * <li>for best performance and quality, call this method when you are about 3606 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 3607 * about 33 msec. 3608 * </ul> 3609 * </td></tr> 3610 * </table> 3611 * 3612 * Once an output buffer is released to the codec, it MUST NOT 3613 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3614 * to a {@link #dequeueOutputBuffer} return value or a 3615 * {@link Callback#onOutputBufferAvailable} callback. 3616 * 3617 * @param index The index of a client-owned output buffer previously returned 3618 * from a call to {@link #dequeueOutputBuffer}. 3619 * @param renderTimestampNs The timestamp to associate with this buffer when 3620 * it is sent to the Surface. 3621 * @throws IllegalStateException if not in the Executing state. 3622 * @throws MediaCodec.CodecException upon codec error. 3623 */ releaseOutputBuffer(int index, long renderTimestampNs)3624 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 3625 releaseOutputBufferInternal( 3626 index, true /* render */, true /* updatePTS */, renderTimestampNs); 3627 } 3628 releaseOutputBufferInternal( int index, boolean render, boolean updatePts, long renderTimestampNs)3629 private void releaseOutputBufferInternal( 3630 int index, boolean render, boolean updatePts, long renderTimestampNs) { 3631 BufferInfo info = null; 3632 synchronized(mBufferLock) { 3633 switch (mBufferMode) { 3634 case BUFFER_MODE_LEGACY: 3635 invalidateByteBuffer(mCachedOutputBuffers, index); 3636 mDequeuedOutputBuffers.remove(index); 3637 if (mHasSurface) { 3638 info = mDequeuedOutputInfos.remove(index); 3639 } 3640 break; 3641 case BUFFER_MODE_BLOCK: 3642 OutputFrame frame = mOutputFrames.get(index); 3643 frame.setAccessible(false); 3644 frame.clear(); 3645 break; 3646 default: 3647 throw new IllegalStateException( 3648 "Unrecognized buffer mode: " + mBufferMode); 3649 } 3650 } 3651 releaseOutputBuffer( 3652 index, render, updatePts, renderTimestampNs); 3653 } 3654 3655 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)3656 private native final void releaseOutputBuffer( 3657 int index, boolean render, boolean updatePTS, long timeNs); 3658 3659 /** 3660 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 3661 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 3662 * encoders receiving input from a Surface created by {@link #createInputSurface}. 3663 * @throws IllegalStateException if not in the Executing state. 3664 * @throws MediaCodec.CodecException upon codec error. 3665 */ signalEndOfInputStream()3666 public native final void signalEndOfInputStream(); 3667 3668 /** 3669 * Call this after dequeueOutputBuffer signals a format change by returning 3670 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 3671 * You can also call this after {@link #configure} returns 3672 * successfully to get the output format initially configured 3673 * for the codec. Do this to determine what optional 3674 * configuration parameters were supported by the codec. 3675 * 3676 * @throws IllegalStateException if not in the Executing or 3677 * Configured state. 3678 * @throws MediaCodec.CodecException upon codec error. 3679 */ 3680 @NonNull getOutputFormat()3681 public final MediaFormat getOutputFormat() { 3682 return new MediaFormat(getFormatNative(false /* input */)); 3683 } 3684 3685 /** 3686 * Call this after {@link #configure} returns successfully to 3687 * get the input format accepted by the codec. Do this to 3688 * determine what optional configuration parameters were 3689 * supported by the codec. 3690 * 3691 * @throws IllegalStateException if not in the Executing or 3692 * Configured state. 3693 * @throws MediaCodec.CodecException upon codec error. 3694 */ 3695 @NonNull getInputFormat()3696 public final MediaFormat getInputFormat() { 3697 return new MediaFormat(getFormatNative(true /* input */)); 3698 } 3699 3700 /** 3701 * Returns the output format for a specific output buffer. 3702 * 3703 * @param index The index of a client-owned input buffer previously 3704 * returned from a call to {@link #dequeueInputBuffer}. 3705 * 3706 * @return the format for the output buffer, or null if the index 3707 * is not a dequeued output buffer. 3708 */ 3709 @NonNull getOutputFormat(int index)3710 public final MediaFormat getOutputFormat(int index) { 3711 return new MediaFormat(getOutputFormatNative(index)); 3712 } 3713 3714 @NonNull getFormatNative(boolean input)3715 private native final Map<String, Object> getFormatNative(boolean input); 3716 3717 @NonNull getOutputFormatNative(int index)3718 private native final Map<String, Object> getOutputFormatNative(int index); 3719 3720 // used to track dequeued buffers 3721 private static class BufferMap { 3722 // various returned representations of the codec buffer 3723 private static class CodecBuffer { 3724 private Image mImage; 3725 private ByteBuffer mByteBuffer; 3726 free()3727 public void free() { 3728 if (mByteBuffer != null) { 3729 // all of our ByteBuffers are direct 3730 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 3731 mByteBuffer = null; 3732 } 3733 if (mImage != null) { 3734 mImage.close(); 3735 mImage = null; 3736 } 3737 } 3738 setImage(@ullable Image image)3739 public void setImage(@Nullable Image image) { 3740 free(); 3741 mImage = image; 3742 } 3743 setByteBuffer(@ullable ByteBuffer buffer)3744 public void setByteBuffer(@Nullable ByteBuffer buffer) { 3745 free(); 3746 mByteBuffer = buffer; 3747 } 3748 } 3749 3750 private final Map<Integer, CodecBuffer> mMap = 3751 new HashMap<Integer, CodecBuffer>(); 3752 remove(int index)3753 public void remove(int index) { 3754 CodecBuffer buffer = mMap.get(index); 3755 if (buffer != null) { 3756 buffer.free(); 3757 mMap.remove(index); 3758 } 3759 } 3760 put(int index, @Nullable ByteBuffer newBuffer)3761 public void put(int index, @Nullable ByteBuffer newBuffer) { 3762 CodecBuffer buffer = mMap.get(index); 3763 if (buffer == null) { // likely 3764 buffer = new CodecBuffer(); 3765 mMap.put(index, buffer); 3766 } 3767 buffer.setByteBuffer(newBuffer); 3768 } 3769 put(int index, @Nullable Image newImage)3770 public void put(int index, @Nullable Image newImage) { 3771 CodecBuffer buffer = mMap.get(index); 3772 if (buffer == null) { // likely 3773 buffer = new CodecBuffer(); 3774 mMap.put(index, buffer); 3775 } 3776 buffer.setImage(newImage); 3777 } 3778 clear()3779 public void clear() { 3780 for (CodecBuffer buffer: mMap.values()) { 3781 buffer.free(); 3782 } 3783 mMap.clear(); 3784 } 3785 } 3786 3787 private ByteBuffer[] mCachedInputBuffers; 3788 private ByteBuffer[] mCachedOutputBuffers; 3789 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 3790 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 3791 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 3792 new HashMap<Integer, BufferInfo>(); 3793 final private Object mBufferLock; 3794 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3795 private final void invalidateByteBuffer( 3796 @Nullable ByteBuffer[] buffers, int index) { 3797 if (buffers != null && index >= 0 && index < buffers.length) { 3798 ByteBuffer buffer = buffers[index]; 3799 if (buffer != null) { 3800 buffer.setAccessible(false); 3801 } 3802 } 3803 } 3804 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)3805 private final void validateInputByteBuffer( 3806 @Nullable ByteBuffer[] buffers, int index) { 3807 if (buffers != null && index >= 0 && index < buffers.length) { 3808 ByteBuffer buffer = buffers[index]; 3809 if (buffer != null) { 3810 buffer.setAccessible(true); 3811 buffer.clear(); 3812 } 3813 } 3814 } 3815 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3816 private final void revalidateByteBuffer( 3817 @Nullable ByteBuffer[] buffers, int index) { 3818 synchronized(mBufferLock) { 3819 if (buffers != null && index >= 0 && index < buffers.length) { 3820 ByteBuffer buffer = buffers[index]; 3821 if (buffer != null) { 3822 buffer.setAccessible(true); 3823 } 3824 } 3825 } 3826 } 3827 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)3828 private final void validateOutputByteBuffer( 3829 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 3830 if (buffers != null && index >= 0 && index < buffers.length) { 3831 ByteBuffer buffer = buffers[index]; 3832 if (buffer != null) { 3833 buffer.setAccessible(true); 3834 buffer.limit(info.offset + info.size).position(info.offset); 3835 } 3836 } 3837 } 3838 invalidateByteBuffers(@ullable ByteBuffer[] buffers)3839 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 3840 if (buffers != null) { 3841 for (ByteBuffer buffer: buffers) { 3842 if (buffer != null) { 3843 buffer.setAccessible(false); 3844 } 3845 } 3846 } 3847 } 3848 freeByteBuffer(@ullable ByteBuffer buffer)3849 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 3850 if (buffer != null /* && buffer.isDirect() */) { 3851 // all of our ByteBuffers are direct 3852 java.nio.NioUtils.freeDirectBuffer(buffer); 3853 } 3854 } 3855 freeByteBuffers(@ullable ByteBuffer[] buffers)3856 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 3857 if (buffers != null) { 3858 for (ByteBuffer buffer: buffers) { 3859 freeByteBuffer(buffer); 3860 } 3861 } 3862 } 3863 freeAllTrackedBuffers()3864 private final void freeAllTrackedBuffers() { 3865 synchronized(mBufferLock) { 3866 freeByteBuffers(mCachedInputBuffers); 3867 freeByteBuffers(mCachedOutputBuffers); 3868 mCachedInputBuffers = null; 3869 mCachedOutputBuffers = null; 3870 mDequeuedInputBuffers.clear(); 3871 mDequeuedOutputBuffers.clear(); 3872 mQueueRequests.clear(); 3873 mOutputFrames.clear(); 3874 } 3875 } 3876 cacheBuffers(boolean input)3877 private final void cacheBuffers(boolean input) { 3878 ByteBuffer[] buffers = null; 3879 try { 3880 buffers = getBuffers(input); 3881 invalidateByteBuffers(buffers); 3882 } catch (IllegalStateException e) { 3883 // we don't get buffers in async mode 3884 } 3885 if (input) { 3886 mCachedInputBuffers = buffers; 3887 } else { 3888 mCachedOutputBuffers = buffers; 3889 } 3890 } 3891 3892 /** 3893 * Retrieve the set of input buffers. Call this after start() 3894 * returns. After calling this method, any ByteBuffers 3895 * previously returned by an earlier call to this method MUST no 3896 * longer be used. 3897 * 3898 * @deprecated Use the new {@link #getInputBuffer} method instead 3899 * each time an input buffer is dequeued. 3900 * 3901 * <b>Note:</b> As of API 21, dequeued input buffers are 3902 * automatically {@link java.nio.Buffer#clear cleared}. 3903 * 3904 * <em>Do not use this method if using an input surface.</em> 3905 * 3906 * @throws IllegalStateException if not in the Executing state, 3907 * or codec is configured in asynchronous mode. 3908 * @throws MediaCodec.CodecException upon codec error. 3909 */ 3910 @NonNull getInputBuffers()3911 public ByteBuffer[] getInputBuffers() { 3912 synchronized (mBufferLock) { 3913 if (mBufferMode == BUFFER_MODE_BLOCK) { 3914 throw new IncompatibleWithBlockModelException("getInputBuffers() " 3915 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3916 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 3917 + "objects and attach to QueueRequest objects."); 3918 } 3919 if (mCachedInputBuffers == null) { 3920 throw new IllegalStateException(); 3921 } 3922 // FIXME: check codec status 3923 return mCachedInputBuffers; 3924 } 3925 } 3926 3927 /** 3928 * Retrieve the set of output buffers. Call this after start() 3929 * returns and whenever dequeueOutputBuffer signals an output 3930 * buffer change by returning {@link 3931 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 3932 * ByteBuffers previously returned by an earlier call to this 3933 * method MUST no longer be used. 3934 * 3935 * @deprecated Use the new {@link #getOutputBuffer} method instead 3936 * each time an output buffer is dequeued. This method is not 3937 * supported if codec is configured in asynchronous mode. 3938 * 3939 * <b>Note:</b> As of API 21, the position and limit of output 3940 * buffers that are dequeued will be set to the valid data 3941 * range. 3942 * 3943 * <em>Do not use this method if using an output surface.</em> 3944 * 3945 * @throws IllegalStateException if not in the Executing state, 3946 * or codec is configured in asynchronous mode. 3947 * @throws MediaCodec.CodecException upon codec error. 3948 */ 3949 @NonNull getOutputBuffers()3950 public ByteBuffer[] getOutputBuffers() { 3951 synchronized (mBufferLock) { 3952 if (mBufferMode == BUFFER_MODE_BLOCK) { 3953 throw new IncompatibleWithBlockModelException("getOutputBuffers() " 3954 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3955 + "Please use getOutputFrame to get output frames."); 3956 } 3957 if (mCachedOutputBuffers == null) { 3958 throw new IllegalStateException(); 3959 } 3960 // FIXME: check codec status 3961 return mCachedOutputBuffers; 3962 } 3963 } 3964 3965 /** 3966 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 3967 * object for a dequeued input buffer index to contain the input data. 3968 * 3969 * After calling this method any ByteBuffer or Image object 3970 * previously returned for the same input index MUST no longer 3971 * be used. 3972 * 3973 * @param index The index of a client-owned input buffer previously 3974 * returned from a call to {@link #dequeueInputBuffer}, 3975 * or received via an onInputBufferAvailable callback. 3976 * 3977 * @return the input buffer, or null if the index is not a dequeued 3978 * input buffer, or if the codec is configured for surface input. 3979 * 3980 * @throws IllegalStateException if not in the Executing state. 3981 * @throws MediaCodec.CodecException upon codec error. 3982 */ 3983 @Nullable getInputBuffer(int index)3984 public ByteBuffer getInputBuffer(int index) { 3985 synchronized (mBufferLock) { 3986 if (mBufferMode == BUFFER_MODE_BLOCK) { 3987 throw new IncompatibleWithBlockModelException("getInputBuffer() " 3988 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3989 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 3990 + "objects and attach to QueueRequest objects."); 3991 } 3992 } 3993 ByteBuffer newBuffer = getBuffer(true /* input */, index); 3994 synchronized (mBufferLock) { 3995 invalidateByteBuffer(mCachedInputBuffers, index); 3996 mDequeuedInputBuffers.put(index, newBuffer); 3997 } 3998 return newBuffer; 3999 } 4000 4001 /** 4002 * Returns a writable Image object for a dequeued input buffer 4003 * index to contain the raw input video frame. 4004 * 4005 * After calling this method any ByteBuffer or Image object 4006 * previously returned for the same input index MUST no longer 4007 * be used. 4008 * 4009 * @param index The index of a client-owned input buffer previously 4010 * returned from a call to {@link #dequeueInputBuffer}, 4011 * or received via an onInputBufferAvailable callback. 4012 * 4013 * @return the input image, or null if the index is not a 4014 * dequeued input buffer, or not a ByteBuffer that contains a 4015 * raw image. 4016 * 4017 * @throws IllegalStateException if not in the Executing state. 4018 * @throws MediaCodec.CodecException upon codec error. 4019 */ 4020 @Nullable getInputImage(int index)4021 public Image getInputImage(int index) { 4022 synchronized (mBufferLock) { 4023 if (mBufferMode == BUFFER_MODE_BLOCK) { 4024 throw new IncompatibleWithBlockModelException("getInputImage() " 4025 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4026 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4027 + "objects and attach to QueueRequest objects."); 4028 } 4029 } 4030 Image newImage = getImage(true /* input */, index); 4031 synchronized (mBufferLock) { 4032 invalidateByteBuffer(mCachedInputBuffers, index); 4033 mDequeuedInputBuffers.put(index, newImage); 4034 } 4035 return newImage; 4036 } 4037 4038 /** 4039 * Returns a read-only ByteBuffer for a dequeued output buffer 4040 * index. The position and limit of the returned buffer are set 4041 * to the valid output data. 4042 * 4043 * After calling this method, any ByteBuffer or Image object 4044 * previously returned for the same output index MUST no longer 4045 * be used. 4046 * 4047 * @param index The index of a client-owned output buffer previously 4048 * returned from a call to {@link #dequeueOutputBuffer}, 4049 * or received via an onOutputBufferAvailable callback. 4050 * 4051 * @return the output buffer, or null if the index is not a dequeued 4052 * output buffer, or the codec is configured with an output surface. 4053 * 4054 * @throws IllegalStateException if not in the Executing state. 4055 * @throws MediaCodec.CodecException upon codec error. 4056 */ 4057 @Nullable getOutputBuffer(int index)4058 public ByteBuffer getOutputBuffer(int index) { 4059 synchronized (mBufferLock) { 4060 if (mBufferMode == BUFFER_MODE_BLOCK) { 4061 throw new IncompatibleWithBlockModelException("getOutputBuffer() " 4062 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4063 + "Please use getOutputFrame() to get output frames."); 4064 } 4065 } 4066 ByteBuffer newBuffer = getBuffer(false /* input */, index); 4067 synchronized (mBufferLock) { 4068 invalidateByteBuffer(mCachedOutputBuffers, index); 4069 mDequeuedOutputBuffers.put(index, newBuffer); 4070 } 4071 return newBuffer; 4072 } 4073 4074 /** 4075 * Returns a read-only Image object for a dequeued output buffer 4076 * index that contains the raw video frame. 4077 * 4078 * After calling this method, any ByteBuffer or Image object previously 4079 * returned for the same output index MUST no longer be used. 4080 * 4081 * @param index The index of a client-owned output buffer previously 4082 * returned from a call to {@link #dequeueOutputBuffer}, 4083 * or received via an onOutputBufferAvailable callback. 4084 * 4085 * @return the output image, or null if the index is not a 4086 * dequeued output buffer, not a raw video frame, or if the codec 4087 * was configured with an output surface. 4088 * 4089 * @throws IllegalStateException if not in the Executing state. 4090 * @throws MediaCodec.CodecException upon codec error. 4091 */ 4092 @Nullable getOutputImage(int index)4093 public Image getOutputImage(int index) { 4094 synchronized (mBufferLock) { 4095 if (mBufferMode == BUFFER_MODE_BLOCK) { 4096 throw new IncompatibleWithBlockModelException("getOutputImage() " 4097 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4098 + "Please use getOutputFrame() to get output frames."); 4099 } 4100 } 4101 Image newImage = getImage(false /* input */, index); 4102 synchronized (mBufferLock) { 4103 invalidateByteBuffer(mCachedOutputBuffers, index); 4104 mDequeuedOutputBuffers.put(index, newImage); 4105 } 4106 return newImage; 4107 } 4108 4109 /** 4110 * A single output frame and its associated metadata. 4111 */ 4112 public static final class OutputFrame { 4113 // No public constructor OutputFrame(int index)4114 OutputFrame(int index) { 4115 mIndex = index; 4116 } 4117 4118 /** 4119 * Returns the output linear block, or null if this frame is empty. 4120 * 4121 * @throws IllegalStateException if this output frame is not linear. 4122 */ getLinearBlock()4123 public @Nullable LinearBlock getLinearBlock() { 4124 if (mHardwareBuffer != null) { 4125 throw new IllegalStateException("This output frame is not linear"); 4126 } 4127 return mLinearBlock; 4128 } 4129 4130 /** 4131 * Returns the output hardware graphic buffer, or null if this frame is empty. 4132 * 4133 * @throws IllegalStateException if this output frame is not graphic. 4134 */ getHardwareBuffer()4135 public @Nullable HardwareBuffer getHardwareBuffer() { 4136 if (mLinearBlock != null) { 4137 throw new IllegalStateException("This output frame is not graphic"); 4138 } 4139 return mHardwareBuffer; 4140 } 4141 4142 /** 4143 * Returns the presentation timestamp in microseconds. 4144 */ getPresentationTimeUs()4145 public long getPresentationTimeUs() { 4146 return mPresentationTimeUs; 4147 } 4148 4149 /** 4150 * Returns the buffer flags. 4151 */ getFlags()4152 public @BufferFlag int getFlags() { 4153 return mFlags; 4154 } 4155 4156 /** 4157 * Returns a read-only {@link MediaFormat} for this frame. The returned 4158 * object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}. 4159 */ getFormat()4160 public @NonNull MediaFormat getFormat() { 4161 return mFormat; 4162 } 4163 4164 /** 4165 * Returns an unmodifiable set of the names of entries that has changed from 4166 * the previous frame. The entries may have been removed/changed/added. 4167 * Client can find out what the change is by querying {@link MediaFormat} 4168 * object returned from {@link #getFormat}. 4169 */ getChangedKeys()4170 public @NonNull Set<String> getChangedKeys() { 4171 if (mKeySet.isEmpty() && !mChangedKeys.isEmpty()) { 4172 mKeySet.addAll(mChangedKeys); 4173 } 4174 return Collections.unmodifiableSet(mKeySet); 4175 } 4176 clear()4177 void clear() { 4178 mLinearBlock = null; 4179 mHardwareBuffer = null; 4180 mFormat = null; 4181 mChangedKeys.clear(); 4182 mKeySet.clear(); 4183 mLoaded = false; 4184 } 4185 isAccessible()4186 boolean isAccessible() { 4187 return mAccessible; 4188 } 4189 setAccessible(boolean accessible)4190 void setAccessible(boolean accessible) { 4191 mAccessible = accessible; 4192 } 4193 setBufferInfo(MediaCodec.BufferInfo info)4194 void setBufferInfo(MediaCodec.BufferInfo info) { 4195 mPresentationTimeUs = info.presentationTimeUs; 4196 mFlags = info.flags; 4197 } 4198 isLoaded()4199 boolean isLoaded() { 4200 return mLoaded; 4201 } 4202 setLoaded(boolean loaded)4203 void setLoaded(boolean loaded) { 4204 mLoaded = loaded; 4205 } 4206 4207 private final int mIndex; 4208 private LinearBlock mLinearBlock = null; 4209 private HardwareBuffer mHardwareBuffer = null; 4210 private long mPresentationTimeUs = 0; 4211 private @BufferFlag int mFlags = 0; 4212 private MediaFormat mFormat = null; 4213 private final ArrayList<String> mChangedKeys = new ArrayList<>(); 4214 private final Set<String> mKeySet = new HashSet<>(); 4215 private boolean mAccessible = false; 4216 private boolean mLoaded = false; 4217 } 4218 4219 private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); 4220 4221 /** 4222 * Returns an {@link OutputFrame} object. 4223 * 4224 * @param index output buffer index from 4225 * {@link Callback#onOutputBufferAvailable} 4226 * @return {@link OutputFrame} object describing the output buffer 4227 * @throws IllegalStateException if not using block model 4228 * @throws IllegalArgumentException if the output buffer is not available or 4229 * the index is out of range 4230 */ getOutputFrame(int index)4231 public @NonNull OutputFrame getOutputFrame(int index) { 4232 synchronized (mBufferLock) { 4233 if (mBufferMode != BUFFER_MODE_BLOCK) { 4234 throw new IllegalStateException("The codec is not configured for block model"); 4235 } 4236 if (index < 0 || index >= mOutputFrames.size()) { 4237 throw new IndexOutOfBoundsException("Expected range of index: [0," 4238 + (mQueueRequests.size() - 1) + "]; actual: " + index); 4239 } 4240 OutputFrame frame = mOutputFrames.get(index); 4241 if (frame == null) { 4242 throw new IllegalArgumentException("Unavailable index: " + index); 4243 } 4244 if (!frame.isAccessible()) { 4245 throw new IllegalArgumentException( 4246 "The output frame is stale at index " + index); 4247 } 4248 if (!frame.isLoaded()) { 4249 native_getOutputFrame(frame, index); 4250 frame.setLoaded(true); 4251 } 4252 return frame; 4253 } 4254 } 4255 native_getOutputFrame(OutputFrame frame, int index)4256 private native void native_getOutputFrame(OutputFrame frame, int index); 4257 4258 /** 4259 * The content is scaled to the surface dimensions 4260 */ 4261 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 4262 4263 /** 4264 * The content is scaled, maintaining its aspect ratio, the whole 4265 * surface area is used, content may be cropped. 4266 * <p class=note> 4267 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 4268 * configure the pixel aspect ratio for a {@link Surface}. 4269 * <p class=note> 4270 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 4271 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 4272 */ 4273 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 4274 4275 /** @hide */ 4276 @IntDef({ 4277 VIDEO_SCALING_MODE_SCALE_TO_FIT, 4278 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 4279 }) 4280 @Retention(RetentionPolicy.SOURCE) 4281 public @interface VideoScalingMode {} 4282 4283 /** 4284 * If a surface has been specified in a previous call to {@link #configure} 4285 * specifies the scaling mode to use. The default is "scale to fit". 4286 * <p class=note> 4287 * The scaling mode may be reset to the <strong>default</strong> each time an 4288 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 4289 * must call this method after every buffer change event (and before the first output buffer is 4290 * released for rendering) to ensure consistent scaling mode. 4291 * <p class=note> 4292 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 4293 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 4294 * 4295 * @throws IllegalArgumentException if mode is not recognized. 4296 * @throws IllegalStateException if in the Released state. 4297 */ setVideoScalingMode(@ideoScalingMode int mode)4298 public native final void setVideoScalingMode(@VideoScalingMode int mode); 4299 4300 /** 4301 * Sets the audio presentation. 4302 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 4303 */ setAudioPresentation(@onNull AudioPresentation presentation)4304 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 4305 if (presentation == null) { 4306 throw new NullPointerException("audio presentation is null"); 4307 } 4308 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 4309 } 4310 native_setAudioPresentation(int presentationId, int programId)4311 private native void native_setAudioPresentation(int presentationId, int programId); 4312 4313 /** 4314 * Retrieve the codec name. 4315 * 4316 * If the codec was created by createDecoderByType or createEncoderByType, what component is 4317 * chosen is not known beforehand. This method returns the name of the codec that was 4318 * selected by the platform. 4319 * 4320 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 4321 * names) for the same underlying codec, any of which can be used to instantiate the same 4322 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 4323 * name used to create the codec in this case. 4324 * 4325 * @throws IllegalStateException if in the Released state. 4326 */ 4327 @NonNull getName()4328 public final String getName() { 4329 // get canonical name to handle exception 4330 String canonicalName = getCanonicalName(); 4331 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 4332 } 4333 4334 /** 4335 * Retrieve the underlying codec name. 4336 * 4337 * This method is similar to {@link #getName}, except that it returns the underlying component 4338 * name even if an alias was used to create this MediaCodec object by name, 4339 * 4340 * @throws IllegalStateException if in the Released state. 4341 */ 4342 @NonNull getCanonicalName()4343 public native final String getCanonicalName(); 4344 4345 /** 4346 * Return Metrics data about the current codec instance. 4347 * 4348 * @return a {@link PersistableBundle} containing the set of attributes and values 4349 * available for the media being handled by this instance of MediaCodec 4350 * The attributes are descibed in {@link MetricsConstants}. 4351 * 4352 * Additional vendor-specific fields may also be present in 4353 * the return value. 4354 */ getMetrics()4355 public PersistableBundle getMetrics() { 4356 PersistableBundle bundle = native_getMetrics(); 4357 return bundle; 4358 } 4359 native_getMetrics()4360 private native PersistableBundle native_getMetrics(); 4361 4362 /** 4363 * Change a video encoder's target bitrate on the fly. The value is an 4364 * Integer object containing the new bitrate in bps. 4365 * 4366 * @see #setParameters(Bundle) 4367 */ 4368 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 4369 4370 /** 4371 * Temporarily suspend/resume encoding of input data. While suspended 4372 * input data is effectively discarded instead of being fed into the 4373 * encoder. This parameter really only makes sense to use with an encoder 4374 * in "surface-input" mode, as the client code has no control over the 4375 * input-side of the encoder in that case. 4376 * The value is an Integer object containing the value 1 to suspend 4377 * or the value 0 to resume. 4378 * 4379 * @see #setParameters(Bundle) 4380 */ 4381 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 4382 4383 /** 4384 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 4385 * optionally use this key to specify the timestamp (in micro-second) 4386 * at which the suspend/resume operation takes effect. 4387 * 4388 * Note that the specified timestamp must be greater than or equal to the 4389 * timestamp of any previously queued suspend/resume operations. 4390 * 4391 * The value is a long int, indicating the timestamp to suspend/resume. 4392 * 4393 * @see #setParameters(Bundle) 4394 */ 4395 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 4396 4397 /** 4398 * Specify an offset (in micro-second) to be added on top of the timestamps 4399 * onward. A typical use case is to apply an adjust to the timestamps after 4400 * a period of pause by the user. 4401 * 4402 * This parameter can only be used on an encoder in "surface-input" mode. 4403 * 4404 * The value is a long int, indicating the timestamp offset to be applied. 4405 * 4406 * @see #setParameters(Bundle) 4407 */ 4408 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 4409 4410 /** 4411 * Request that the encoder produce a sync frame "soon". 4412 * Provide an Integer with the value 0. 4413 * 4414 * @see #setParameters(Bundle) 4415 */ 4416 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 4417 4418 /** 4419 * Set the HDR10+ metadata on the next queued input frame. 4420 * 4421 * Provide a byte array of data that's conforming to the 4422 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 4423 *<p> 4424 * For decoders: 4425 *<p> 4426 * When a decoder is configured for one of the HDR10+ profiles that uses 4427 * out-of-band metadata (such as {@link 4428 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 4429 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 4430 * parameter sets the HDR10+ metadata on the next input buffer queued 4431 * to the decoder. A decoder supporting these profiles must propagate 4432 * the metadata to the format of the output buffer corresponding to this 4433 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 4434 * The metadata should be applied to that output buffer and the buffers 4435 * following it (in display order), until the next output buffer (in 4436 * display order) upon which an HDR10+ metadata is set. 4437 *<p> 4438 * This parameter shouldn't be set if the decoder is not configured for 4439 * an HDR10+ profile that uses out-of-band metadata. In particular, 4440 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 4441 * where the metadata is embedded in the input buffers, for example 4442 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 4443 *<p> 4444 * For encoders: 4445 *<p> 4446 * When an encoder is configured for one of the HDR10+ profiles and the 4447 * operates in byte buffer input mode (instead of surface input mode), 4448 * this parameter sets the HDR10+ metadata on the next input buffer queued 4449 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 4450 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 4451 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 4452 * the metadata must be propagated to the format of the output buffer 4453 * corresponding to this particular input buffer (under key {@link 4454 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 4455 * in-band metadata (such as {@link 4456 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 4457 * metadata info must be embedded in the corresponding output buffer itself. 4458 *<p> 4459 * This parameter shouldn't be set if the encoder is not configured for 4460 * an HDR10+ profile, or if it's operating in surface input mode. 4461 *<p> 4462 * 4463 * @see MediaFormat#KEY_HDR10_PLUS_INFO 4464 */ 4465 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 4466 4467 /** 4468 * Enable/disable low latency decoding mode. 4469 * When enabled, the decoder doesn't hold input and output data more than 4470 * required by the codec standards. 4471 * The value is an Integer object containing the value 1 to enable 4472 * or the value 0 to disable. 4473 * 4474 * @see #setParameters(Bundle) 4475 * @see MediaFormat#KEY_LOW_LATENCY 4476 */ 4477 public static final String PARAMETER_KEY_LOW_LATENCY = 4478 MediaFormat.KEY_LOW_LATENCY; 4479 4480 /** 4481 * Control video peek of the first frame when a codec is configured for tunnel mode with 4482 * {@link MediaFormat#KEY_AUDIO_SESSION_ID} while the {@link AudioTrack} is paused. 4483 *<p> 4484 * When disabled (1) after a {@link #flush} or {@link #start}, (2) while the corresponding 4485 * {@link AudioTrack} is paused and (3) before any buffers are queued, the first frame is not to 4486 * be rendered until either this parameter is enabled or the corresponding {@link AudioTrack} 4487 * has begun playback. Once the frame is decoded and ready to be rendered, 4488 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called but the frame is 4489 * not rendered. The surface continues to show the previously-rendered content, or black if the 4490 * surface is new. A subsequent call to {@link AudioTrack#play} renders this frame and triggers 4491 * a callback to {@link OnFrameRenderedListener#onFrameRendered}, and video playback begins. 4492 *<p> 4493 * <b>Note</b>: To clear any previously rendered content and show black, configure the 4494 * MediaCodec with {@code KEY_PUSH_BLANK_BUFFERS_ON_STOP(1)}, and call {@link #stop} before 4495 * pushing new video frames to the codec. 4496 *<p> 4497 * When enabled (1) after a {@link #flush} or {@link #start} and (2) while the corresponding 4498 * {@link AudioTrack} is paused, the first frame is rendered as soon as it is decoded, or 4499 * immediately, if it has already been decoded. If not already decoded, when the frame is 4500 * decoded and ready to be rendered, 4501 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called. The frame is then 4502 * immediately rendered and {@link OnFrameRenderedListener#onFrameRendered} is subsequently 4503 * called. 4504 *<p> 4505 * The value is an Integer object containing the value 1 to enable or the value 0 to disable. 4506 *<p> 4507 * The default for this parameter is <b>enabled</b>. Once a frame has been rendered, changing 4508 * this parameter has no effect until a subsequent {@link #flush} or 4509 * {@link #stop}/{@link #start}. 4510 * 4511 * @see #setParameters(Bundle) 4512 */ 4513 public static final String PARAMETER_KEY_TUNNEL_PEEK = "tunnel-peek"; 4514 4515 /** 4516 * Communicate additional parameter changes to the component instance. 4517 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 4518 * 4519 * @param params The bundle of parameters to set. 4520 * @throws IllegalStateException if in the Released state. 4521 */ setParameters(@ullable Bundle params)4522 public final void setParameters(@Nullable Bundle params) { 4523 if (params == null) { 4524 return; 4525 } 4526 4527 String[] keys = new String[params.size()]; 4528 Object[] values = new Object[params.size()]; 4529 4530 int i = 0; 4531 for (final String key: params.keySet()) { 4532 if (key.equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 4533 int sessionId = 0; 4534 try { 4535 sessionId = (Integer)params.get(key); 4536 } catch (Exception e) { 4537 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 4538 } 4539 keys[i] = "audio-hw-sync"; 4540 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 4541 } else { 4542 keys[i] = key; 4543 Object value = params.get(key); 4544 4545 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 4546 if (value instanceof byte[]) { 4547 values[i] = ByteBuffer.wrap((byte[])value); 4548 } else { 4549 values[i] = value; 4550 } 4551 } 4552 ++i; 4553 } 4554 4555 setParameters(keys, values); 4556 } 4557 4558 /** 4559 * Sets an asynchronous callback for actionable MediaCodec events. 4560 * 4561 * If the client intends to use the component in asynchronous mode, 4562 * a valid callback should be provided before {@link #configure} is called. 4563 * 4564 * When asynchronous callback is enabled, the client should not call 4565 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 4566 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 4567 * <p> 4568 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 4569 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 4570 * even if an input surface was created. 4571 * 4572 * @param cb The callback that will run. Use {@code null} to clear a previously 4573 * set callback (before {@link #configure configure} is called and run 4574 * in synchronous mode). 4575 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 4576 * callbacks are done on the default thread (the caller's thread or the 4577 * main thread.) 4578 */ setCallback(@ullable Callback cb, @Nullable Handler handler)4579 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 4580 if (cb != null) { 4581 synchronized (mListenerLock) { 4582 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 4583 // NOTE: there are no callbacks on the handler at this time, but check anyways 4584 // even if we were to extend this to be callable dynamically, it must 4585 // be called when codec is flushed, so no messages are pending. 4586 if (newHandler != mCallbackHandler) { 4587 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4588 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4589 mCallbackHandler = newHandler; 4590 } 4591 } 4592 } else if (mCallbackHandler != null) { 4593 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4594 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4595 } 4596 4597 if (mCallbackHandler != null) { 4598 // set java callback on main handler 4599 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 4600 mCallbackHandler.sendMessage(msg); 4601 4602 // set native handler here, don't post to handler because 4603 // it may cause the callback to be delayed and set in a wrong state. 4604 // Note that native codec may start sending events to the callback 4605 // handler after this returns. 4606 native_setCallback(cb); 4607 } 4608 } 4609 4610 /** 4611 * Sets an asynchronous callback for actionable MediaCodec events on the default 4612 * looper. 4613 * <p> 4614 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 4615 * @param cb The callback that will run. Use {@code null} to clear a previously 4616 * set callback (before {@link #configure configure} is called and run 4617 * in synchronous mode). 4618 * @see #setCallback(Callback, Handler) 4619 */ setCallback(@ullable Callback cb)4620 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 4621 setCallback(cb, null /* handler */); 4622 } 4623 4624 /** 4625 * Listener to be called when the first output frame has been decoded 4626 * and is ready to be rendered for a codec configured for tunnel mode with 4627 * {@code KEY_AUDIO_SESSION_ID}. 4628 * 4629 * @see MediaCodec#setOnFirstTunnelFrameReadyListener 4630 */ 4631 public interface OnFirstTunnelFrameReadyListener { 4632 4633 /** 4634 * Called when the first output frame has been decoded and is ready to be 4635 * rendered. 4636 */ onFirstTunnelFrameReady(@onNull MediaCodec codec)4637 void onFirstTunnelFrameReady(@NonNull MediaCodec codec); 4638 } 4639 4640 /** 4641 * Registers a callback to be invoked when the first output frame has been decoded 4642 * and is ready to be rendered on a codec configured for tunnel mode with {@code 4643 * KEY_AUDIO_SESSION_ID}. 4644 * 4645 * @param handler the callback will be run on the handler's thread. If {@code 4646 * null}, the callback will be run on the default thread, which is the looper from 4647 * which the codec was created, or a new thread if there was none. 4648 * 4649 * @param listener the callback that will be run. If {@code null}, clears any registered 4650 * listener. 4651 */ setOnFirstTunnelFrameReadyListener( @ullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener)4652 public void setOnFirstTunnelFrameReadyListener( 4653 @Nullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener) { 4654 synchronized (mListenerLock) { 4655 mOnFirstTunnelFrameReadyListener = listener; 4656 if (listener != null) { 4657 EventHandler newHandler = getEventHandlerOn( 4658 handler, 4659 mOnFirstTunnelFrameReadyHandler); 4660 if (newHandler != mOnFirstTunnelFrameReadyHandler) { 4661 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 4662 } 4663 mOnFirstTunnelFrameReadyHandler = newHandler; 4664 } else if (mOnFirstTunnelFrameReadyHandler != null) { 4665 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 4666 } 4667 native_enableOnFirstTunnelFrameReadyListener(listener != null); 4668 } 4669 } 4670 native_enableOnFirstTunnelFrameReadyListener(boolean enable)4671 private native void native_enableOnFirstTunnelFrameReadyListener(boolean enable); 4672 4673 /** 4674 * Listener to be called when an output frame has rendered on the output surface 4675 * 4676 * @see MediaCodec#setOnFrameRenderedListener 4677 */ 4678 public interface OnFrameRenderedListener { 4679 4680 /** 4681 * Called when an output frame has rendered on the output surface. 4682 * <p> 4683 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4684 * render timing samples, and can be significantly delayed and batched. Some frames may have 4685 * been rendered even if there was no callback generated. 4686 * 4687 * @param codec the MediaCodec instance 4688 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 4689 * This is usually the same as specified in {@link #queueInputBuffer}; however, 4690 * some codecs may alter the media time by applying some time-based transformation, 4691 * such as frame rate conversion. In that case, presentation time corresponds 4692 * to the actual output frame rendered. 4693 * @param nanoTime The system time when the frame was rendered. 4694 * 4695 * @see System#nanoTime 4696 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)4697 public void onFrameRendered( 4698 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 4699 } 4700 4701 /** 4702 * Registers a callback to be invoked when an output frame is rendered on the output surface. 4703 * <p> 4704 * This method can be called in any codec state, but will only have an effect in the 4705 * Executing state for codecs that render buffers to the output surface. 4706 * <p> 4707 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4708 * render timing samples, and can be significantly delayed and batched. Some frames may have 4709 * been rendered even if there was no callback generated. 4710 * 4711 * @param listener the callback that will be run 4712 * @param handler the callback will be run on the handler's thread. If {@code null}, 4713 * the callback will be run on the default thread, which is the looper 4714 * from which the codec was created, or a new thread if there was none. 4715 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)4716 public void setOnFrameRenderedListener( 4717 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 4718 synchronized (mListenerLock) { 4719 mOnFrameRenderedListener = listener; 4720 if (listener != null) { 4721 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 4722 if (newHandler != mOnFrameRenderedHandler) { 4723 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4724 } 4725 mOnFrameRenderedHandler = newHandler; 4726 } else if (mOnFrameRenderedHandler != null) { 4727 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4728 } 4729 native_enableOnFrameRenderedListener(listener != null); 4730 } 4731 } 4732 native_enableOnFrameRenderedListener(boolean enable)4733 private native void native_enableOnFrameRenderedListener(boolean enable); 4734 4735 /** 4736 * Returns a list of vendor parameter names. 4737 * <p> 4738 * This method can be called in any codec state except for released state. 4739 * 4740 * @return a list containing supported vendor parameters; an empty 4741 * list if no vendor parameters are supported. The order of the 4742 * parameters is arbitrary. 4743 * @throws IllegalStateException if in the Released state. 4744 */ 4745 @NonNull getSupportedVendorParameters()4746 public List<String> getSupportedVendorParameters() { 4747 return native_getSupportedVendorParameters(); 4748 } 4749 4750 @NonNull native_getSupportedVendorParameters()4751 private native List<String> native_getSupportedVendorParameters(); 4752 4753 /** 4754 * Contains description of a parameter. 4755 */ 4756 public static class ParameterDescriptor { ParameterDescriptor()4757 private ParameterDescriptor() {} 4758 4759 /** 4760 * Returns the name of the parameter. 4761 */ 4762 @NonNull getName()4763 public String getName() { 4764 return mName; 4765 } 4766 4767 /** 4768 * Returns the type of the parameter. 4769 * {@link MediaFormat#TYPE_NULL} is never returned. 4770 */ 4771 @MediaFormat.Type getType()4772 public int getType() { 4773 return mType; 4774 } 4775 4776 @Override equals(Object o)4777 public boolean equals(Object o) { 4778 if (o == null) { 4779 return false; 4780 } 4781 if (!(o instanceof ParameterDescriptor)) { 4782 return false; 4783 } 4784 ParameterDescriptor other = (ParameterDescriptor) o; 4785 return this.mName.equals(other.mName) && this.mType == other.mType; 4786 } 4787 4788 @Override hashCode()4789 public int hashCode() { 4790 return Arrays.asList( 4791 (Object) mName, 4792 (Object) Integer.valueOf(mType)).hashCode(); 4793 } 4794 4795 private String mName; 4796 private @MediaFormat.Type int mType; 4797 } 4798 4799 /** 4800 * Describe a parameter with the name. 4801 * <p> 4802 * This method can be called in any codec state except for released state. 4803 * 4804 * @param name name of the parameter to describe, typically one from 4805 * {@link #getSupportedVendorParameters}. 4806 * @return {@link ParameterDescriptor} object that describes the parameter. 4807 * {@code null} if unrecognized / not able to describe. 4808 * @throws IllegalStateException if in the Released state. 4809 */ 4810 @Nullable getParameterDescriptor(@onNull String name)4811 public ParameterDescriptor getParameterDescriptor(@NonNull String name) { 4812 return native_getParameterDescriptor(name); 4813 } 4814 4815 @Nullable native_getParameterDescriptor(@onNull String name)4816 private native ParameterDescriptor native_getParameterDescriptor(@NonNull String name); 4817 4818 /** 4819 * Subscribe to vendor parameters, so that these parameters will be present in 4820 * {@link #getOutputFormat} and changes to these parameters generate 4821 * output format change event. 4822 * <p> 4823 * Unrecognized parameter names or standard (non-vendor) parameter names will be ignored. 4824 * {@link #reset} also resets the list of subscribed parameters. 4825 * If a parameter in {@code names} is already subscribed, it will remain subscribed. 4826 * <p> 4827 * This method can be called in any codec state except for released state. When called in 4828 * running state with newly subscribed parameters, it takes effect no later than the 4829 * processing of the subsequently queued buffer. For the new parameters, the codec will generate 4830 * output format change event. 4831 * <p> 4832 * Note that any vendor parameters set in a {@link #configure} or 4833 * {@link #setParameters} call are automatically subscribed. 4834 * <p> 4835 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 4836 * for output format change events. 4837 * 4838 * @param names names of the vendor parameters to subscribe. This may be an empty list, 4839 * and in that case this method will not change the list of subscribed parameters. 4840 * @throws IllegalStateException if in the Released state. 4841 */ subscribeToVendorParameters(@onNull List<String> names)4842 public void subscribeToVendorParameters(@NonNull List<String> names) { 4843 native_subscribeToVendorParameters(names); 4844 } 4845 native_subscribeToVendorParameters(@onNull List<String> names)4846 private native void native_subscribeToVendorParameters(@NonNull List<String> names); 4847 4848 /** 4849 * Unsubscribe from vendor parameters, so that these parameters will not be present in 4850 * {@link #getOutputFormat} and changes to these parameters no longer generate 4851 * output format change event. 4852 * <p> 4853 * Unrecognized parameter names, standard (non-vendor) parameter names will be ignored. 4854 * {@link #reset} also resets the list of subscribed parameters. 4855 * If a parameter in {@code names} is already unsubscribed, it will remain unsubscribed. 4856 * <p> 4857 * This method can be called in any codec state except for released state. When called in 4858 * running state with newly unsubscribed parameters, it takes effect no later than the 4859 * processing of the subsequently queued buffer. For the removed parameters, the codec will 4860 * generate output format change event. 4861 * <p> 4862 * Note that any vendor parameters set in a {@link #configure} or 4863 * {@link #setParameters} call are automatically subscribed, and with this method 4864 * they can be unsubscribed. 4865 * <p> 4866 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 4867 * for output format change events. 4868 * 4869 * @param names names of the vendor parameters to unsubscribe. This may be an empty list, 4870 * and in that case this method will not change the list of subscribed parameters. 4871 * @throws IllegalStateException if in the Released state. 4872 */ unsubscribeFromVendorParameters(@onNull List<String> names)4873 public void unsubscribeFromVendorParameters(@NonNull List<String> names) { 4874 native_unsubscribeFromVendorParameters(names); 4875 } 4876 native_unsubscribeFromVendorParameters(@onNull List<String> names)4877 private native void native_unsubscribeFromVendorParameters(@NonNull List<String> names); 4878 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)4879 private EventHandler getEventHandlerOn( 4880 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 4881 if (handler == null) { 4882 return mEventHandler; 4883 } else { 4884 Looper looper = handler.getLooper(); 4885 if (lastHandler.getLooper() == looper) { 4886 return lastHandler; 4887 } else { 4888 return new EventHandler(this, looper); 4889 } 4890 } 4891 } 4892 4893 /** 4894 * MediaCodec callback interface. Used to notify the user asynchronously 4895 * of various MediaCodec events. 4896 */ 4897 public static abstract class Callback { 4898 /** 4899 * Called when an input buffer becomes available. 4900 * 4901 * @param codec The MediaCodec object. 4902 * @param index The index of the available input buffer. 4903 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)4904 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 4905 4906 /** 4907 * Called when an output buffer becomes available. 4908 * 4909 * @param codec The MediaCodec object. 4910 * @param index The index of the available output buffer. 4911 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 4912 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)4913 public abstract void onOutputBufferAvailable( 4914 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 4915 4916 /** 4917 * Called when the MediaCodec encountered an error 4918 * 4919 * @param codec The MediaCodec object. 4920 * @param e The {@link MediaCodec.CodecException} object describing the error. 4921 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)4922 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 4923 4924 /** 4925 * Called when the output format has changed 4926 * 4927 * @param codec The MediaCodec object. 4928 * @param format The new output format. 4929 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)4930 public abstract void onOutputFormatChanged( 4931 @NonNull MediaCodec codec, @NonNull MediaFormat format); 4932 } 4933 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)4934 private void postEventFromNative( 4935 int what, int arg1, int arg2, @Nullable Object obj) { 4936 synchronized (mListenerLock) { 4937 EventHandler handler = mEventHandler; 4938 if (what == EVENT_CALLBACK) { 4939 handler = mCallbackHandler; 4940 } else if (what == EVENT_FIRST_TUNNEL_FRAME_READY) { 4941 handler = mOnFirstTunnelFrameReadyHandler; 4942 } else if (what == EVENT_FRAME_RENDERED) { 4943 handler = mOnFrameRenderedHandler; 4944 } 4945 if (handler != null) { 4946 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 4947 handler.sendMessage(msg); 4948 } 4949 } 4950 } 4951 4952 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)4953 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 4954 4955 /** 4956 * Get the codec info. If the codec was created by createDecoderByType 4957 * or createEncoderByType, what component is chosen is not known beforehand, 4958 * and thus the caller does not have the MediaCodecInfo. 4959 * @throws IllegalStateException if in the Released state. 4960 */ 4961 @NonNull getCodecInfo()4962 public MediaCodecInfo getCodecInfo() { 4963 // Get the codec name first. If the codec is already released, 4964 // IllegalStateException will be thrown here. 4965 String name = getName(); 4966 synchronized (mCodecInfoLock) { 4967 if (mCodecInfo == null) { 4968 // Get the codec info for this codec itself first. Only initialize 4969 // the full codec list if this somehow fails because it can be slow. 4970 mCodecInfo = getOwnCodecInfo(); 4971 if (mCodecInfo == null) { 4972 mCodecInfo = MediaCodecList.getInfoFor(name); 4973 } 4974 } 4975 return mCodecInfo; 4976 } 4977 } 4978 4979 @NonNull getOwnCodecInfo()4980 private native final MediaCodecInfo getOwnCodecInfo(); 4981 4982 @NonNull 4983 @UnsupportedAppUsage getBuffers(boolean input)4984 private native final ByteBuffer[] getBuffers(boolean input); 4985 4986 @Nullable getBuffer(boolean input, int index)4987 private native final ByteBuffer getBuffer(boolean input, int index); 4988 4989 @Nullable getImage(boolean input, int index)4990 private native final Image getImage(boolean input, int index); 4991 native_init()4992 private static native final void native_init(); 4993 native_setup( @onNull String name, boolean nameIsType, boolean encoder)4994 private native final void native_setup( 4995 @NonNull String name, boolean nameIsType, boolean encoder); 4996 native_finalize()4997 private native final void native_finalize(); 4998 4999 static { 5000 System.loadLibrary("media_jni"); native_init()5001 native_init(); 5002 } 5003 5004 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 5005 private long mNativeContext = 0; 5006 private final Lock mNativeContextLock = new ReentrantLock(); 5007 lockAndGetContext()5008 private final long lockAndGetContext() { 5009 mNativeContextLock.lock(); 5010 return mNativeContext; 5011 } 5012 setAndUnlockContext(long context)5013 private final void setAndUnlockContext(long context) { 5014 mNativeContext = context; 5015 mNativeContextLock.unlock(); 5016 } 5017 5018 /** @hide */ 5019 public static class MediaImage extends Image { 5020 private final boolean mIsReadOnly; 5021 private final int mWidth; 5022 private final int mHeight; 5023 private final int mFormat; 5024 private long mTimestamp; 5025 private final Plane[] mPlanes; 5026 private final ByteBuffer mBuffer; 5027 private final ByteBuffer mInfo; 5028 private final int mXOffset; 5029 private final int mYOffset; 5030 private final long mBufferContext; 5031 5032 private final static int TYPE_YUV = 1; 5033 5034 private final int mTransform = 0; //Default no transform 5035 private final int mScalingMode = 0; //Default frozen scaling mode 5036 5037 @Override getFormat()5038 public int getFormat() { 5039 throwISEIfImageIsInvalid(); 5040 return mFormat; 5041 } 5042 5043 @Override getHeight()5044 public int getHeight() { 5045 throwISEIfImageIsInvalid(); 5046 return mHeight; 5047 } 5048 5049 @Override getWidth()5050 public int getWidth() { 5051 throwISEIfImageIsInvalid(); 5052 return mWidth; 5053 } 5054 5055 @Override getTransform()5056 public int getTransform() { 5057 throwISEIfImageIsInvalid(); 5058 return mTransform; 5059 } 5060 5061 @Override getScalingMode()5062 public int getScalingMode() { 5063 throwISEIfImageIsInvalid(); 5064 return mScalingMode; 5065 } 5066 5067 @Override getTimestamp()5068 public long getTimestamp() { 5069 throwISEIfImageIsInvalid(); 5070 return mTimestamp; 5071 } 5072 5073 @Override 5074 @NonNull getPlanes()5075 public Plane[] getPlanes() { 5076 throwISEIfImageIsInvalid(); 5077 return Arrays.copyOf(mPlanes, mPlanes.length); 5078 } 5079 5080 @Override close()5081 public void close() { 5082 if (mIsImageValid) { 5083 if (mBuffer != null) { 5084 java.nio.NioUtils.freeDirectBuffer(mBuffer); 5085 } 5086 if (mBufferContext != 0) { 5087 native_closeMediaImage(mBufferContext); 5088 } 5089 mIsImageValid = false; 5090 } 5091 } 5092 5093 /** 5094 * Set the crop rectangle associated with this frame. 5095 * <p> 5096 * The crop rectangle specifies the region of valid pixels in the image, 5097 * using coordinates in the largest-resolution plane. 5098 */ 5099 @Override setCropRect(@ullable Rect cropRect)5100 public void setCropRect(@Nullable Rect cropRect) { 5101 if (mIsReadOnly) { 5102 throw new ReadOnlyBufferException(); 5103 } 5104 super.setCropRect(cropRect); 5105 } 5106 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)5107 public MediaImage( 5108 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 5109 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 5110 mFormat = ImageFormat.YUV_420_888; 5111 mTimestamp = timestamp; 5112 mIsImageValid = true; 5113 mIsReadOnly = buffer.isReadOnly(); 5114 mBuffer = buffer.duplicate(); 5115 5116 // save offsets and info 5117 mXOffset = xOffset; 5118 mYOffset = yOffset; 5119 mInfo = info; 5120 5121 mBufferContext = 0; 5122 5123 // read media-info. See MediaImage2 5124 if (info.remaining() == 104) { 5125 int type = info.getInt(); 5126 if (type != TYPE_YUV) { 5127 throw new UnsupportedOperationException("unsupported type: " + type); 5128 } 5129 int numPlanes = info.getInt(); 5130 if (numPlanes != 3) { 5131 throw new RuntimeException("unexpected number of planes: " + numPlanes); 5132 } 5133 mWidth = info.getInt(); 5134 mHeight = info.getInt(); 5135 if (mWidth < 1 || mHeight < 1) { 5136 throw new UnsupportedOperationException( 5137 "unsupported size: " + mWidth + "x" + mHeight); 5138 } 5139 int bitDepth = info.getInt(); 5140 if (bitDepth != 8) { 5141 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 5142 } 5143 int bitDepthAllocated = info.getInt(); 5144 if (bitDepthAllocated != 8) { 5145 throw new UnsupportedOperationException( 5146 "unsupported allocated bit depth: " + bitDepthAllocated); 5147 } 5148 mPlanes = new MediaPlane[numPlanes]; 5149 for (int ix = 0; ix < numPlanes; ix++) { 5150 int planeOffset = info.getInt(); 5151 int colInc = info.getInt(); 5152 int rowInc = info.getInt(); 5153 int horiz = info.getInt(); 5154 int vert = info.getInt(); 5155 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 5156 throw new UnsupportedOperationException("unexpected subsampling: " 5157 + horiz + "x" + vert + " on plane " + ix); 5158 } 5159 if (colInc < 1 || rowInc < 1) { 5160 throw new UnsupportedOperationException("unexpected strides: " 5161 + colInc + " pixel, " + rowInc + " row on plane " + ix); 5162 } 5163 buffer.clear(); 5164 buffer.position(mBuffer.position() + planeOffset 5165 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 5166 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 5167 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 5168 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 5169 } 5170 } else { 5171 throw new UnsupportedOperationException( 5172 "unsupported info length: " + info.remaining()); 5173 } 5174 5175 if (cropRect == null) { 5176 cropRect = new Rect(0, 0, mWidth, mHeight); 5177 } 5178 cropRect.offset(-xOffset, -yOffset); 5179 super.setCropRect(cropRect); 5180 } 5181 MediaImage( @onNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, int width, int height, int format, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context)5182 public MediaImage( 5183 @NonNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, 5184 int width, int height, int format, boolean readOnly, 5185 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context) { 5186 if (buffers.length != rowStrides.length || buffers.length != pixelStrides.length) { 5187 throw new IllegalArgumentException( 5188 "buffers, rowStrides and pixelStrides should have the same length"); 5189 } 5190 mWidth = width; 5191 mHeight = height; 5192 mFormat = format; 5193 mTimestamp = timestamp; 5194 mIsImageValid = true; 5195 mIsReadOnly = readOnly; 5196 mBuffer = null; 5197 mInfo = null; 5198 mPlanes = new MediaPlane[buffers.length]; 5199 for (int i = 0; i < buffers.length; ++i) { 5200 mPlanes[i] = new MediaPlane(buffers[i], rowStrides[i], pixelStrides[i]); 5201 } 5202 5203 // save offsets and info 5204 mXOffset = xOffset; 5205 mYOffset = yOffset; 5206 5207 if (cropRect == null) { 5208 cropRect = new Rect(0, 0, mWidth, mHeight); 5209 } 5210 cropRect.offset(-xOffset, -yOffset); 5211 super.setCropRect(cropRect); 5212 5213 mBufferContext = context; 5214 } 5215 5216 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)5217 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 5218 mData = buffer; 5219 mRowInc = rowInc; 5220 mColInc = colInc; 5221 } 5222 5223 @Override getRowStride()5224 public int getRowStride() { 5225 throwISEIfImageIsInvalid(); 5226 return mRowInc; 5227 } 5228 5229 @Override getPixelStride()5230 public int getPixelStride() { 5231 throwISEIfImageIsInvalid(); 5232 return mColInc; 5233 } 5234 5235 @Override 5236 @NonNull getBuffer()5237 public ByteBuffer getBuffer() { 5238 throwISEIfImageIsInvalid(); 5239 return mData; 5240 } 5241 5242 private final int mRowInc; 5243 private final int mColInc; 5244 private final ByteBuffer mData; 5245 } 5246 } 5247 5248 public final static class MetricsConstants 5249 { MetricsConstants()5250 private MetricsConstants() {} 5251 5252 /** 5253 * Key to extract the codec being used 5254 * from the {@link MediaCodec#getMetrics} return value. 5255 * The value is a String. 5256 */ 5257 public static final String CODEC = "android.media.mediacodec.codec"; 5258 5259 /** 5260 * Key to extract the MIME type 5261 * from the {@link MediaCodec#getMetrics} return value. 5262 * The value is a String. 5263 */ 5264 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 5265 5266 /** 5267 * Key to extract what the codec mode 5268 * from the {@link MediaCodec#getMetrics} return value. 5269 * The value is a String. Values will be one of the constants 5270 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 5271 */ 5272 public static final String MODE = "android.media.mediacodec.mode"; 5273 5274 /** 5275 * The value returned for the key {@link #MODE} when the 5276 * codec is a audio codec. 5277 */ 5278 public static final String MODE_AUDIO = "audio"; 5279 5280 /** 5281 * The value returned for the key {@link #MODE} when the 5282 * codec is a video codec. 5283 */ 5284 public static final String MODE_VIDEO = "video"; 5285 5286 /** 5287 * Key to extract the flag indicating whether the codec is running 5288 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 5289 * The value is an integer. 5290 * A 0 indicates decoder; 1 indicates encoder. 5291 */ 5292 public static final String ENCODER = "android.media.mediacodec.encoder"; 5293 5294 /** 5295 * Key to extract the flag indicating whether the codec is running 5296 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 5297 * The value is an integer. 5298 */ 5299 public static final String SECURE = "android.media.mediacodec.secure"; 5300 5301 /** 5302 * Key to extract the width (in pixels) of the video track 5303 * from the {@link MediaCodec#getMetrics} return value. 5304 * The value is an integer. 5305 */ 5306 public static final String WIDTH = "android.media.mediacodec.width"; 5307 5308 /** 5309 * Key to extract the height (in pixels) of the video track 5310 * from the {@link MediaCodec#getMetrics} return value. 5311 * The value is an integer. 5312 */ 5313 public static final String HEIGHT = "android.media.mediacodec.height"; 5314 5315 /** 5316 * Key to extract the rotation (in degrees) to properly orient the video 5317 * from the {@link MediaCodec#getMetrics} return. 5318 * The value is a integer. 5319 */ 5320 public static final String ROTATION = "android.media.mediacodec.rotation"; 5321 5322 } 5323 } 5324
