1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.Manifest; 20 import android.annotation.IntDef; 21 import android.annotation.NonNull; 22 import android.annotation.Nullable; 23 import android.annotation.RequiresPermission; 24 import android.annotation.SystemApi; 25 import android.compat.annotation.UnsupportedAppUsage; 26 import android.graphics.ImageFormat; 27 import android.graphics.Rect; 28 import android.graphics.SurfaceTexture; 29 import android.hardware.HardwareBuffer; 30 import android.media.MediaCodecInfo.CodecCapabilities; 31 import android.os.Build; 32 import android.os.Bundle; 33 import android.os.Handler; 34 import android.os.IHwBinder; 35 import android.os.Looper; 36 import android.os.Message; 37 import android.os.PersistableBundle; 38 import android.view.Surface; 39 40 import java.io.IOException; 41 import java.lang.annotation.Retention; 42 import java.lang.annotation.RetentionPolicy; 43 import java.nio.ByteBuffer; 44 import java.nio.ByteOrder; 45 import java.nio.ReadOnlyBufferException; 46 import java.util.ArrayList; 47 import java.util.Arrays; 48 import java.util.BitSet; 49 import java.util.Collections; 50 import java.util.HashMap; 51 import java.util.HashSet; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Objects; 55 import java.util.Set; 56 import java.util.concurrent.BlockingQueue; 57 import java.util.concurrent.LinkedBlockingQueue; 58 import java.util.concurrent.locks.Lock; 59 import java.util.concurrent.locks.ReentrantLock; 60 61 /** 62 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 63 It is part of the Android low-level multimedia support infrastructure (normally used together 64 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 65 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 66 <p> 67 <center> 68 <img src="../../../images/media/mediacodec_buffers.svg" style="width: 540px; height: 205px" 69 alt="MediaCodec buffer flow diagram"> 70 </center> 71 <p> 72 In broad terms, a codec processes input data to generate output data. It processes data 73 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 74 (or receive) an empty input buffer, fill it up with data and send it to the codec for 75 processing. The codec uses up the data and transforms it into one of its empty output buffers. 76 Finally, you request (or receive) a filled output buffer, consume its contents and release it 77 back to the codec. 78 79 <h3 id=qualityFloor><a name="qualityFloor">Minimum Quality Floor for Video Encoding</h3> 80 <p> 81 Beginning with {@link android.os.Build.VERSION_CODES#S}, Android's Video MediaCodecs enforce a 82 minimum quality floor. The intent is to eliminate poor quality video encodings. This quality 83 floor is applied when the codec is in Variable Bitrate (VBR) mode; it is not applied when 84 the codec is in Constant Bitrate (CBR) mode. The quality floor enforcement is also restricted 85 to a particular size range; this size range is currently for video resolutions 86 larger than 320x240 up through 1920x1080. 87 88 <p> 89 When this quality floor is in effect, the codec and supporting framework code will work to 90 ensure that the generated video is of at least a "fair" or "good" quality. The metric 91 used to choose these targets is the VMAF (Video Multi-method Assessment Function) with a 92 target score of 70 for selected test sequences. 93 94 <p> 95 The typical effect is that 96 some videos will generate a higher bitrate than originally configured. This will be most 97 notable for videos which were configured with very low bitrates; the codec will use a bitrate 98 that is determined to be more likely to generate an "fair" or "good" quality video. Another 99 situation is where a video includes very complicated content (lots of motion and detail); 100 in such configurations, the codec will use extra bitrate as needed to avoid losing all of 101 the content's finer detail. 102 103 <p> 104 This quality floor will not impact content captured at high bitrates (a high bitrate should 105 already provide the codec with sufficient capacity to encode all of the detail). 106 The quality floor does not operate on CBR encodings. 107 The quality floor currently does not operate on resolutions of 320x240 or lower, nor on 108 videos with resolution above 1920x1080. 109 110 <h3>Data Types</h3> 111 <p> 112 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 113 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 114 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 115 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 116 You normally cannot access the raw video data when using a Surface, but you can use the 117 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 118 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 119 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 120 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 121 OutputImage(int)}. 122 123 <h4>Compressed Buffers</h4> 124 <p> 125 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 126 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 127 compressed video frame. For audio data this is normally a single access unit (an encoded audio 128 segment typically containing a few milliseconds of audio as dictated by the format type), but 129 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 130 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 131 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 132 133 <h4>Raw Audio Buffers</h4> 134 <p> 135 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 136 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 137 in native byte order. 138 Raw audio buffers in the float PCM encoding are only possible 139 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 140 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 141 {@link #configure configure(…)} 142 and confirmed by {@link #getOutputFormat} for decoders 143 or {@link #getInputFormat} for encoders. 144 A sample method to check for float PCM in the MediaFormat is as follows: 145 146 <pre class=prettyprint> 147 static boolean isPcmFloat(MediaFormat format) { 148 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 149 == AudioFormat.ENCODING_PCM_FLOAT; 150 }</pre> 151 152 In order to extract, in a short array, 153 one channel of a buffer containing 16 bit signed integer audio data, 154 the following code may be used: 155 156 <pre class=prettyprint> 157 // Assumes the buffer PCM encoding is 16 bit. 158 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 159 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 160 MediaFormat format = codec.getOutputFormat(bufferId); 161 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 162 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 163 if (channelIx < 0 || channelIx >= numChannels) { 164 return null; 165 } 166 short[] res = new short[samples.remaining() / numChannels]; 167 for (int i = 0; i < res.length; ++i) { 168 res[i] = samples.get(i * numChannels + channelIx); 169 } 170 return res; 171 }</pre> 172 173 <h4>Raw Video Buffers</h4> 174 <p> 175 In ByteBuffer mode video buffers are laid out according to their {@linkplain 176 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 177 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 178 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 179 Video codecs may support three kinds of color formats: 180 <ul> 181 <li><strong>native raw video format:</strong> This is marked by {@link 182 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 183 <li><strong>flexible YUV buffers</strong> (such as {@link 184 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 185 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 186 OutputImage(int)}.</li> 187 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 188 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 189 For color formats that are equivalent to a flexible format, you can still use {@link 190 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 191 </ul> 192 <p> 193 All video codecs support flexible YUV 4:2:0 buffers since {@link 194 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 195 196 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 197 <p> 198 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 199 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 200 values to understand the layout of the raw output buffers. 201 <p class=note> 202 Note that on some devices the slice-height is advertised as 0. This could mean either that the 203 slice-height is the same as the frame height, or that the slice-height is the frame height 204 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 205 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 206 plane in planar formats is also not specified or defined, though usually it is half of the slice 207 height. 208 <p> 209 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 210 video frames; however, for most encondings the video (picture) only occupies a portion of the 211 video frame. This is represented by the 'crop rectangle'. 212 <p> 213 You need to use the following keys to get the crop rectangle of raw output images from the 214 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 215 entire video frame.The crop rectangle is understood in the context of the output frame 216 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 217 <table style="width: 0%"> 218 <thead> 219 <tr> 220 <th>Format Key</th> 221 <th>Type</th> 222 <th>Description</th> 223 </tr> 224 </thead> 225 <tbody> 226 <tr> 227 <td>{@link MediaFormat#KEY_CROP_LEFT}</td> 228 <td>Integer</td> 229 <td>The left-coordinate (x) of the crop rectangle</td> 230 </tr><tr> 231 <td>{@link MediaFormat#KEY_CROP_TOP}</td> 232 <td>Integer</td> 233 <td>The top-coordinate (y) of the crop rectangle</td> 234 </tr><tr> 235 <td>{@link MediaFormat#KEY_CROP_RIGHT}</td> 236 <td>Integer</td> 237 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 238 </tr><tr> 239 <td>{@link MediaFormat#KEY_CROP_BOTTOM}</td> 240 <td>Integer</td> 241 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 242 </tr><tr> 243 <td colspan=3> 244 The right and bottom coordinates can be understood as the coordinates of the right-most 245 valid column/bottom-most valid row of the cropped output image. 246 </td> 247 </tr> 248 </tbody> 249 </table> 250 <p> 251 The size of the video frame (before rotation) can be calculated as such: 252 <pre class=prettyprint> 253 MediaFormat format = decoder.getOutputFormat(…); 254 int width = format.getInteger(MediaFormat.KEY_WIDTH); 255 if (format.containsKey(MediaFormat.KEY_CROP_LEFT) 256 && format.containsKey(MediaFormat.KEY_CROP_RIGHT)) { 257 width = format.getInteger(MediaFormat.KEY_CROP_RIGHT) + 1 258 - format.getInteger(MediaFormat.KEY_CROP_LEFT); 259 } 260 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 261 if (format.containsKey(MediaFormat.KEY_CROP_TOP) 262 && format.containsKey(MediaFormat.KEY_CROP_BOTTOM)) { 263 height = format.getInteger(MediaFormat.KEY_CROP_BOTTOM) + 1 264 - format.getInteger(MediaFormat.KEY_CROP_TOP); 265 } 266 </pre> 267 <p class=note> 268 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 269 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 270 most devices it pointed to the top-left pixel of the entire frame. 271 272 <h3>States</h3> 273 <p> 274 During its life a codec conceptually exists in one of three states: Stopped, Executing or 275 Released. The Stopped collective state is actually the conglomeration of three states: 276 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 277 three sub-states: Flushed, Running and End-of-Stream. 278 <p> 279 <center> 280 <img src="../../../images/media/mediacodec_states.svg" style="width: 519px; height: 356px" 281 alt="MediaCodec state diagram"> 282 </center> 283 <p> 284 When you create a codec using one of the factory methods, the codec is in the Uninitialized 285 state. First, you need to configure it via {@link #configure configure(…)}, which brings 286 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 287 state you can process data through the buffer queue manipulation described above. 288 <p> 289 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 290 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 291 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 292 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 293 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 294 codec no longer accepts further input buffers, but still generates output buffers until the 295 end-of-stream is reached on the output. For decoders, you can move back to the Flushed sub-state 296 at any time while in the Executing state using {@link #flush}. 297 <p class=note> 298 <strong>Note:</strong> Going back to Flushed state is only supported for decoders, and may not 299 work for encoders (the behavior is undefined). 300 <p> 301 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 302 again. When you are done using a codec, you must release it by calling {@link #release}. 303 <p> 304 On rare occasions the codec may encounter an error and move to the Error state. This is 305 communicated using an invalid return value from a queuing operation, or sometimes via an 306 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 307 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 308 terminal Released state. 309 310 <h3>Creation</h3> 311 <p> 312 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 313 decoding a file or a stream, you can get the desired format from {@link 314 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 315 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 316 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 317 name of a codec that can handle that specific media format. Finally, create the codec using 318 {@link #createByCodecName}. 319 <p class=note> 320 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 321 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 322 MediaFormat#KEY_FRAME_RATE frame rate}. Use 323 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 324 to clear any existing frame rate setting in the format. 325 <p> 326 You can also create the preferred codec for a specific MIME type using {@link 327 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 328 This, however, cannot be used to inject features, and may create a codec that cannot handle the 329 specific desired media format. 330 331 <h4>Creating secure decoders</h4> 332 <p> 333 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 334 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 335 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 336 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 337 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 338 <p> 339 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 340 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 341 342 <h3>Initialization</h3> 343 <p> 344 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 345 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 346 specific media format. This is when you can specify the output {@link Surface} for video 347 producers – codecs that generate raw video data (e.g. video decoders). This is also when 348 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 349 some codecs can operate in multiple modes, you must specify whether you want it to work as a 350 decoder or an encoder. 351 <p> 352 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 353 output format in the Configured state. You can use this to verify the resulting configuration, 354 e.g. color formats, before starting the codec. 355 <p> 356 If you want to process raw input video buffers natively with a video consumer – a codec 357 that processes raw video input, such as a video encoder – create a destination Surface for 358 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 359 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 360 surface} by calling {@link #setInputSurface}. 361 362 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 363 <p> 364 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 365 to be prefixed by a number of buffers containing setup data, or codec specific data. When 366 processing such compressed formats, this data must be submitted to the codec after {@link 367 #start} and before any frame data. Such data must be marked using the flag {@link 368 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 369 <p> 370 Codec-specific data can also be included in the format passed to {@link #configure configure} in 371 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 372 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 373 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 374 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 375 specific data, you can choose to submit it using the specified number of buffers in the correct 376 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 377 codec-specific data and submit it as a single codec-config buffer. 378 <p> 379 Android uses the following codec-specific data buffers. These are also required to be set in 380 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 381 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 382 {@code "\x00\x00\x00\x01"}. 383 <p> 384 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 385 <table> 386 <thead> 387 <th>Format</th> 388 <th>CSD buffer #0</th> 389 <th>CSD buffer #1</th> 390 <th>CSD buffer #2</th> 391 </thead> 392 <tbody class=mid> 393 <tr> 394 <td>AAC</td> 395 <td>Decoder-specific information from ESDS<sup>*</sup></td> 396 <td class=NA>Not Used</td> 397 <td class=NA>Not Used</td> 398 </tr> 399 <tr> 400 <td>VORBIS</td> 401 <td>Identification header</td> 402 <td>Setup header</td> 403 <td class=NA>Not Used</td> 404 </tr> 405 <tr> 406 <td>OPUS</td> 407 <td>Identification header</td> 408 <td>Pre-skip in nanosecs<br> 409 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 410 This overrides the pre-skip value in the identification header.</td> 411 <td>Seek Pre-roll in nanosecs<br> 412 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 413 </tr> 414 <tr> 415 <td>FLAC</td> 416 <td>"fLaC", the FLAC stream marker in ASCII,<br> 417 followed by the STREAMINFO block (the mandatory metadata block),<br> 418 optionally followed by any number of other metadata blocks</td> 419 <td class=NA>Not Used</td> 420 <td class=NA>Not Used</td> 421 </tr> 422 <tr> 423 <td>MPEG-4</td> 424 <td>Decoder-specific information from ESDS<sup>*</sup></td> 425 <td class=NA>Not Used</td> 426 <td class=NA>Not Used</td> 427 </tr> 428 <tr> 429 <td>H.264 AVC</td> 430 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 431 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 432 <td class=NA>Not Used</td> 433 </tr> 434 <tr> 435 <td>H.265 HEVC</td> 436 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 437 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 438 PPS (Picture Parameter Sets<sup>*</sup>)</td> 439 <td class=NA>Not Used</td> 440 <td class=NA>Not Used</td> 441 </tr> 442 <tr> 443 <td>VP9</td> 444 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 445 (optional)</td> 446 <td class=NA>Not Used</td> 447 <td class=NA>Not Used</td> 448 </tr> 449 </tbody> 450 </table> 451 452 <p class=note> 453 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 454 after start, before any output buffer or output format change has been returned, as the codec 455 specific data may be lost during the flush. You must resubmit the data using buffers marked with 456 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 457 <p> 458 Encoders (or codecs that generate compressed data) will create and return the codec specific data 459 before any valid output buffer in output buffers marked with the {@linkplain 460 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 461 meaningful timestamps. 462 463 <h3>Data Processing</h3> 464 <p> 465 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 466 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 467 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 468 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 469 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 470 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 471 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 472 <p> 473 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 474 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 475 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 476 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 477 <p> 478 The codec in turn will return a read-only output buffer via the {@link 479 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 480 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 481 output buffer has been processed, call one of the {@link #releaseOutputBuffer 482 releaseOutputBuffer} methods to return the buffer to the codec. 483 <p> 484 While you are not required to resubmit/release buffers immediately to the codec, holding onto 485 input and/or output buffers may stall the codec, and this behavior is device dependent. 486 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 487 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 488 hold onto to available buffers as little as possible. 489 <p> 490 Depending on the API version, you can process data in three ways: 491 <table> 492 <thead> 493 <tr> 494 <th>Processing Mode</th> 495 <th>API version <= 20<br>Jelly Bean/KitKat</th> 496 <th>API version >= 21<br>Lollipop and later</th> 497 </tr> 498 </thead> 499 <tbody> 500 <tr> 501 <td>Synchronous API using buffer arrays</td> 502 <td>Supported</td> 503 <td>Deprecated</td> 504 </tr> 505 <tr> 506 <td>Synchronous API using buffers</td> 507 <td class=NA>Not Available</td> 508 <td>Supported</td> 509 </tr> 510 <tr> 511 <td>Asynchronous API using buffers</td> 512 <td class=NA>Not Available</td> 513 <td>Supported</td> 514 </tr> 515 </tbody> 516 </table> 517 518 <h4>Asynchronous Processing using Buffers</h4> 519 <p> 520 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 521 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 522 mode changes the state transitions slightly, because you must call {@link #start} after {@link 523 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 524 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 525 sub-state and start passing available input buffers via the callback. 526 <p> 527 <center> 528 <img src="../../../images/media/mediacodec_async_states.svg" style="width: 516px; height: 353px" 529 alt="MediaCodec state diagram for asynchronous operation"> 530 </center> 531 <p> 532 MediaCodec is typically used like this in asynchronous mode: 533 <pre class=prettyprint> 534 MediaCodec codec = MediaCodec.createByCodecName(name); 535 MediaFormat mOutputFormat; // member variable 536 codec.setCallback(new MediaCodec.Callback() { 537 {@literal @Override} 538 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 539 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 540 // fill inputBuffer with valid data 541 … 542 codec.queueInputBuffer(inputBufferId, …); 543 } 544 545 {@literal @Override} 546 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 547 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 548 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 549 // bufferFormat is equivalent to mOutputFormat 550 // outputBuffer is ready to be processed or rendered. 551 … 552 codec.releaseOutputBuffer(outputBufferId, …); 553 } 554 555 {@literal @Override} 556 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 557 // Subsequent data will conform to new format. 558 // Can ignore if using getOutputFormat(outputBufferId) 559 mOutputFormat = format; // option B 560 } 561 562 {@literal @Override} 563 void onError(…) { 564 … 565 } 566 }); 567 codec.configure(format, …); 568 mOutputFormat = codec.getOutputFormat(); // option B 569 codec.start(); 570 // wait for processing to complete 571 codec.stop(); 572 codec.release();</pre> 573 574 <h4>Synchronous Processing using Buffers</h4> 575 <p> 576 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 577 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 578 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 579 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 580 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 581 getInput}/{@link #getOutputBuffers OutputBuffers()}. 582 583 <p class=note> 584 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 585 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 586 #start} or after having dequeued an output buffer ID with the value of {@link 587 #INFO_OUTPUT_FORMAT_CHANGED}. 588 <p> 589 MediaCodec is typically used like this in synchronous mode: 590 <pre> 591 MediaCodec codec = MediaCodec.createByCodecName(name); 592 codec.configure(format, …); 593 MediaFormat outputFormat = codec.getOutputFormat(); // option B 594 codec.start(); 595 for (;;) { 596 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 597 if (inputBufferId >= 0) { 598 ByteBuffer inputBuffer = codec.getInputBuffer(…); 599 // fill inputBuffer with valid data 600 … 601 codec.queueInputBuffer(inputBufferId, …); 602 } 603 int outputBufferId = codec.dequeueOutputBuffer(…); 604 if (outputBufferId >= 0) { 605 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 606 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 607 // bufferFormat is identical to outputFormat 608 // outputBuffer is ready to be processed or rendered. 609 … 610 codec.releaseOutputBuffer(outputBufferId, …); 611 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 612 // Subsequent data will conform to new format. 613 // Can ignore if using getOutputFormat(outputBufferId) 614 outputFormat = codec.getOutputFormat(); // option B 615 } 616 } 617 codec.stop(); 618 codec.release();</pre> 619 620 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 621 <p> 622 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 623 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 624 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 625 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 626 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 627 between the size of the arrays and the number of input and output buffers used by the system, 628 although the array size provides an upper bound. 629 <pre> 630 MediaCodec codec = MediaCodec.createByCodecName(name); 631 codec.configure(format, …); 632 codec.start(); 633 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 634 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 635 for (;;) { 636 int inputBufferId = codec.dequeueInputBuffer(…); 637 if (inputBufferId >= 0) { 638 // fill inputBuffers[inputBufferId] with valid data 639 … 640 codec.queueInputBuffer(inputBufferId, …); 641 } 642 int outputBufferId = codec.dequeueOutputBuffer(…); 643 if (outputBufferId >= 0) { 644 // outputBuffers[outputBufferId] is ready to be processed or rendered. 645 … 646 codec.releaseOutputBuffer(outputBufferId, …); 647 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 648 outputBuffers = codec.getOutputBuffers(); 649 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 650 // Subsequent data will conform to new format. 651 MediaFormat format = codec.getOutputFormat(); 652 } 653 } 654 codec.stop(); 655 codec.release();</pre> 656 657 <h4>End-of-stream Handling</h4> 658 <p> 659 When you reach the end of the input data, you must signal it to the codec by specifying the 660 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 661 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 662 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 663 be ignored. 664 <p> 665 The codec will continue to return output buffers until it eventually signals the end of the 666 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 667 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 668 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 669 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 670 <p> 671 Do not submit additional input buffers after signaling the end of the input stream, unless the 672 codec has been flushed, or stopped and restarted. 673 674 <h4>Using an Output Surface</h4> 675 <p> 676 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 677 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 678 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 679 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 680 null}-s. 681 <p> 682 When using an output Surface, you can select whether or not to render each output buffer on the 683 surface. You have three choices: 684 <ul> 685 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 686 releaseOutputBuffer(bufferId, false)}.</li> 687 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 688 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 689 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 690 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 691 </ul> 692 <p> 693 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 694 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 695 It was not defined prior to that. 696 <p> 697 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 698 dynamically using {@link #setOutputSurface setOutputSurface}. 699 <p> 700 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 701 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 702 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 703 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 704 was undefined, with the exception that View surfaces (SurfaceView or TextureView) always dropped 705 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 706 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 707 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 708 setting the key {@link MediaFormat#KEY_ALLOW_FRAME_DROP} to {@code 0} 709 in their configure format. 710 711 <h4>Transformations When Rendering onto Surface</h4> 712 713 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 714 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 715 mode} will be automatically applied with one exception: 716 <p class=note> 717 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 718 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 719 and simple way to identify software decoders, or if they apply the rotation other than by trying 720 it out. 721 <p> 722 There are also some caveats. 723 <p class=note> 724 Note that the pixel aspect ratio is not considered when displaying the output onto the 725 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 726 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 727 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 728 square pixels (pixel aspect ratio or 1:1). 729 <p class=note> 730 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 731 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 732 by 90 or 270 degrees. 733 <p class=note> 734 When setting the video scaling mode, note that it must be reset after each time the output 735 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 736 do this after each time the output format changes. 737 738 <h4>Using an Input Surface</h4> 739 <p> 740 When using an input Surface, there are no accessible input buffers, as buffers are automatically 741 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 742 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 743 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 744 <p> 745 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 746 submitting data to the codec immediately after this call. 747 <p> 748 749 <h3>Seeking & Adaptive Playback Support</h3> 750 <p> 751 Video decoders (and in general codecs that consume compressed video data) behave differently 752 regarding seek and format change whether or not they support and are configured for adaptive 753 playback. You can check if a decoder supports {@linkplain 754 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 755 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 756 playback support for video decoders is only activated if you configure the codec to decode onto a 757 {@link Surface}. 758 759 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 760 <p> 761 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 762 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 763 completely on its own (for most codecs this means an I-frame), and no frames that are to be 764 displayed after a key frame refer to frames before the key frame. 765 <p> 766 The following table summarizes suitable key frames for various video formats. 767 <table> 768 <thead> 769 <tr> 770 <th>Format</th> 771 <th>Suitable key frame</th> 772 </tr> 773 </thead> 774 <tbody class=mid> 775 <tr> 776 <td>VP9/VP8</td> 777 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 778 <i>(There is no specific name for such key frame.)</i></td> 779 </tr> 780 <tr> 781 <td>H.265 HEVC</td> 782 <td>IDR or CRA</td> 783 </tr> 784 <tr> 785 <td>H.264 AVC</td> 786 <td>IDR</td> 787 </tr> 788 <tr> 789 <td>MPEG-4<br>H.263<br>MPEG-2</td> 790 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 791 <i>(There is no specific name for such key frame.)</td> 792 </tr> 793 </tbody> 794 </table> 795 796 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 797 Surface)</h4> 798 <p> 799 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 800 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 801 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 802 before you call {@code flush}. It is important that the input data after a flush starts at a 803 suitable stream boundary/key frame. 804 <p class=note> 805 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 806 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 807 #configure configure(…)} - {@link #start} cycle is necessary. 808 809 <p class=note> 810 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 811 generally, before the first output buffer or output format change is received – you 812 will need to resubmit the codec-specific-data to the codec. See the <a 813 href="#CSD">codec-specific-data section</a> for more info. 814 815 <h4>For decoders that support and are configured for adaptive playback</h4> 816 <p> 817 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 818 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 819 discontinuity must start at a suitable stream boundary/key frame. 820 <p> 821 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 822 picture size or configuration mid-stream. To do this you must package the entire new 823 codec-specific configuration data together with the key frame into a single buffer (including 824 any start codes), and submit it as a <strong>regular</strong> input buffer. 825 <p> 826 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 827 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 828 onOutputFormatChanged} callback just after the picture-size change takes place and before any 829 frames with the new size have been returned. 830 <p class=note> 831 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 832 {@link #flush} shortly after you have changed the picture size. If you have not received 833 confirmation of the picture size change, you will need to repeat the request for the new picture 834 size. 835 836 <h3>Error handling</h3> 837 <p> 838 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 839 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 840 which you must catch or declare to pass up. MediaCodec methods throw {@code 841 IllegalStateException} when the method is called from a codec state that does not allow it; this 842 is typically due to incorrect application API usage. Methods involving secure buffers may throw 843 {@link CryptoException}, which has further error information obtainable from {@link 844 CryptoException#getErrorCode}. 845 <p> 846 Internal codec errors result in a {@link CodecException}, which may be due to media content 847 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 848 correctly using the API. The recommended action when receiving a {@code CodecException} 849 can be determined by calling {@link CodecException#isRecoverable} and {@link 850 CodecException#isTransient}: 851 <ul> 852 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 853 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 854 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 855 temporarily unavailable and the method may be retried at a later time.</li> 856 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 857 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 858 reset} or {@linkplain #release released}.</li> 859 </ul> 860 <p> 861 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 862 863 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 864 <p> 865 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 866 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 867 868 <style> 869 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 870 .api > tr > th { vertical-align: bottom; } 871 .api > tr > td { vertical-align: middle; } 872 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 873 .fn { text-align: left; } 874 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 875 .deg45 { 876 white-space: nowrap; background: none; border: none; vertical-align: bottom; 877 width: 30px; height: 83px; 878 } 879 .deg45 > div { 880 transform: skew(-45deg, 0deg) translate(1px, -67px); 881 transform-origin: bottom left 0; 882 width: 30px; height: 20px; 883 } 884 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 885 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 886 </style> 887 888 <table align="right" style="width: 0%"> 889 <thead> 890 <tr><th>Symbol</th><th>Meaning</th></tr> 891 </thead> 892 <tbody class=sml> 893 <tr><td>●</td><td>Supported</td></tr> 894 <tr><td>⁕</td><td>Semantics changed</td></tr> 895 <tr><td>○</td><td>Experimental support</td></tr> 896 <tr><td>[ ]</td><td>Deprecated</td></tr> 897 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 898 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 899 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 900 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 901 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 902 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 903 </tbody> 904 </table> 905 906 <table style="width: 100%;"> 907 <thead class=api> 908 <tr> 909 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 910 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 911 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 912 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 913 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 914 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 915 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 916 <th></th> 917 <th colspan="8">SDK Version</th> 918 </tr> 919 <tr> 920 <th colspan="7">State</th> 921 <th>Method</th> 922 <th>16</th> 923 <th>17</th> 924 <th>18</th> 925 <th>19</th> 926 <th>20</th> 927 <th>21</th> 928 <th>22</th> 929 <th>23</th> 930 </tr> 931 </thead> 932 <tbody class=api> 933 <tr> 934 <td></td> 935 <td></td> 936 <td></td> 937 <td></td> 938 <td></td> 939 <td></td> 940 <td></td> 941 <td class=fn>{@link #createByCodecName createByCodecName}</td> 942 <td>●</td> 943 <td>●</td> 944 <td>●</td> 945 <td>●</td> 946 <td>●</td> 947 <td>●</td> 948 <td>●</td> 949 <td>●</td> 950 </tr> 951 <tr> 952 <td></td> 953 <td></td> 954 <td></td> 955 <td></td> 956 <td></td> 957 <td></td> 958 <td></td> 959 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 960 <td>●</td> 961 <td>●</td> 962 <td>●</td> 963 <td>●</td> 964 <td>●</td> 965 <td>●</td> 966 <td>●</td> 967 <td>●</td> 968 </tr> 969 <tr> 970 <td></td> 971 <td></td> 972 <td></td> 973 <td></td> 974 <td></td> 975 <td></td> 976 <td></td> 977 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 978 <td>●</td> 979 <td>●</td> 980 <td>●</td> 981 <td>●</td> 982 <td>●</td> 983 <td>●</td> 984 <td>●</td> 985 <td>●</td> 986 </tr> 987 <tr> 988 <td></td> 989 <td></td> 990 <td></td> 991 <td></td> 992 <td></td> 993 <td></td> 994 <td></td> 995 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 996 <td></td> 997 <td></td> 998 <td></td> 999 <td></td> 1000 <td></td> 1001 <td></td> 1002 <td></td> 1003 <td>●</td> 1004 </tr> 1005 <tr> 1006 <td>16+</td> 1007 <td>-</td> 1008 <td>-</td> 1009 <td>-</td> 1010 <td>-</td> 1011 <td>-</td> 1012 <td>-</td> 1013 <td class=fn>{@link #configure configure}</td> 1014 <td>●</td> 1015 <td>●</td> 1016 <td>●</td> 1017 <td>●</td> 1018 <td>●</td> 1019 <td>⁕</td> 1020 <td>●</td> 1021 <td>●</td> 1022 </tr> 1023 <tr> 1024 <td>-</td> 1025 <td>18+</td> 1026 <td>-</td> 1027 <td>-</td> 1028 <td>-</td> 1029 <td>-</td> 1030 <td>-</td> 1031 <td class=fn>{@link #createInputSurface createInputSurface}</td> 1032 <td></td> 1033 <td></td> 1034 <td>⎋</td> 1035 <td>⎋</td> 1036 <td>⎋</td> 1037 <td>⎋</td> 1038 <td>⎋</td> 1039 <td>⎋</td> 1040 </tr> 1041 <tr> 1042 <td>-</td> 1043 <td>-</td> 1044 <td>16+</td> 1045 <td>16+</td> 1046 <td>(16+)</td> 1047 <td>-</td> 1048 <td>-</td> 1049 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 1050 <td>●</td> 1051 <td>●</td> 1052 <td>▧</td> 1053 <td>▧</td> 1054 <td>▧</td> 1055 <td>⁕▧↩</td> 1056 <td>▧↩</td> 1057 <td>▧↩</td> 1058 </tr> 1059 <tr> 1060 <td>-</td> 1061 <td>-</td> 1062 <td>16+</td> 1063 <td>16+</td> 1064 <td>16+</td> 1065 <td>-</td> 1066 <td>-</td> 1067 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1068 <td>●</td> 1069 <td>●</td> 1070 <td>●</td> 1071 <td>●</td> 1072 <td>●</td> 1073 <td>⁕↩</td> 1074 <td>↩</td> 1075 <td>↩</td> 1076 </tr> 1077 <tr> 1078 <td>-</td> 1079 <td>-</td> 1080 <td>16+</td> 1081 <td>16+</td> 1082 <td>16+</td> 1083 <td>-</td> 1084 <td>-</td> 1085 <td class=fn>{@link #flush flush}</td> 1086 <td>●</td> 1087 <td>●</td> 1088 <td>●</td> 1089 <td>●</td> 1090 <td>●</td> 1091 <td>●</td> 1092 <td>●</td> 1093 <td>●</td> 1094 </tr> 1095 <tr> 1096 <td>18+</td> 1097 <td>18+</td> 1098 <td>18+</td> 1099 <td>18+</td> 1100 <td>18+</td> 1101 <td>18+</td> 1102 <td>-</td> 1103 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1104 <td></td> 1105 <td></td> 1106 <td>●</td> 1107 <td>●</td> 1108 <td>●</td> 1109 <td>●</td> 1110 <td>●</td> 1111 <td>●</td> 1112 </tr> 1113 <tr> 1114 <td>-</td> 1115 <td>-</td> 1116 <td>(21+)</td> 1117 <td>21+</td> 1118 <td>(21+)</td> 1119 <td>-</td> 1120 <td>-</td> 1121 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1122 <td></td> 1123 <td></td> 1124 <td></td> 1125 <td></td> 1126 <td></td> 1127 <td>●</td> 1128 <td>●</td> 1129 <td>●</td> 1130 </tr> 1131 <tr> 1132 <td>-</td> 1133 <td>-</td> 1134 <td>16+</td> 1135 <td>(16+)</td> 1136 <td>(16+)</td> 1137 <td>-</td> 1138 <td>-</td> 1139 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1140 <td>●</td> 1141 <td>●</td> 1142 <td>●</td> 1143 <td>●</td> 1144 <td>●</td> 1145 <td>[⁕↩]</td> 1146 <td>[↩]</td> 1147 <td>[↩]</td> 1148 </tr> 1149 <tr> 1150 <td>-</td> 1151 <td>21+</td> 1152 <td>(21+)</td> 1153 <td>(21+)</td> 1154 <td>(21+)</td> 1155 <td>-</td> 1156 <td>-</td> 1157 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1158 <td></td> 1159 <td></td> 1160 <td></td> 1161 <td></td> 1162 <td></td> 1163 <td>●</td> 1164 <td>●</td> 1165 <td>●</td> 1166 </tr> 1167 <tr> 1168 <td>-</td> 1169 <td>-</td> 1170 <td>(21+)</td> 1171 <td>21+</td> 1172 <td>(21+)</td> 1173 <td>-</td> 1174 <td>-</td> 1175 <td class=fn>{@link #getInputImage getInputImage}</td> 1176 <td></td> 1177 <td></td> 1178 <td></td> 1179 <td></td> 1180 <td></td> 1181 <td>○</td> 1182 <td>●</td> 1183 <td>●</td> 1184 </tr> 1185 <tr> 1186 <td>18+</td> 1187 <td>18+</td> 1188 <td>18+</td> 1189 <td>18+</td> 1190 <td>18+</td> 1191 <td>18+</td> 1192 <td>-</td> 1193 <td class=fn>{@link #getName getName}</td> 1194 <td></td> 1195 <td></td> 1196 <td>●</td> 1197 <td>●</td> 1198 <td>●</td> 1199 <td>●</td> 1200 <td>●</td> 1201 <td>●</td> 1202 </tr> 1203 <tr> 1204 <td>-</td> 1205 <td>-</td> 1206 <td>(21+)</td> 1207 <td>21+</td> 1208 <td>21+</td> 1209 <td>-</td> 1210 <td>-</td> 1211 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1212 <td></td> 1213 <td></td> 1214 <td></td> 1215 <td></td> 1216 <td></td> 1217 <td>●</td> 1218 <td>●</td> 1219 <td>●</td> 1220 </tr> 1221 <tr> 1222 <td>-</td> 1223 <td>-</td> 1224 <td>16+</td> 1225 <td>16+</td> 1226 <td>16+</td> 1227 <td>-</td> 1228 <td>-</td> 1229 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1230 <td>●</td> 1231 <td>●</td> 1232 <td>●</td> 1233 <td>●</td> 1234 <td>●</td> 1235 <td>[⁕↩]</td> 1236 <td>[↩]</td> 1237 <td>[↩]</td> 1238 </tr> 1239 <tr> 1240 <td>-</td> 1241 <td>21+</td> 1242 <td>16+</td> 1243 <td>16+</td> 1244 <td>16+</td> 1245 <td>-</td> 1246 <td>-</td> 1247 <td class=fn>{@link #getOutputFormat()}</td> 1248 <td>●</td> 1249 <td>●</td> 1250 <td>●</td> 1251 <td>●</td> 1252 <td>●</td> 1253 <td>●</td> 1254 <td>●</td> 1255 <td>●</td> 1256 </tr> 1257 <tr> 1258 <td>-</td> 1259 <td>-</td> 1260 <td>(21+)</td> 1261 <td>21+</td> 1262 <td>21+</td> 1263 <td>-</td> 1264 <td>-</td> 1265 <td class=fn>{@link #getOutputFormat(int)}</td> 1266 <td></td> 1267 <td></td> 1268 <td></td> 1269 <td></td> 1270 <td></td> 1271 <td>●</td> 1272 <td>●</td> 1273 <td>●</td> 1274 </tr> 1275 <tr> 1276 <td>-</td> 1277 <td>-</td> 1278 <td>(21+)</td> 1279 <td>21+</td> 1280 <td>21+</td> 1281 <td>-</td> 1282 <td>-</td> 1283 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1284 <td></td> 1285 <td></td> 1286 <td></td> 1287 <td></td> 1288 <td></td> 1289 <td>○</td> 1290 <td>●</td> 1291 <td>●</td> 1292 </tr> 1293 <tr> 1294 <td>-</td> 1295 <td>-</td> 1296 <td>-</td> 1297 <td>16+</td> 1298 <td>(16+)</td> 1299 <td>-</td> 1300 <td>-</td> 1301 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1302 <td>●</td> 1303 <td>●</td> 1304 <td>●</td> 1305 <td>●</td> 1306 <td>●</td> 1307 <td>⁕</td> 1308 <td>●</td> 1309 <td>●</td> 1310 </tr> 1311 <tr> 1312 <td>-</td> 1313 <td>-</td> 1314 <td>-</td> 1315 <td>16+</td> 1316 <td>(16+)</td> 1317 <td>-</td> 1318 <td>-</td> 1319 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1320 <td>●</td> 1321 <td>●</td> 1322 <td>●</td> 1323 <td>●</td> 1324 <td>●</td> 1325 <td>⁕</td> 1326 <td>●</td> 1327 <td>●</td> 1328 </tr> 1329 <tr> 1330 <td>16+</td> 1331 <td>16+</td> 1332 <td>16+</td> 1333 <td>16+</td> 1334 <td>16+</td> 1335 <td>16+</td> 1336 <td>16+</td> 1337 <td class=fn>{@link #release release}</td> 1338 <td>●</td> 1339 <td>●</td> 1340 <td>●</td> 1341 <td>●</td> 1342 <td>●</td> 1343 <td>●</td> 1344 <td>●</td> 1345 <td>●</td> 1346 </tr> 1347 <tr> 1348 <td>-</td> 1349 <td>-</td> 1350 <td>-</td> 1351 <td>16+</td> 1352 <td>16+</td> 1353 <td>-</td> 1354 <td>-</td> 1355 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1356 <td>●</td> 1357 <td>●</td> 1358 <td>●</td> 1359 <td>●</td> 1360 <td>●</td> 1361 <td>⁕</td> 1362 <td>●</td> 1363 <td>⁕</td> 1364 </tr> 1365 <tr> 1366 <td>-</td> 1367 <td>-</td> 1368 <td>-</td> 1369 <td>21+</td> 1370 <td>21+</td> 1371 <td>-</td> 1372 <td>-</td> 1373 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1374 <td></td> 1375 <td></td> 1376 <td></td> 1377 <td></td> 1378 <td></td> 1379 <td>⎆</td> 1380 <td>⎆</td> 1381 <td>⎆</td> 1382 </tr> 1383 <tr> 1384 <td>21+</td> 1385 <td>21+</td> 1386 <td>21+</td> 1387 <td>21+</td> 1388 <td>21+</td> 1389 <td>21+</td> 1390 <td>-</td> 1391 <td class=fn>{@link #reset reset}</td> 1392 <td></td> 1393 <td></td> 1394 <td></td> 1395 <td></td> 1396 <td></td> 1397 <td>●</td> 1398 <td>●</td> 1399 <td>●</td> 1400 </tr> 1401 <tr> 1402 <td>21+</td> 1403 <td>-</td> 1404 <td>-</td> 1405 <td>-</td> 1406 <td>-</td> 1407 <td>-</td> 1408 <td>-</td> 1409 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1410 <td></td> 1411 <td></td> 1412 <td></td> 1413 <td></td> 1414 <td></td> 1415 <td>●</td> 1416 <td>●</td> 1417 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1418 </tr> 1419 <tr> 1420 <td>-</td> 1421 <td>23+</td> 1422 <td>-</td> 1423 <td>-</td> 1424 <td>-</td> 1425 <td>-</td> 1426 <td>-</td> 1427 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1428 <td></td> 1429 <td></td> 1430 <td></td> 1431 <td></td> 1432 <td></td> 1433 <td></td> 1434 <td></td> 1435 <td>⎋</td> 1436 </tr> 1437 <tr> 1438 <td>23+</td> 1439 <td>23+</td> 1440 <td>23+</td> 1441 <td>23+</td> 1442 <td>23+</td> 1443 <td>(23+)</td> 1444 <td>(23+)</td> 1445 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1446 <td></td> 1447 <td></td> 1448 <td></td> 1449 <td></td> 1450 <td></td> 1451 <td></td> 1452 <td></td> 1453 <td>○ ⎆</td> 1454 </tr> 1455 <tr> 1456 <td>-</td> 1457 <td>23+</td> 1458 <td>23+</td> 1459 <td>23+</td> 1460 <td>23+</td> 1461 <td>-</td> 1462 <td>-</td> 1463 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1464 <td></td> 1465 <td></td> 1466 <td></td> 1467 <td></td> 1468 <td></td> 1469 <td></td> 1470 <td></td> 1471 <td>⎆</td> 1472 </tr> 1473 <tr> 1474 <td>19+</td> 1475 <td>19+</td> 1476 <td>19+</td> 1477 <td>19+</td> 1478 <td>19+</td> 1479 <td>(19+)</td> 1480 <td>-</td> 1481 <td class=fn>{@link #setParameters setParameters}</td> 1482 <td></td> 1483 <td></td> 1484 <td></td> 1485 <td>●</td> 1486 <td>●</td> 1487 <td>●</td> 1488 <td>●</td> 1489 <td>●</td> 1490 </tr> 1491 <tr> 1492 <td>-</td> 1493 <td>(16+)</td> 1494 <td>(16+)</td> 1495 <td>16+</td> 1496 <td>(16+)</td> 1497 <td>(16+)</td> 1498 <td>-</td> 1499 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1500 <td>⎆</td> 1501 <td>⎆</td> 1502 <td>⎆</td> 1503 <td>⎆</td> 1504 <td>⎆</td> 1505 <td>⎆</td> 1506 <td>⎆</td> 1507 <td>⎆</td> 1508 </tr> 1509 <tr> 1510 <td>(29+)</td> 1511 <td>29+</td> 1512 <td>29+</td> 1513 <td>29+</td> 1514 <td>(29+)</td> 1515 <td>(29+)</td> 1516 <td>-</td> 1517 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1518 <td></td> 1519 <td></td> 1520 <td></td> 1521 <td></td> 1522 <td></td> 1523 <td></td> 1524 <td></td> 1525 <td></td> 1526 </tr> 1527 <tr> 1528 <td>-</td> 1529 <td>-</td> 1530 <td>18+</td> 1531 <td>18+</td> 1532 <td>-</td> 1533 <td>-</td> 1534 <td>-</td> 1535 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1536 <td></td> 1537 <td></td> 1538 <td>⎋</td> 1539 <td>⎋</td> 1540 <td>⎋</td> 1541 <td>⎋</td> 1542 <td>⎋</td> 1543 <td>⎋</td> 1544 </tr> 1545 <tr> 1546 <td>-</td> 1547 <td>16+</td> 1548 <td>21+(⇄)</td> 1549 <td>-</td> 1550 <td>-</td> 1551 <td>-</td> 1552 <td>-</td> 1553 <td class=fn>{@link #start start}</td> 1554 <td>●</td> 1555 <td>●</td> 1556 <td>●</td> 1557 <td>●</td> 1558 <td>●</td> 1559 <td>⁕</td> 1560 <td>●</td> 1561 <td>●</td> 1562 </tr> 1563 <tr> 1564 <td>-</td> 1565 <td>-</td> 1566 <td>16+</td> 1567 <td>16+</td> 1568 <td>16+</td> 1569 <td>-</td> 1570 <td>-</td> 1571 <td class=fn>{@link #stop stop}</td> 1572 <td>●</td> 1573 <td>●</td> 1574 <td>●</td> 1575 <td>●</td> 1576 <td>●</td> 1577 <td>●</td> 1578 <td>●</td> 1579 <td>●</td> 1580 </tr> 1581 </tbody> 1582 </table> 1583 */ 1584 final public class MediaCodec { 1585 1586 /** 1587 * Per buffer metadata includes an offset and size specifying 1588 * the range of valid data in the associated codec (output) buffer. 1589 */ 1590 public final static class BufferInfo { 1591 /** 1592 * Update the buffer metadata information. 1593 * 1594 * @param newOffset the start-offset of the data in the buffer. 1595 * @param newSize the amount of data (in bytes) in the buffer. 1596 * @param newTimeUs the presentation timestamp in microseconds. 1597 * @param newFlags buffer flags associated with the buffer. This 1598 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1599 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1600 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1601 public void set( 1602 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1603 offset = newOffset; 1604 size = newSize; 1605 presentationTimeUs = newTimeUs; 1606 flags = newFlags; 1607 } 1608 1609 /** 1610 * The start-offset of the data in the buffer. 1611 */ 1612 public int offset; 1613 1614 /** 1615 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1616 * the buffer has no data in it and can be discarded. The only 1617 * use of a 0-size buffer is to carry the end-of-stream marker. 1618 */ 1619 public int size; 1620 1621 /** 1622 * The presentation timestamp in microseconds for the buffer. 1623 * This is derived from the presentation timestamp passed in 1624 * with the corresponding input buffer. This should be ignored for 1625 * a 0-sized buffer. 1626 */ 1627 public long presentationTimeUs; 1628 1629 /** 1630 * Buffer flags associated with the buffer. A combination of 1631 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1632 * 1633 * <p>Encoded buffers that are key frames are marked with 1634 * {@link #BUFFER_FLAG_KEY_FRAME}. 1635 * 1636 * <p>The last output buffer corresponding to the input buffer 1637 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1638 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1639 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1640 * marker. 1641 */ 1642 @BufferFlag 1643 public int flags; 1644 1645 /** @hide */ 1646 @NonNull dup()1647 public BufferInfo dup() { 1648 BufferInfo copy = new BufferInfo(); 1649 copy.set(offset, size, presentationTimeUs, flags); 1650 return copy; 1651 } 1652 }; 1653 1654 // The follow flag constants MUST stay in sync with their equivalents 1655 // in MediaCodec.h ! 1656 1657 /** 1658 * This indicates that the (encoded) buffer marked as such contains 1659 * the data for a key frame. 1660 * 1661 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1662 */ 1663 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1664 1665 /** 1666 * This indicates that the (encoded) buffer marked as such contains 1667 * the data for a key frame. 1668 */ 1669 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1670 1671 /** 1672 * This indicated that the buffer marked as such contains codec 1673 * initialization / codec specific data instead of media data. 1674 */ 1675 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1676 1677 /** 1678 * This signals the end of stream, i.e. no buffers will be available 1679 * after this, unless of course, {@link #flush} follows. 1680 */ 1681 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1682 1683 /** 1684 * This indicates that the buffer only contains part of a frame, 1685 * and the decoder should batch the data until a buffer without 1686 * this flag appears before decoding the frame. 1687 */ 1688 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1689 1690 /** 1691 * This indicates that the buffer contains non-media data for the 1692 * muxer to process. 1693 * 1694 * All muxer data should start with a FOURCC header that determines the type of data. 1695 * 1696 * For example, when it contains Exif data sent to a MediaMuxer track of 1697 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1698 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1699 * 1700 * @hide 1701 */ 1702 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1703 1704 /** @hide */ 1705 @IntDef( 1706 flag = true, 1707 value = { 1708 BUFFER_FLAG_SYNC_FRAME, 1709 BUFFER_FLAG_KEY_FRAME, 1710 BUFFER_FLAG_CODEC_CONFIG, 1711 BUFFER_FLAG_END_OF_STREAM, 1712 BUFFER_FLAG_PARTIAL_FRAME, 1713 BUFFER_FLAG_MUXER_DATA, 1714 }) 1715 @Retention(RetentionPolicy.SOURCE) 1716 public @interface BufferFlag {} 1717 1718 private EventHandler mEventHandler; 1719 private EventHandler mOnFirstTunnelFrameReadyHandler; 1720 private EventHandler mOnFrameRenderedHandler; 1721 private EventHandler mCallbackHandler; 1722 private Callback mCallback; 1723 private OnFirstTunnelFrameReadyListener mOnFirstTunnelFrameReadyListener; 1724 private OnFrameRenderedListener mOnFrameRenderedListener; 1725 private final Object mListenerLock = new Object(); 1726 private MediaCodecInfo mCodecInfo; 1727 private final Object mCodecInfoLock = new Object(); 1728 private MediaCrypto mCrypto; 1729 1730 private static final int EVENT_CALLBACK = 1; 1731 private static final int EVENT_SET_CALLBACK = 2; 1732 private static final int EVENT_FRAME_RENDERED = 3; 1733 private static final int EVENT_FIRST_TUNNEL_FRAME_READY = 4; 1734 1735 private static final int CB_INPUT_AVAILABLE = 1; 1736 private static final int CB_OUTPUT_AVAILABLE = 2; 1737 private static final int CB_ERROR = 3; 1738 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1739 1740 private class EventHandler extends Handler { 1741 private MediaCodec mCodec; 1742 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1743 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1744 super(looper); 1745 mCodec = codec; 1746 } 1747 1748 @Override handleMessage(@onNull Message msg)1749 public void handleMessage(@NonNull Message msg) { 1750 switch (msg.what) { 1751 case EVENT_CALLBACK: 1752 { 1753 handleCallback(msg); 1754 break; 1755 } 1756 case EVENT_SET_CALLBACK: 1757 { 1758 mCallback = (MediaCodec.Callback) msg.obj; 1759 break; 1760 } 1761 case EVENT_FRAME_RENDERED: 1762 Map<String, Object> map = (Map<String, Object>)msg.obj; 1763 for (int i = 0; ; ++i) { 1764 Object mediaTimeUs = map.get(i + "-media-time-us"); 1765 Object systemNano = map.get(i + "-system-nano"); 1766 OnFrameRenderedListener onFrameRenderedListener; 1767 synchronized (mListenerLock) { 1768 onFrameRenderedListener = mOnFrameRenderedListener; 1769 } 1770 if (mediaTimeUs == null || systemNano == null 1771 || onFrameRenderedListener == null) { 1772 break; 1773 } 1774 onFrameRenderedListener.onFrameRendered( 1775 mCodec, (long)mediaTimeUs, (long)systemNano); 1776 } 1777 break; 1778 case EVENT_FIRST_TUNNEL_FRAME_READY: 1779 OnFirstTunnelFrameReadyListener onFirstTunnelFrameReadyListener; 1780 synchronized (mListenerLock) { 1781 onFirstTunnelFrameReadyListener = mOnFirstTunnelFrameReadyListener; 1782 } 1783 if (onFirstTunnelFrameReadyListener == null) { 1784 break; 1785 } 1786 onFirstTunnelFrameReadyListener.onFirstTunnelFrameReady(mCodec); 1787 break; 1788 default: 1789 { 1790 break; 1791 } 1792 } 1793 } 1794 handleCallback(@onNull Message msg)1795 private void handleCallback(@NonNull Message msg) { 1796 if (mCallback == null) { 1797 return; 1798 } 1799 1800 switch (msg.arg1) { 1801 case CB_INPUT_AVAILABLE: 1802 { 1803 int index = msg.arg2; 1804 synchronized(mBufferLock) { 1805 switch (mBufferMode) { 1806 case BUFFER_MODE_LEGACY: 1807 validateInputByteBufferLocked(mCachedInputBuffers, index); 1808 break; 1809 case BUFFER_MODE_BLOCK: 1810 while (mQueueRequests.size() <= index) { 1811 mQueueRequests.add(null); 1812 } 1813 QueueRequest request = mQueueRequests.get(index); 1814 if (request == null) { 1815 request = new QueueRequest(mCodec, index); 1816 mQueueRequests.set(index, request); 1817 } 1818 request.setAccessible(true); 1819 break; 1820 default: 1821 throw new IllegalStateException( 1822 "Unrecognized buffer mode: " + mBufferMode); 1823 } 1824 } 1825 mCallback.onInputBufferAvailable(mCodec, index); 1826 break; 1827 } 1828 1829 case CB_OUTPUT_AVAILABLE: 1830 { 1831 int index = msg.arg2; 1832 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1833 synchronized(mBufferLock) { 1834 switch (mBufferMode) { 1835 case BUFFER_MODE_LEGACY: 1836 validateOutputByteBufferLocked(mCachedOutputBuffers, index, info); 1837 break; 1838 case BUFFER_MODE_BLOCK: 1839 while (mOutputFrames.size() <= index) { 1840 mOutputFrames.add(null); 1841 } 1842 OutputFrame frame = mOutputFrames.get(index); 1843 if (frame == null) { 1844 frame = new OutputFrame(index); 1845 mOutputFrames.set(index, frame); 1846 } 1847 frame.setBufferInfo(info); 1848 frame.setAccessible(true); 1849 break; 1850 default: 1851 throw new IllegalStateException( 1852 "Unrecognized buffer mode: " + mBufferMode); 1853 } 1854 } 1855 mCallback.onOutputBufferAvailable( 1856 mCodec, index, info); 1857 break; 1858 } 1859 1860 case CB_ERROR: 1861 { 1862 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1863 break; 1864 } 1865 1866 case CB_OUTPUT_FORMAT_CHANGE: 1867 { 1868 mCallback.onOutputFormatChanged(mCodec, 1869 new MediaFormat((Map<String, Object>) msg.obj)); 1870 break; 1871 } 1872 1873 default: 1874 { 1875 break; 1876 } 1877 } 1878 } 1879 } 1880 1881 private boolean mHasSurface = false; 1882 1883 /** 1884 * Instantiate the preferred decoder supporting input data of the given mime type. 1885 * 1886 * The following is a partial list of defined mime types and their semantics: 1887 * <ul> 1888 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1889 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1890 * <li>"video/avc" - H.264/AVC video 1891 * <li>"video/hevc" - H.265/HEVC video 1892 * <li>"video/mp4v-es" - MPEG4 video 1893 * <li>"video/3gpp" - H.263 video 1894 * <li>"audio/3gpp" - AMR narrowband audio 1895 * <li>"audio/amr-wb" - AMR wideband audio 1896 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1897 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1898 * <li>"audio/vorbis" - vorbis audio 1899 * <li>"audio/g711-alaw" - G.711 alaw audio 1900 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1901 * </ul> 1902 * 1903 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1904 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1905 * given format. 1906 * 1907 * @param type The mime type of the input data. 1908 * @throws IOException if the codec cannot be created. 1909 * @throws IllegalArgumentException if type is not a valid mime type. 1910 * @throws NullPointerException if type is null. 1911 */ 1912 @NonNull createDecoderByType(@onNull String type)1913 public static MediaCodec createDecoderByType(@NonNull String type) 1914 throws IOException { 1915 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1916 } 1917 1918 /** 1919 * Instantiate the preferred encoder supporting output data of the given mime type. 1920 * 1921 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1922 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1923 * given format. 1924 * 1925 * @param type The desired mime type of the output data. 1926 * @throws IOException if the codec cannot be created. 1927 * @throws IllegalArgumentException if type is not a valid mime type. 1928 * @throws NullPointerException if type is null. 1929 */ 1930 @NonNull createEncoderByType(@onNull String type)1931 public static MediaCodec createEncoderByType(@NonNull String type) 1932 throws IOException { 1933 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1934 } 1935 1936 /** 1937 * If you know the exact name of the component you want to instantiate 1938 * use this method to instantiate it. Use with caution. 1939 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1940 * @param name The name of the codec to be instantiated. 1941 * @throws IOException if the codec cannot be created. 1942 * @throws IllegalArgumentException if name is not valid. 1943 * @throws NullPointerException if name is null. 1944 */ 1945 @NonNull createByCodecName(@onNull String name)1946 public static MediaCodec createByCodecName(@NonNull String name) 1947 throws IOException { 1948 return new MediaCodec(name, false /* nameIsType */, false /* encoder */); 1949 } 1950 1951 /** 1952 * This is the same as createByCodecName, but allows for instantiating a codec on behalf of a 1953 * client process. This is used for system apps or system services that create MediaCodecs on 1954 * behalf of other processes and will reclaim resources as necessary from processes with lower 1955 * priority than the client process, rather than processes with lower priority than the system 1956 * app or system service. Likely to be used with information obtained from 1957 * {@link android.media.MediaCodecList}. 1958 * @param name 1959 * @param clientPid 1960 * @param clientUid 1961 * @throws IOException if the codec cannot be created. 1962 * @throws IllegalArgumentException if name is not valid. 1963 * @throws NullPointerException if name is null. 1964 * @throws SecurityException if the MEDIA_RESOURCE_OVERRIDE_PID permission is not granted. 1965 * 1966 * @hide 1967 */ 1968 @NonNull 1969 @SystemApi 1970 @RequiresPermission(Manifest.permission.MEDIA_RESOURCE_OVERRIDE_PID) createByCodecNameForClient(@onNull String name, int clientPid, int clientUid)1971 public static MediaCodec createByCodecNameForClient(@NonNull String name, int clientPid, 1972 int clientUid) throws IOException { 1973 return new MediaCodec(name, false /* nameIsType */, false /* encoder */, clientPid, 1974 clientUid); 1975 } 1976 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder)1977 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder) { 1978 this(name, nameIsType, encoder, -1 /* pid */, -1 /* uid */); 1979 } 1980 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)1981 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder, int pid, 1982 int uid) { 1983 Looper looper; 1984 if ((looper = Looper.myLooper()) != null) { 1985 mEventHandler = new EventHandler(this, looper); 1986 } else if ((looper = Looper.getMainLooper()) != null) { 1987 mEventHandler = new EventHandler(this, looper); 1988 } else { 1989 mEventHandler = null; 1990 } 1991 mCallbackHandler = mEventHandler; 1992 mOnFirstTunnelFrameReadyHandler = mEventHandler; 1993 mOnFrameRenderedHandler = mEventHandler; 1994 1995 mBufferLock = new Object(); 1996 1997 // save name used at creation 1998 mNameAtCreation = nameIsType ? null : name; 1999 2000 native_setup(name, nameIsType, encoder, pid, uid); 2001 } 2002 2003 private String mNameAtCreation; 2004 2005 @Override finalize()2006 protected void finalize() { 2007 native_finalize(); 2008 mCrypto = null; 2009 } 2010 2011 /** 2012 * Returns the codec to its initial (Uninitialized) state. 2013 * 2014 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 2015 * error has occured to reset the codec to its initial state after creation. 2016 * 2017 * @throws CodecException if an unrecoverable error has occured and the codec 2018 * could not be reset. 2019 * @throws IllegalStateException if in the Released state. 2020 */ reset()2021 public final void reset() { 2022 freeAllTrackedBuffers(); // free buffers first 2023 native_reset(); 2024 mCrypto = null; 2025 } 2026 native_reset()2027 private native final void native_reset(); 2028 2029 /** 2030 * Free up resources used by the codec instance. 2031 * 2032 * Make sure you call this when you're done to free up any opened 2033 * component instance instead of relying on the garbage collector 2034 * to do this for you at some point in the future. 2035 */ release()2036 public final void release() { 2037 freeAllTrackedBuffers(); // free buffers first 2038 native_release(); 2039 mCrypto = null; 2040 } 2041 native_release()2042 private native final void native_release(); 2043 2044 /** 2045 * If this codec is to be used as an encoder, pass this flag. 2046 */ 2047 public static final int CONFIGURE_FLAG_ENCODE = 1; 2048 2049 /** 2050 * If this codec is to be used with {@link LinearBlock} and/or {@link 2051 * HardwareBuffer}, pass this flag. 2052 * <p> 2053 * When this flag is set, the following APIs throw {@link IncompatibleWithBlockModelException}. 2054 * <ul> 2055 * <li>{@link #getInputBuffer} 2056 * <li>{@link #getInputImage} 2057 * <li>{@link #getInputBuffers} 2058 * <li>{@link #getOutputBuffer} 2059 * <li>{@link #getOutputImage} 2060 * <li>{@link #getOutputBuffers} 2061 * <li>{@link #queueInputBuffer} 2062 * <li>{@link #queueSecureInputBuffer} 2063 * <li>{@link #dequeueInputBuffer} 2064 * <li>{@link #dequeueOutputBuffer} 2065 * </ul> 2066 */ 2067 public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; 2068 2069 /** @hide */ 2070 @IntDef( 2071 flag = true, 2072 value = { 2073 CONFIGURE_FLAG_ENCODE, 2074 CONFIGURE_FLAG_USE_BLOCK_MODEL, 2075 }) 2076 @Retention(RetentionPolicy.SOURCE) 2077 public @interface ConfigureFlag {} 2078 2079 /** 2080 * Thrown when the codec is configured for block model and an incompatible API is called. 2081 */ 2082 public class IncompatibleWithBlockModelException extends RuntimeException { IncompatibleWithBlockModelException()2083 IncompatibleWithBlockModelException() { } 2084 IncompatibleWithBlockModelException(String message)2085 IncompatibleWithBlockModelException(String message) { 2086 super(message); 2087 } 2088 IncompatibleWithBlockModelException(String message, Throwable cause)2089 IncompatibleWithBlockModelException(String message, Throwable cause) { 2090 super(message, cause); 2091 } 2092 IncompatibleWithBlockModelException(Throwable cause)2093 IncompatibleWithBlockModelException(Throwable cause) { 2094 super(cause); 2095 } 2096 } 2097 2098 /** 2099 * Configures a component. 2100 * 2101 * @param format The format of the input data (decoder) or the desired 2102 * format of the output data (encoder). Passing {@code null} 2103 * as {@code format} is equivalent to passing an 2104 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2105 * @param surface Specify a surface on which to render the output of this 2106 * decoder. Pass {@code null} as {@code surface} if the 2107 * codec does not generate raw video output (e.g. not a video 2108 * decoder) and/or if you want to configure the codec for 2109 * {@link ByteBuffer} output. 2110 * @param crypto Specify a crypto object to facilitate secure decryption 2111 * of the media data. Pass {@code null} as {@code crypto} for 2112 * non-secure codecs. 2113 * Please note that {@link MediaCodec} does NOT take ownership 2114 * of the {@link MediaCrypto} object; it is the application's 2115 * responsibility to properly cleanup the {@link MediaCrypto} object 2116 * when not in use. 2117 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2118 * component as an encoder. 2119 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2120 * or the format is unacceptable (e.g. missing a mandatory key), 2121 * or the flags are not set properly 2122 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2123 * @throws IllegalStateException if not in the Uninitialized state. 2124 * @throws CryptoException upon DRM error. 2125 * @throws CodecException upon codec error. 2126 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)2127 public void configure( 2128 @Nullable MediaFormat format, 2129 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2130 @ConfigureFlag int flags) { 2131 configure(format, surface, crypto, null, flags); 2132 } 2133 2134 /** 2135 * Configure a component to be used with a descrambler. 2136 * @param format The format of the input data (decoder) or the desired 2137 * format of the output data (encoder). Passing {@code null} 2138 * as {@code format} is equivalent to passing an 2139 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2140 * @param surface Specify a surface on which to render the output of this 2141 * decoder. Pass {@code null} as {@code surface} if the 2142 * codec does not generate raw video output (e.g. not a video 2143 * decoder) and/or if you want to configure the codec for 2144 * {@link ByteBuffer} output. 2145 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2146 * component as an encoder. 2147 * @param descrambler Specify a descrambler object to facilitate secure 2148 * descrambling of the media data, or null for non-secure codecs. 2149 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2150 * or the format is unacceptable (e.g. missing a mandatory key), 2151 * or the flags are not set properly 2152 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2153 * @throws IllegalStateException if not in the Uninitialized state. 2154 * @throws CryptoException upon DRM error. 2155 * @throws CodecException upon codec error. 2156 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)2157 public void configure( 2158 @Nullable MediaFormat format, @Nullable Surface surface, 2159 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 2160 configure(format, surface, null, 2161 descrambler != null ? descrambler.getBinder() : null, flags); 2162 } 2163 2164 private static final int BUFFER_MODE_INVALID = -1; 2165 private static final int BUFFER_MODE_LEGACY = 0; 2166 private static final int BUFFER_MODE_BLOCK = 1; 2167 private int mBufferMode = BUFFER_MODE_INVALID; 2168 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2169 private void configure( 2170 @Nullable MediaFormat format, @Nullable Surface surface, 2171 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 2172 @ConfigureFlag int flags) { 2173 if (crypto != null && descramblerBinder != null) { 2174 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 2175 } 2176 2177 String[] keys = null; 2178 Object[] values = null; 2179 2180 if (format != null) { 2181 Map<String, Object> formatMap = format.getMap(); 2182 keys = new String[formatMap.size()]; 2183 values = new Object[formatMap.size()]; 2184 2185 int i = 0; 2186 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2187 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2188 int sessionId = 0; 2189 try { 2190 sessionId = (Integer)entry.getValue(); 2191 } 2192 catch (Exception e) { 2193 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2194 } 2195 keys[i] = "audio-hw-sync"; 2196 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2197 } else { 2198 keys[i] = entry.getKey(); 2199 values[i] = entry.getValue(); 2200 } 2201 ++i; 2202 } 2203 } 2204 2205 mHasSurface = surface != null; 2206 mCrypto = crypto; 2207 synchronized (mBufferLock) { 2208 if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { 2209 mBufferMode = BUFFER_MODE_BLOCK; 2210 } else { 2211 mBufferMode = BUFFER_MODE_LEGACY; 2212 } 2213 } 2214 2215 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2216 } 2217 2218 /** 2219 * Dynamically sets the output surface of a codec. 2220 * <p> 2221 * This can only be used if the codec was configured with an output surface. The 2222 * new output surface should have a compatible usage type to the original output surface. 2223 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2224 * to ImageReader (software readable) output. 2225 * @param surface the output surface to use. It must not be {@code null}. 2226 * @throws IllegalStateException if the codec does not support setting the output 2227 * surface in the current state. 2228 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2229 */ setOutputSurface(@onNull Surface surface)2230 public void setOutputSurface(@NonNull Surface surface) { 2231 if (!mHasSurface) { 2232 throw new IllegalStateException("codec was not configured for an output surface"); 2233 } 2234 native_setSurface(surface); 2235 } 2236 native_setSurface(@onNull Surface surface)2237 private native void native_setSurface(@NonNull Surface surface); 2238 2239 /** 2240 * Create a persistent input surface that can be used with codecs that normally have an input 2241 * surface, such as video encoders. A persistent input can be reused by subsequent 2242 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2243 * most one codec or recorder instance concurrently. 2244 * <p> 2245 * The application is responsible for calling release() on the Surface when done. 2246 * 2247 * @return an input surface that can be used with {@link #setInputSurface}. 2248 */ 2249 @NonNull createPersistentInputSurface()2250 public static Surface createPersistentInputSurface() { 2251 return native_createPersistentInputSurface(); 2252 } 2253 2254 static class PersistentSurface extends Surface { 2255 @SuppressWarnings("unused") PersistentSurface()2256 PersistentSurface() {} // used by native 2257 2258 @Override release()2259 public void release() { 2260 native_releasePersistentInputSurface(this); 2261 super.release(); 2262 } 2263 2264 private long mPersistentObject; 2265 }; 2266 2267 /** 2268 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2269 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2270 * lieu of {@link #createInputSurface}. 2271 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2272 * @throws IllegalStateException if not in the Configured state or does not require an input 2273 * surface. 2274 * @throws IllegalArgumentException if the surface was not created by 2275 * {@link #createPersistentInputSurface}. 2276 */ setInputSurface(@onNull Surface surface)2277 public void setInputSurface(@NonNull Surface surface) { 2278 if (!(surface instanceof PersistentSurface)) { 2279 throw new IllegalArgumentException("not a PersistentSurface"); 2280 } 2281 native_setInputSurface(surface); 2282 } 2283 2284 @NonNull native_createPersistentInputSurface()2285 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2286 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2287 private native final void native_setInputSurface(@NonNull Surface surface); 2288 native_setCallback(@ullable Callback cb)2289 private native final void native_setCallback(@Nullable Callback cb); 2290 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2291 private native final void native_configure( 2292 @Nullable String[] keys, @Nullable Object[] values, 2293 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2294 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2295 2296 /** 2297 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2298 * may only be called after {@link #configure} and before {@link #start}. 2299 * <p> 2300 * The application is responsible for calling release() on the Surface when 2301 * done. 2302 * <p> 2303 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2304 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2305 * unexpected results. 2306 * @throws IllegalStateException if not in the Configured state. 2307 */ 2308 @NonNull createInputSurface()2309 public native final Surface createInputSurface(); 2310 2311 /** 2312 * After successfully configuring the component, call {@code start}. 2313 * <p> 2314 * Call {@code start} also if the codec is configured in asynchronous mode, 2315 * and it has just been flushed, to resume requesting input buffers. 2316 * @throws IllegalStateException if not in the Configured state 2317 * or just after {@link #flush} for a codec that is configured 2318 * in asynchronous mode. 2319 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2320 * for start may be attributed to future method calls. 2321 */ start()2322 public final void start() { 2323 native_start(); 2324 } native_start()2325 private native final void native_start(); 2326 2327 /** 2328 * Finish the decode/encode session, note that the codec instance 2329 * remains active and ready to be {@link #start}ed again. 2330 * To ensure that it is available to other client call {@link #release} 2331 * and don't just rely on garbage collection to eventually do this for you. 2332 * @throws IllegalStateException if in the Released state. 2333 */ stop()2334 public final void stop() { 2335 native_stop(); 2336 freeAllTrackedBuffers(); 2337 2338 synchronized (mListenerLock) { 2339 if (mCallbackHandler != null) { 2340 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2341 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2342 } 2343 if (mOnFirstTunnelFrameReadyHandler != null) { 2344 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 2345 } 2346 if (mOnFrameRenderedHandler != null) { 2347 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2348 } 2349 } 2350 } 2351 native_stop()2352 private native final void native_stop(); 2353 2354 /** 2355 * Flush both input and output ports of the component. 2356 * <p> 2357 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2358 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2359 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2360 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2361 * invalid, and all buffers are owned by the codec. 2362 * <p> 2363 * If the codec is configured in asynchronous mode, call {@link #start} 2364 * after {@code flush} has returned to resume codec operations. The codec 2365 * will not request input buffers until this has happened. 2366 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2367 * callbacks that were not handled prior to calling {@code flush}. 2368 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2369 * should be discarded.</strong> 2370 * <p> 2371 * If the codec is configured in synchronous mode, codec will resume 2372 * automatically if it is configured with an input surface. Otherwise, it 2373 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2374 * 2375 * @throws IllegalStateException if not in the Executing state. 2376 * @throws MediaCodec.CodecException upon codec error. 2377 */ flush()2378 public final void flush() { 2379 synchronized(mBufferLock) { 2380 invalidateByteBuffersLocked(mCachedInputBuffers); 2381 invalidateByteBuffersLocked(mCachedOutputBuffers); 2382 mValidInputIndices.clear(); 2383 mValidOutputIndices.clear(); 2384 mDequeuedInputBuffers.clear(); 2385 mDequeuedOutputBuffers.clear(); 2386 } 2387 native_flush(); 2388 } 2389 native_flush()2390 private native final void native_flush(); 2391 2392 /** 2393 * Thrown when an internal codec error occurs. 2394 */ 2395 public final static class CodecException extends IllegalStateException { 2396 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2397 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2398 super(detailMessage); 2399 mErrorCode = errorCode; 2400 mActionCode = actionCode; 2401 2402 // TODO get this from codec 2403 final String sign = errorCode < 0 ? "neg_" : ""; 2404 mDiagnosticInfo = 2405 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2406 } 2407 2408 /** 2409 * Returns true if the codec exception is a transient issue, 2410 * perhaps due to resource constraints, and that the method 2411 * (or encoding/decoding) may be retried at a later time. 2412 */ 2413 public boolean isTransient() { 2414 return mActionCode == ACTION_TRANSIENT; 2415 } 2416 2417 /** 2418 * Returns true if the codec cannot proceed further, 2419 * but can be recovered by stopping, configuring, 2420 * and starting again. 2421 */ 2422 public boolean isRecoverable() { 2423 return mActionCode == ACTION_RECOVERABLE; 2424 } 2425 2426 /** 2427 * Retrieve the error code associated with a CodecException 2428 */ 2429 public int getErrorCode() { 2430 return mErrorCode; 2431 } 2432 2433 /** 2434 * Retrieve a developer-readable diagnostic information string 2435 * associated with the exception. Do not show this to end-users, 2436 * since this string will not be localized or generally 2437 * comprehensible to end-users. 2438 */ 2439 public @NonNull String getDiagnosticInfo() { 2440 return mDiagnosticInfo; 2441 } 2442 2443 /** 2444 * This indicates required resource was not able to be allocated. 2445 */ 2446 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2447 2448 /** 2449 * This indicates the resource manager reclaimed the media resource used by the codec. 2450 * <p> 2451 * With this exception, the codec must be released, as it has moved to terminal state. 2452 */ 2453 public static final int ERROR_RECLAIMED = 1101; 2454 2455 /** @hide */ 2456 @IntDef({ 2457 ERROR_INSUFFICIENT_RESOURCE, 2458 ERROR_RECLAIMED, 2459 }) 2460 @Retention(RetentionPolicy.SOURCE) 2461 public @interface ReasonCode {} 2462 2463 /* Must be in sync with android_media_MediaCodec.cpp */ 2464 private final static int ACTION_TRANSIENT = 1; 2465 private final static int ACTION_RECOVERABLE = 2; 2466 2467 private final String mDiagnosticInfo; 2468 private final int mErrorCode; 2469 private final int mActionCode; 2470 } 2471 2472 /** 2473 * Thrown when a crypto error occurs while queueing a secure input buffer. 2474 */ 2475 public final static class CryptoException extends RuntimeException { 2476 public CryptoException(int errorCode, @Nullable String detailMessage) { 2477 super(detailMessage); 2478 mErrorCode = errorCode; 2479 } 2480 2481 /** 2482 * This indicates that the requested key was not found when trying to 2483 * perform a decrypt operation. The operation can be retried after adding 2484 * the correct decryption key. 2485 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_NO_KEY}. 2486 */ 2487 public static final int ERROR_NO_KEY = MediaDrm.ErrorCodes.ERROR_NO_KEY; 2488 2489 /** 2490 * This indicates that the key used for decryption is no longer 2491 * valid due to license term expiration. The operation can be retried 2492 * after updating the expired keys. 2493 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_KEY_EXPIRED}. 2494 */ 2495 public static final int ERROR_KEY_EXPIRED = MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED; 2496 2497 /** 2498 * This indicates that a required crypto resource was not able to be 2499 * allocated while attempting the requested operation. The operation 2500 * can be retried if the app is able to release resources. 2501 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_RESOURCE_BUSY} 2502 */ 2503 public static final int ERROR_RESOURCE_BUSY = MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY; 2504 2505 /** 2506 * This indicates that the output protection levels supported by the 2507 * device are not sufficient to meet the requirements set by the 2508 * content owner in the license policy. 2509 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_OUTPUT_PROTECTION} 2510 */ 2511 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 2512 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION; 2513 2514 /** 2515 * This indicates that decryption was attempted on a session that is 2516 * not opened, which could be due to a failure to open the session, 2517 * closing the session prematurely, or the session being reclaimed 2518 * by the resource manager. 2519 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_SESSION_NOT_OPENED} 2520 */ 2521 public static final int ERROR_SESSION_NOT_OPENED = 2522 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED; 2523 2524 /** 2525 * This indicates that an operation was attempted that could not be 2526 * supported by the crypto system of the device in its current 2527 * configuration. It may occur when the license policy requires 2528 * device security features that aren't supported by the device, 2529 * or due to an internal error in the crypto system that prevents 2530 * the specified security policy from being met. 2531 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_UNSUPPORTED_OPERATION} 2532 */ 2533 public static final int ERROR_UNSUPPORTED_OPERATION = 2534 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION; 2535 2536 /** 2537 * This indicates that the security level of the device is not 2538 * sufficient to meet the requirements set by the content owner 2539 * in the license policy. 2540 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_SECURITY} 2541 */ 2542 public static final int ERROR_INSUFFICIENT_SECURITY = 2543 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY; 2544 2545 /** 2546 * This indicates that the video frame being decrypted exceeds 2547 * the size of the device's protected output buffers. When 2548 * encountering this error the app should try playing content 2549 * of a lower resolution. 2550 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_FRAME_TOO_LARGE} 2551 */ 2552 public static final int ERROR_FRAME_TOO_LARGE = MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE; 2553 2554 /** 2555 * This error indicates that session state has been 2556 * invalidated. It can occur on devices that are not capable 2557 * of retaining crypto session state across device 2558 * suspend/resume. The session must be closed and a new 2559 * session opened to resume operation. 2560 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_LOST_STATE} 2561 */ 2562 public static final int ERROR_LOST_STATE = MediaDrm.ErrorCodes.ERROR_LOST_STATE; 2563 2564 /** @hide */ 2565 @IntDef({ 2566 MediaDrm.ErrorCodes.ERROR_NO_KEY, 2567 MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED, 2568 MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY, 2569 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2570 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED, 2571 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION, 2572 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY, 2573 MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE, 2574 MediaDrm.ErrorCodes.ERROR_LOST_STATE, 2575 MediaDrm.ErrorCodes.ERROR_GENERIC_OEM, 2576 MediaDrm.ErrorCodes.ERROR_GENERIC_PLUGIN, 2577 MediaDrm.ErrorCodes.ERROR_LICENSE_PARSE, 2578 MediaDrm.ErrorCodes.ERROR_MEDIA_FRAMEWORK, 2579 MediaDrm.ErrorCodes.ERROR_ZERO_SUBSAMPLES 2580 }) 2581 @Retention(RetentionPolicy.SOURCE) 2582 public @interface CryptoErrorCode {} 2583 2584 /** 2585 * Returns error code associated with this {@link CryptoException}. 2586 * <p> 2587 * Please refer to {@link MediaDrm.ErrorCodes} for the general error 2588 * handling strategy and details about each possible return value. 2589 * 2590 * @return an error code defined in {@link MediaDrm.ErrorCodes}. 2591 */ 2592 @CryptoErrorCode 2593 public int getErrorCode() { 2594 return mErrorCode; 2595 } 2596 2597 private int mErrorCode; 2598 } 2599 2600 /** 2601 * After filling a range of the input buffer at the specified index 2602 * submit it to the component. Once an input buffer is queued to 2603 * the codec, it MUST NOT be used until it is later retrieved by 2604 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2605 * return value or a {@link Callback#onInputBufferAvailable} 2606 * callback. 2607 * <p> 2608 * Many decoders require the actual compressed data stream to be 2609 * preceded by "codec specific data", i.e. setup data used to initialize 2610 * the codec such as PPS/SPS in the case of AVC video or code tables 2611 * in the case of vorbis audio. 2612 * The class {@link android.media.MediaExtractor} provides codec 2613 * specific data as part of 2614 * the returned track format in entries named "csd-0", "csd-1" ... 2615 * <p> 2616 * These buffers can be submitted directly after {@link #start} or 2617 * {@link #flush} by specifying the flag {@link 2618 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2619 * codec with a {@link MediaFormat} containing these keys, they 2620 * will be automatically submitted by MediaCodec directly after 2621 * start. Therefore, the use of {@link 2622 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2623 * recommended only for advanced users. 2624 * <p> 2625 * To indicate that this is the final piece of input data (or rather that 2626 * no more input data follows unless the decoder is subsequently flushed) 2627 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2628 * <p class=note> 2629 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2630 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2631 * Surface output buffers, and the resulting frame timestamp was undefined. 2632 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2633 * Similarly, since frame timestamps can be used by the destination surface for rendering 2634 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2635 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2636 * SurfaceView specifics}).</strong> 2637 * 2638 * @param index The index of a client-owned input buffer previously returned 2639 * in a call to {@link #dequeueInputBuffer}. 2640 * @param offset The byte offset into the input buffer at which the data starts. 2641 * @param size The number of bytes of valid input data. 2642 * @param presentationTimeUs The presentation timestamp in microseconds for this 2643 * buffer. This is normally the media time at which this 2644 * buffer should be presented (rendered). When using an output 2645 * surface, this will be propagated as the {@link 2646 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2647 * conversion to nanoseconds). 2648 * @param flags A bitmask of flags 2649 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2650 * While not prohibited, most codecs do not use the 2651 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2652 * @throws IllegalStateException if not in the Executing state. 2653 * @throws MediaCodec.CodecException upon codec error. 2654 * @throws CryptoException if a crypto object has been specified in 2655 * {@link #configure} 2656 */ 2657 public final void queueInputBuffer( 2658 int index, 2659 int offset, int size, long presentationTimeUs, int flags) 2660 throws CryptoException { 2661 synchronized(mBufferLock) { 2662 if (mBufferMode == BUFFER_MODE_BLOCK) { 2663 throw new IncompatibleWithBlockModelException("queueInputBuffer() " 2664 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2665 + "Please use getQueueRequest() to queue buffers"); 2666 } 2667 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 2668 mDequeuedInputBuffers.remove(index); 2669 } 2670 try { 2671 native_queueInputBuffer( 2672 index, offset, size, presentationTimeUs, flags); 2673 } catch (CryptoException | IllegalStateException e) { 2674 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 2675 throw e; 2676 } 2677 } 2678 2679 private native final void native_queueInputBuffer( 2680 int index, 2681 int offset, int size, long presentationTimeUs, int flags) 2682 throws CryptoException; 2683 2684 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2685 public static final int CRYPTO_MODE_AES_CTR = 1; 2686 public static final int CRYPTO_MODE_AES_CBC = 2; 2687 2688 /** 2689 * Metadata describing the structure of an encrypted input sample. 2690 * <p> 2691 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2692 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2693 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2694 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2695 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2696 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2697 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2698 * <p> 2699 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2700 * "Common encryption in ISO base media file format files". 2701 * <p> 2702 * <h3>ISO-CENC Schemes</h3> 2703 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2704 * corresponding to each possible combination of an AES mode with the presence or absence of 2705 * patterned encryption. 2706 * 2707 * <table style="width: 0%"> 2708 * <thead> 2709 * <tr> 2710 * <th> </th> 2711 * <th>AES-CTR</th> 2712 * <th>AES-CBC</th> 2713 * </tr> 2714 * </thead> 2715 * <tbody> 2716 * <tr> 2717 * <th>Without Patterns</th> 2718 * <td>cenc</td> 2719 * <td>cbc1</td> 2720 * </tr><tr> 2721 * <th>With Patterns</th> 2722 * <td>cens</td> 2723 * <td>cbcs</td> 2724 * </tr> 2725 * </tbody> 2726 * </table> 2727 * 2728 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2729 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2730 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2731 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2732 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2733 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2734 * {@link #setPattern} is never called is all zeroes. 2735 * <p> 2736 * <h4>HLS SAMPLE-AES Audio</h4> 2737 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2738 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2739 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2740 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2741 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2742 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2743 * while still decrypting every block. 2744 */ 2745 public final static class CryptoInfo { 2746 /** 2747 * The number of subSamples that make up the buffer's contents. 2748 */ 2749 public int numSubSamples; 2750 /** 2751 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2752 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2753 */ 2754 public int[] numBytesOfClearData; 2755 /** 2756 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2757 * as clear and {@link #numBytesOfClearData} must be specified. 2758 */ 2759 public int[] numBytesOfEncryptedData; 2760 /** 2761 * A 16-byte key id 2762 */ 2763 public byte[] key; 2764 /** 2765 * A 16-byte initialization vector 2766 */ 2767 public byte[] iv; 2768 /** 2769 * The type of encryption that has been applied, 2770 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2771 * and {@link #CRYPTO_MODE_AES_CBC} 2772 */ 2773 public int mode; 2774 2775 /** 2776 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2777 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2778 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2779 */ 2780 public final static class Pattern { 2781 /** 2782 * Number of blocks to be encrypted in the pattern. If both this and 2783 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2784 */ 2785 private int mEncryptBlocks; 2786 2787 /** 2788 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2789 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2790 */ 2791 private int mSkipBlocks; 2792 2793 /** 2794 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2795 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2796 */ 2797 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2798 set(blocksToEncrypt, blocksToSkip); 2799 } 2800 2801 /** 2802 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2803 * parameters are zero, pattern encryption is inoperative. 2804 */ 2805 public void set(int blocksToEncrypt, int blocksToSkip) { 2806 mEncryptBlocks = blocksToEncrypt; 2807 mSkipBlocks = blocksToSkip; 2808 } 2809 2810 /** 2811 * Return the number of blocks to skip in a sample encryption pattern. 2812 */ 2813 public int getSkipBlocks() { 2814 return mSkipBlocks; 2815 } 2816 2817 /** 2818 * Return the number of blocks to encrypt in a sample encryption pattern. 2819 */ 2820 public int getEncryptBlocks() { 2821 return mEncryptBlocks; 2822 } 2823 }; 2824 2825 private static final Pattern ZERO_PATTERN = new Pattern(0, 0); 2826 2827 /** 2828 * The pattern applicable to the protected data in each subsample. 2829 */ 2830 private Pattern mPattern = ZERO_PATTERN; 2831 2832 /** 2833 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2834 * a {@link MediaCodec.CryptoInfo} instance. 2835 */ 2836 public void set( 2837 int newNumSubSamples, 2838 @NonNull int[] newNumBytesOfClearData, 2839 @NonNull int[] newNumBytesOfEncryptedData, 2840 @NonNull byte[] newKey, 2841 @NonNull byte[] newIV, 2842 int newMode) { 2843 numSubSamples = newNumSubSamples; 2844 numBytesOfClearData = newNumBytesOfClearData; 2845 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2846 key = newKey; 2847 iv = newIV; 2848 mode = newMode; 2849 mPattern = ZERO_PATTERN; 2850 } 2851 2852 /** 2853 * Returns the {@link Pattern encryption pattern}. 2854 */ 2855 public @NonNull Pattern getPattern() { 2856 return new Pattern(mPattern.getEncryptBlocks(), mPattern.getSkipBlocks()); 2857 } 2858 2859 /** 2860 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2861 * See {@link Pattern}. 2862 */ 2863 public void setPattern(Pattern newPattern) { 2864 if (newPattern == null) { 2865 newPattern = ZERO_PATTERN; 2866 } 2867 setPattern(newPattern.getEncryptBlocks(), newPattern.getSkipBlocks()); 2868 } 2869 2870 // Accessed from android_media_MediaExtractor.cpp. 2871 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 2872 mPattern = new Pattern(blocksToEncrypt, blocksToSkip); 2873 } 2874 2875 @Override 2876 public String toString() { 2877 StringBuilder builder = new StringBuilder(); 2878 builder.append(numSubSamples + " subsamples, key ["); 2879 String hexdigits = "0123456789abcdef"; 2880 for (int i = 0; i < key.length; i++) { 2881 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2882 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2883 } 2884 builder.append("], iv ["); 2885 for (int i = 0; i < iv.length; i++) { 2886 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2887 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2888 } 2889 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2890 builder.append(Arrays.toString(numBytesOfClearData)); 2891 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2892 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2893 builder.append(", pattern (encrypt: "); builder.append(mPattern.mEncryptBlocks)2894 builder.append(mPattern.mEncryptBlocks); 2895 builder.append(", skip: "); builder.append(mPattern.mSkipBlocks)2896 builder.append(mPattern.mSkipBlocks); 2897 builder.append(")"); 2898 return builder.toString(); 2899 } 2900 }; 2901 2902 /** 2903 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2904 * potentially encrypted. 2905 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2906 * 2907 * @param index The index of a client-owned input buffer previously returned 2908 * in a call to {@link #dequeueInputBuffer}. 2909 * @param offset The byte offset into the input buffer at which the data starts. 2910 * @param info Metadata required to facilitate decryption, the object can be 2911 * reused immediately after this call returns. 2912 * @param presentationTimeUs The presentation timestamp in microseconds for this 2913 * buffer. This is normally the media time at which this 2914 * buffer should be presented (rendered). 2915 * @param flags A bitmask of flags 2916 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2917 * While not prohibited, most codecs do not use the 2918 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2919 * @throws IllegalStateException if not in the Executing state. 2920 * @throws MediaCodec.CodecException upon codec error. 2921 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2922 * An error code associated with the exception helps identify the 2923 * reason for the failure. 2924 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2925 public final void queueSecureInputBuffer( 2926 int index, 2927 int offset, 2928 @NonNull CryptoInfo info, 2929 long presentationTimeUs, 2930 int flags) throws CryptoException { 2931 synchronized(mBufferLock) { 2932 if (mBufferMode == BUFFER_MODE_BLOCK) { 2933 throw new IncompatibleWithBlockModelException("queueSecureInputBuffer() " 2934 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2935 + "Please use getQueueRequest() to queue buffers"); 2936 } 2937 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 2938 mDequeuedInputBuffers.remove(index); 2939 } 2940 try { 2941 native_queueSecureInputBuffer( 2942 index, offset, info, presentationTimeUs, flags); 2943 } catch (CryptoException | IllegalStateException e) { 2944 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 2945 throw e; 2946 } 2947 } 2948 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2949 private native final void native_queueSecureInputBuffer( 2950 int index, 2951 int offset, 2952 @NonNull CryptoInfo info, 2953 long presentationTimeUs, 2954 int flags) throws CryptoException; 2955 2956 /** 2957 * Returns the index of an input buffer to be filled with valid data 2958 * or -1 if no such buffer is currently available. 2959 * This method will return immediately if timeoutUs == 0, wait indefinitely 2960 * for the availability of an input buffer if timeoutUs < 0 or wait up 2961 * to "timeoutUs" microseconds if timeoutUs > 0. 2962 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2963 * @throws IllegalStateException if not in the Executing state, 2964 * or codec is configured in asynchronous mode. 2965 * @throws MediaCodec.CodecException upon codec error. 2966 */ dequeueInputBuffer(long timeoutUs)2967 public final int dequeueInputBuffer(long timeoutUs) { 2968 synchronized (mBufferLock) { 2969 if (mBufferMode == BUFFER_MODE_BLOCK) { 2970 throw new IncompatibleWithBlockModelException("dequeueInputBuffer() " 2971 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2972 + "Please use MediaCodec.Callback objectes to get input buffer slots."); 2973 } 2974 } 2975 int res = native_dequeueInputBuffer(timeoutUs); 2976 if (res >= 0) { 2977 synchronized(mBufferLock) { 2978 validateInputByteBufferLocked(mCachedInputBuffers, res); 2979 } 2980 } 2981 return res; 2982 } 2983 native_dequeueInputBuffer(long timeoutUs)2984 private native final int native_dequeueInputBuffer(long timeoutUs); 2985 2986 /** 2987 * Section of memory that represents a linear block. Applications may 2988 * acquire a block via {@link LinearBlock#obtain} and queue all or part 2989 * of the block as an input buffer to a codec, or get a block allocated by 2990 * codec as an output buffer from {@link OutputFrame}. 2991 * 2992 * {@see QueueRequest#setLinearBlock} 2993 * {@see QueueRequest#setEncryptedLinearBlock} 2994 * {@see OutputFrame#getLinearBlock} 2995 */ 2996 public static final class LinearBlock { 2997 // No public constructors. LinearBlock()2998 private LinearBlock() {} 2999 3000 /** 3001 * Returns true if the buffer is mappable. 3002 * @throws IllegalStateException if invalid 3003 */ isMappable()3004 public boolean isMappable() { 3005 synchronized (mLock) { 3006 if (!mValid) { 3007 throw new IllegalStateException("The linear block is invalid"); 3008 } 3009 return mMappable; 3010 } 3011 } 3012 3013 /** 3014 * Map the memory and return the mapped region. 3015 * <p> 3016 * The returned memory region becomes inaccessible after 3017 * {@link #recycle}, or the buffer is queued to the codecs and not 3018 * returned to the client yet. 3019 * 3020 * @return mapped memory region as {@link ByteBuffer} object 3021 * @throws IllegalStateException if not mappable or invalid 3022 */ map()3023 public @NonNull ByteBuffer map() { 3024 synchronized (mLock) { 3025 if (!mValid) { 3026 throw new IllegalStateException("The linear block is invalid"); 3027 } 3028 if (!mMappable) { 3029 throw new IllegalStateException("The linear block is not mappable"); 3030 } 3031 if (mMapped == null) { 3032 mMapped = native_map(); 3033 } 3034 return mMapped; 3035 } 3036 } 3037 native_map()3038 private native ByteBuffer native_map(); 3039 3040 /** 3041 * Mark this block as ready to be recycled by the framework once it is 3042 * no longer in use. All operations to this object after 3043 * this call will cause exceptions, as well as attempt to access the 3044 * previously mapped memory region. Caller should clear all references 3045 * to this object after this call. 3046 * <p> 3047 * To avoid excessive memory consumption, it is recommended that callers 3048 * recycle buffers as soon as they no longer need the buffers 3049 * 3050 * @throws IllegalStateException if invalid 3051 */ recycle()3052 public void recycle() { 3053 synchronized (mLock) { 3054 if (!mValid) { 3055 throw new IllegalStateException("The linear block is invalid"); 3056 } 3057 if (mMapped != null) { 3058 mMapped.setAccessible(false); 3059 mMapped = null; 3060 } 3061 native_recycle(); 3062 mValid = false; 3063 mNativeContext = 0; 3064 } 3065 sPool.offer(this); 3066 } 3067 native_recycle()3068 private native void native_recycle(); 3069 native_obtain(int capacity, String[] codecNames)3070 private native void native_obtain(int capacity, String[] codecNames); 3071 3072 @Override finalize()3073 protected void finalize() { 3074 native_recycle(); 3075 } 3076 3077 /** 3078 * Returns true if it is possible to allocate a linear block that can be 3079 * passed to all listed codecs as input buffers without copying the 3080 * content. 3081 * <p> 3082 * Note that even if this function returns true, {@link #obtain} may 3083 * still throw due to invalid arguments or allocation failure. 3084 * 3085 * @param codecNames list of codecs that the client wants to use a 3086 * linear block without copying. Null entries are 3087 * ignored. 3088 */ isCodecCopyFreeCompatible(@onNull String[] codecNames)3089 public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { 3090 return native_checkCompatible(codecNames); 3091 } 3092 native_checkCompatible(@onNull String[] codecNames)3093 private static native boolean native_checkCompatible(@NonNull String[] codecNames); 3094 3095 /** 3096 * Obtain a linear block object no smaller than {@code capacity}. 3097 * If {@link #isCodecCopyFreeCompatible} with the same 3098 * {@code codecNames} returned true, the returned 3099 * {@link LinearBlock} object can be queued to the listed codecs without 3100 * copying. The returned {@link LinearBlock} object is always 3101 * read/write mappable. 3102 * 3103 * @param capacity requested capacity of the linear block in bytes 3104 * @param codecNames list of codecs that the client wants to use this 3105 * linear block without copying. Null entries are 3106 * ignored. 3107 * @return a linear block object. 3108 * @throws IllegalArgumentException if the capacity is invalid or 3109 * codecNames contains invalid name 3110 * @throws IOException if an error occurred while allocating a buffer 3111 */ obtain( int capacity, @NonNull String[] codecNames)3112 public static @Nullable LinearBlock obtain( 3113 int capacity, @NonNull String[] codecNames) { 3114 LinearBlock buffer = sPool.poll(); 3115 if (buffer == null) { 3116 buffer = new LinearBlock(); 3117 } 3118 synchronized (buffer.mLock) { 3119 buffer.native_obtain(capacity, codecNames); 3120 } 3121 return buffer; 3122 } 3123 3124 // Called from native setInternalStateLocked(long context, boolean isMappable)3125 private void setInternalStateLocked(long context, boolean isMappable) { 3126 mNativeContext = context; 3127 mMappable = isMappable; 3128 mValid = (context != 0); 3129 } 3130 3131 private static final BlockingQueue<LinearBlock> sPool = 3132 new LinkedBlockingQueue<>(); 3133 3134 private final Object mLock = new Object(); 3135 private boolean mValid = false; 3136 private boolean mMappable = false; 3137 private ByteBuffer mMapped = null; 3138 private long mNativeContext = 0; 3139 } 3140 3141 /** 3142 * Map a {@link HardwareBuffer} object into {@link Image}, so that the content of the buffer is 3143 * accessible. Depending on the usage and pixel format of the hardware buffer, it may not be 3144 * mappable; this method returns null in that case. 3145 * 3146 * @param hardwareBuffer {@link HardwareBuffer} to map. 3147 * @return Mapped {@link Image} object, or null if the buffer is not mappable. 3148 */ mapHardwareBuffer(@onNull HardwareBuffer hardwareBuffer)3149 public static @Nullable Image mapHardwareBuffer(@NonNull HardwareBuffer hardwareBuffer) { 3150 return native_mapHardwareBuffer(hardwareBuffer); 3151 } 3152 native_mapHardwareBuffer( @onNull HardwareBuffer hardwareBuffer)3153 private static native @Nullable Image native_mapHardwareBuffer( 3154 @NonNull HardwareBuffer hardwareBuffer); 3155 native_closeMediaImage(long context)3156 private static native void native_closeMediaImage(long context); 3157 3158 /** 3159 * Builder-like class for queue requests. Use this class to prepare a 3160 * queue request and send it. 3161 */ 3162 public final class QueueRequest { 3163 // No public constructor QueueRequest(@onNull MediaCodec codec, int index)3164 private QueueRequest(@NonNull MediaCodec codec, int index) { 3165 mCodec = codec; 3166 mIndex = index; 3167 } 3168 3169 /** 3170 * Set a linear block to this queue request. Exactly one buffer must be 3171 * set for a queue request before calling {@link #queue}. It is possible 3172 * to use the same {@link LinearBlock} object for multiple queue 3173 * requests. The behavior is undefined if the range of the buffer 3174 * overlaps for multiple requests, or the application writes into the 3175 * region being processed by the codec. 3176 * 3177 * @param block The linear block object 3178 * @param offset The byte offset into the input buffer at which the data starts. 3179 * @param size The number of bytes of valid input data. 3180 * @return this object 3181 * @throws IllegalStateException if a buffer is already set 3182 */ setLinearBlock( @onNull LinearBlock block, int offset, int size)3183 public @NonNull QueueRequest setLinearBlock( 3184 @NonNull LinearBlock block, 3185 int offset, 3186 int size) { 3187 if (!isAccessible()) { 3188 throw new IllegalStateException("The request is stale"); 3189 } 3190 if (mLinearBlock != null || mHardwareBuffer != null) { 3191 throw new IllegalStateException("Cannot set block twice"); 3192 } 3193 mLinearBlock = block; 3194 mOffset = offset; 3195 mSize = size; 3196 mCryptoInfo = null; 3197 return this; 3198 } 3199 3200 /** 3201 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3202 * set for a queue request before calling {@link #queue}. It is possible 3203 * to use the same {@link LinearBlock} object for multiple queue 3204 * requests. The behavior is undefined if the range of the buffer 3205 * overlaps for multiple requests, or the application writes into the 3206 * region being processed by the codec. 3207 * 3208 * @param block The linear block object 3209 * @param offset The byte offset into the input buffer at which the data starts. 3210 * @param size The number of bytes of valid input data. 3211 * @param cryptoInfo Metadata describing the structure of the encrypted input sample. 3212 * @return this object 3213 * @throws IllegalStateException if a buffer is already set 3214 */ setEncryptedLinearBlock( @onNull LinearBlock block, int offset, int size, @NonNull MediaCodec.CryptoInfo cryptoInfo)3215 public @NonNull QueueRequest setEncryptedLinearBlock( 3216 @NonNull LinearBlock block, 3217 int offset, 3218 int size, 3219 @NonNull MediaCodec.CryptoInfo cryptoInfo) { 3220 Objects.requireNonNull(cryptoInfo); 3221 if (!isAccessible()) { 3222 throw new IllegalStateException("The request is stale"); 3223 } 3224 if (mLinearBlock != null || mHardwareBuffer != null) { 3225 throw new IllegalStateException("Cannot set block twice"); 3226 } 3227 mLinearBlock = block; 3228 mOffset = offset; 3229 mSize = size; 3230 mCryptoInfo = cryptoInfo; 3231 return this; 3232 } 3233 3234 /** 3235 * Set a harware graphic buffer to this queue request. Exactly one buffer must 3236 * be set for a queue request before calling {@link #queue}. 3237 * <p> 3238 * Note: buffers should have format {@link HardwareBuffer#YCBCR_420_888}, 3239 * a single layer, and an appropriate usage ({@link HardwareBuffer#USAGE_CPU_READ_OFTEN} 3240 * for software codecs and {@link HardwareBuffer#USAGE_VIDEO_ENCODE} for hardware) 3241 * for codecs to recognize. Codecs may throw exception if the buffer is not recognizable. 3242 * 3243 * @param buffer The hardware graphic buffer object 3244 * @return this object 3245 * @throws IllegalStateException if a buffer is already set 3246 */ setHardwareBuffer( @onNull HardwareBuffer buffer)3247 public @NonNull QueueRequest setHardwareBuffer( 3248 @NonNull HardwareBuffer buffer) { 3249 if (!isAccessible()) { 3250 throw new IllegalStateException("The request is stale"); 3251 } 3252 if (mLinearBlock != null || mHardwareBuffer != null) { 3253 throw new IllegalStateException("Cannot set block twice"); 3254 } 3255 mHardwareBuffer = buffer; 3256 return this; 3257 } 3258 3259 /** 3260 * Set timestamp to this queue request. 3261 * 3262 * @param presentationTimeUs The presentation timestamp in microseconds for this 3263 * buffer. This is normally the media time at which this 3264 * buffer should be presented (rendered). When using an output 3265 * surface, this will be propagated as the {@link 3266 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3267 * conversion to nanoseconds). 3268 * @return this object 3269 */ setPresentationTimeUs(long presentationTimeUs)3270 public @NonNull QueueRequest setPresentationTimeUs(long presentationTimeUs) { 3271 if (!isAccessible()) { 3272 throw new IllegalStateException("The request is stale"); 3273 } 3274 mPresentationTimeUs = presentationTimeUs; 3275 return this; 3276 } 3277 3278 /** 3279 * Set flags to this queue request. 3280 * 3281 * @param flags A bitmask of flags 3282 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3283 * While not prohibited, most codecs do not use the 3284 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3285 * @return this object 3286 */ setFlags(@ufferFlag int flags)3287 public @NonNull QueueRequest setFlags(@BufferFlag int flags) { 3288 if (!isAccessible()) { 3289 throw new IllegalStateException("The request is stale"); 3290 } 3291 mFlags = flags; 3292 return this; 3293 } 3294 3295 /** 3296 * Add an integer parameter. 3297 * See {@link MediaFormat} for an exhaustive list of supported keys with 3298 * values of type int, that can also be set with {@link MediaFormat#setInteger}. 3299 * 3300 * If there was {@link MediaCodec#setParameters} 3301 * call with the same key which is not processed by the codec yet, the 3302 * value set from this method will override the unprocessed value. 3303 * 3304 * @return this object 3305 */ setIntegerParameter( @onNull String key, int value)3306 public @NonNull QueueRequest setIntegerParameter( 3307 @NonNull String key, int value) { 3308 if (!isAccessible()) { 3309 throw new IllegalStateException("The request is stale"); 3310 } 3311 mTuningKeys.add(key); 3312 mTuningValues.add(Integer.valueOf(value)); 3313 return this; 3314 } 3315 3316 /** 3317 * Add a long parameter. 3318 * See {@link MediaFormat} for an exhaustive list of supported keys with 3319 * values of type long, that can also be set with {@link MediaFormat#setLong}. 3320 * 3321 * If there was {@link MediaCodec#setParameters} 3322 * call with the same key which is not processed by the codec yet, the 3323 * value set from this method will override the unprocessed value. 3324 * 3325 * @return this object 3326 */ setLongParameter( @onNull String key, long value)3327 public @NonNull QueueRequest setLongParameter( 3328 @NonNull String key, long value) { 3329 if (!isAccessible()) { 3330 throw new IllegalStateException("The request is stale"); 3331 } 3332 mTuningKeys.add(key); 3333 mTuningValues.add(Long.valueOf(value)); 3334 return this; 3335 } 3336 3337 /** 3338 * Add a float parameter. 3339 * See {@link MediaFormat} for an exhaustive list of supported keys with 3340 * values of type float, that can also be set with {@link MediaFormat#setFloat}. 3341 * 3342 * If there was {@link MediaCodec#setParameters} 3343 * call with the same key which is not processed by the codec yet, the 3344 * value set from this method will override the unprocessed value. 3345 * 3346 * @return this object 3347 */ setFloatParameter( @onNull String key, float value)3348 public @NonNull QueueRequest setFloatParameter( 3349 @NonNull String key, float value) { 3350 if (!isAccessible()) { 3351 throw new IllegalStateException("The request is stale"); 3352 } 3353 mTuningKeys.add(key); 3354 mTuningValues.add(Float.valueOf(value)); 3355 return this; 3356 } 3357 3358 /** 3359 * Add a {@link ByteBuffer} parameter. 3360 * See {@link MediaFormat} for an exhaustive list of supported keys with 3361 * values of byte buffer, that can also be set with {@link MediaFormat#setByteBuffer}. 3362 * 3363 * If there was {@link MediaCodec#setParameters} 3364 * call with the same key which is not processed by the codec yet, the 3365 * value set from this method will override the unprocessed value. 3366 * 3367 * @return this object 3368 */ setByteBufferParameter( @onNull String key, @NonNull ByteBuffer value)3369 public @NonNull QueueRequest setByteBufferParameter( 3370 @NonNull String key, @NonNull ByteBuffer value) { 3371 if (!isAccessible()) { 3372 throw new IllegalStateException("The request is stale"); 3373 } 3374 mTuningKeys.add(key); 3375 mTuningValues.add(value); 3376 return this; 3377 } 3378 3379 /** 3380 * Add a string parameter. 3381 * See {@link MediaFormat} for an exhaustive list of supported keys with 3382 * values of type string, that can also be set with {@link MediaFormat#setString}. 3383 * 3384 * If there was {@link MediaCodec#setParameters} 3385 * call with the same key which is not processed by the codec yet, the 3386 * value set from this method will override the unprocessed value. 3387 * 3388 * @return this object 3389 */ setStringParameter( @onNull String key, @NonNull String value)3390 public @NonNull QueueRequest setStringParameter( 3391 @NonNull String key, @NonNull String value) { 3392 if (!isAccessible()) { 3393 throw new IllegalStateException("The request is stale"); 3394 } 3395 mTuningKeys.add(key); 3396 mTuningValues.add(value); 3397 return this; 3398 } 3399 3400 /** 3401 * Finish building a queue request and queue the buffers with tunings. 3402 */ queue()3403 public void queue() { 3404 if (!isAccessible()) { 3405 throw new IllegalStateException("The request is stale"); 3406 } 3407 if (mLinearBlock == null && mHardwareBuffer == null) { 3408 throw new IllegalStateException("No block is set"); 3409 } 3410 setAccessible(false); 3411 if (mLinearBlock != null) { 3412 mCodec.native_queueLinearBlock( 3413 mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo, 3414 mPresentationTimeUs, mFlags, 3415 mTuningKeys, mTuningValues); 3416 } else if (mHardwareBuffer != null) { 3417 mCodec.native_queueHardwareBuffer( 3418 mIndex, mHardwareBuffer, mPresentationTimeUs, mFlags, 3419 mTuningKeys, mTuningValues); 3420 } 3421 clear(); 3422 } 3423 clear()3424 @NonNull QueueRequest clear() { 3425 mLinearBlock = null; 3426 mOffset = 0; 3427 mSize = 0; 3428 mCryptoInfo = null; 3429 mHardwareBuffer = null; 3430 mPresentationTimeUs = 0; 3431 mFlags = 0; 3432 mTuningKeys.clear(); 3433 mTuningValues.clear(); 3434 return this; 3435 } 3436 isAccessible()3437 boolean isAccessible() { 3438 return mAccessible; 3439 } 3440 setAccessible(boolean accessible)3441 @NonNull QueueRequest setAccessible(boolean accessible) { 3442 mAccessible = accessible; 3443 return this; 3444 } 3445 3446 private final MediaCodec mCodec; 3447 private final int mIndex; 3448 private LinearBlock mLinearBlock = null; 3449 private int mOffset = 0; 3450 private int mSize = 0; 3451 private MediaCodec.CryptoInfo mCryptoInfo = null; 3452 private HardwareBuffer mHardwareBuffer = null; 3453 private long mPresentationTimeUs = 0; 3454 private @BufferFlag int mFlags = 0; 3455 private final ArrayList<String> mTuningKeys = new ArrayList<>(); 3456 private final ArrayList<Object> mTuningValues = new ArrayList<>(); 3457 3458 private boolean mAccessible = false; 3459 } 3460 native_queueLinearBlock( int index, @NonNull LinearBlock block, int offset, int size, @Nullable CryptoInfo cryptoInfo, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3461 private native void native_queueLinearBlock( 3462 int index, 3463 @NonNull LinearBlock block, 3464 int offset, 3465 int size, 3466 @Nullable CryptoInfo cryptoInfo, 3467 long presentationTimeUs, 3468 int flags, 3469 @NonNull ArrayList<String> keys, 3470 @NonNull ArrayList<Object> values); 3471 native_queueHardwareBuffer( int index, @NonNull HardwareBuffer buffer, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3472 private native void native_queueHardwareBuffer( 3473 int index, 3474 @NonNull HardwareBuffer buffer, 3475 long presentationTimeUs, 3476 int flags, 3477 @NonNull ArrayList<String> keys, 3478 @NonNull ArrayList<Object> values); 3479 3480 private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); 3481 3482 /** 3483 * Return a {@link QueueRequest} object for an input slot index. 3484 * 3485 * @param index input slot index from 3486 * {@link Callback#onInputBufferAvailable} 3487 * @return queue request object 3488 * @throws IllegalStateException if not using block model 3489 * @throws IllegalArgumentException if the input slot is not available or 3490 * the index is out of range 3491 */ getQueueRequest(int index)3492 public @NonNull QueueRequest getQueueRequest(int index) { 3493 synchronized (mBufferLock) { 3494 if (mBufferMode != BUFFER_MODE_BLOCK) { 3495 throw new IllegalStateException("The codec is not configured for block model"); 3496 } 3497 if (index < 0 || index >= mQueueRequests.size()) { 3498 throw new IndexOutOfBoundsException("Expected range of index: [0," 3499 + (mQueueRequests.size() - 1) + "]; actual: " + index); 3500 } 3501 QueueRequest request = mQueueRequests.get(index); 3502 if (request == null) { 3503 throw new IllegalArgumentException("Unavailable index: " + index); 3504 } 3505 if (!request.isAccessible()) { 3506 throw new IllegalArgumentException( 3507 "The request is stale at index " + index); 3508 } 3509 return request.clear(); 3510 } 3511 } 3512 3513 /** 3514 * If a non-negative timeout had been specified in the call 3515 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 3516 */ 3517 public static final int INFO_TRY_AGAIN_LATER = -1; 3518 3519 /** 3520 * The output format has changed, subsequent data will follow the new 3521 * format. {@link #getOutputFormat()} returns the new format. Note, that 3522 * you can also use the new {@link #getOutputFormat(int)} method to 3523 * get the format for a specific output buffer. This frees you from 3524 * having to track output format changes. 3525 */ 3526 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 3527 3528 /** 3529 * The output buffers have changed, the client must refer to the new 3530 * set of output buffers returned by {@link #getOutputBuffers} from 3531 * this point on. 3532 * 3533 * <p>Additionally, this event signals that the video scaling mode 3534 * may have been reset to the default.</p> 3535 * 3536 * @deprecated This return value can be ignored as {@link 3537 * #getOutputBuffers} has been deprecated. Client should 3538 * request a current buffer using on of the get-buffer or 3539 * get-image methods each time one has been dequeued. 3540 */ 3541 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 3542 3543 /** @hide */ 3544 @IntDef({ 3545 INFO_TRY_AGAIN_LATER, 3546 INFO_OUTPUT_FORMAT_CHANGED, 3547 INFO_OUTPUT_BUFFERS_CHANGED, 3548 }) 3549 @Retention(RetentionPolicy.SOURCE) 3550 public @interface OutputBufferInfo {} 3551 3552 /** 3553 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 3554 * Returns the index of an output buffer that has been successfully 3555 * decoded or one of the INFO_* constants. 3556 * @param info Will be filled with buffer meta data. 3557 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3558 * @throws IllegalStateException if not in the Executing state, 3559 * or codec is configured in asynchronous mode. 3560 * @throws MediaCodec.CodecException upon codec error. 3561 */ 3562 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3563 public final int dequeueOutputBuffer( 3564 @NonNull BufferInfo info, long timeoutUs) { 3565 synchronized (mBufferLock) { 3566 if (mBufferMode == BUFFER_MODE_BLOCK) { 3567 throw new IncompatibleWithBlockModelException("dequeueOutputBuffer() " 3568 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3569 + "Please use MediaCodec.Callback objects to get output buffer slots."); 3570 } 3571 } 3572 int res = native_dequeueOutputBuffer(info, timeoutUs); 3573 synchronized (mBufferLock) { 3574 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 3575 cacheBuffersLocked(false /* input */); 3576 } else if (res >= 0) { 3577 validateOutputByteBufferLocked(mCachedOutputBuffers, res, info); 3578 if (mHasSurface || mCachedOutputBuffers == null) { 3579 mDequeuedOutputInfos.put(res, info.dup()); 3580 } 3581 } 3582 } 3583 return res; 3584 } 3585 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3586 private native final int native_dequeueOutputBuffer( 3587 @NonNull BufferInfo info, long timeoutUs); 3588 3589 /** 3590 * If you are done with a buffer, use this call to return the buffer to the codec 3591 * or to render it on the output surface. If you configured the codec with an 3592 * output surface, setting {@code render} to {@code true} will first send the buffer 3593 * to that output surface. The surface will release the buffer back to the codec once 3594 * it is no longer used/displayed. 3595 * 3596 * Once an output buffer is released to the codec, it MUST NOT 3597 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3598 * to a {@link #dequeueOutputBuffer} return value or a 3599 * {@link Callback#onOutputBufferAvailable} callback. 3600 * 3601 * @param index The index of a client-owned output buffer previously returned 3602 * from a call to {@link #dequeueOutputBuffer}. 3603 * @param render If a valid surface was specified when configuring the codec, 3604 * passing true renders this output buffer to the surface. 3605 * @throws IllegalStateException if not in the Executing state. 3606 * @throws MediaCodec.CodecException upon codec error. 3607 */ releaseOutputBuffer(int index, boolean render)3608 public final void releaseOutputBuffer(int index, boolean render) { 3609 releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); 3610 } 3611 3612 /** 3613 * If you are done with a buffer, use this call to update its surface timestamp 3614 * and return it to the codec to render it on the output surface. If you 3615 * have not specified an output surface when configuring this video codec, 3616 * this call will simply return the buffer to the codec.<p> 3617 * 3618 * The timestamp may have special meaning depending on the destination surface. 3619 * 3620 * <table> 3621 * <tr><th>SurfaceView specifics</th></tr> 3622 * <tr><td> 3623 * If you render your buffer on a {@link android.view.SurfaceView}, 3624 * you can use the timestamp to render the buffer at a specific time (at the 3625 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 3626 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 3627 * Currently, this is set as within one (1) second. A few notes: 3628 * 3629 * <ul> 3630 * <li>the buffer will not be returned to the codec until the timestamp 3631 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 3632 * <li>buffers are processed sequentially, so you may block subsequent buffers to 3633 * be displayed on the {@link android.view.Surface}. This is important if you 3634 * want to react to user action, e.g. stop the video or seek. 3635 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 3636 * rendered at the same VSYNC, the last one will be shown, and the other ones 3637 * will be dropped. 3638 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 3639 * time, the {@link android.view.Surface} will ignore the timestamp, and 3640 * display the buffer at the earliest feasible time. In this mode it will not 3641 * drop frames. 3642 * <li>for best performance and quality, call this method when you are about 3643 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 3644 * about 33 msec. 3645 * </ul> 3646 * </td></tr> 3647 * </table> 3648 * 3649 * Once an output buffer is released to the codec, it MUST NOT 3650 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3651 * to a {@link #dequeueOutputBuffer} return value or a 3652 * {@link Callback#onOutputBufferAvailable} callback. 3653 * 3654 * @param index The index of a client-owned output buffer previously returned 3655 * from a call to {@link #dequeueOutputBuffer}. 3656 * @param renderTimestampNs The timestamp to associate with this buffer when 3657 * it is sent to the Surface. 3658 * @throws IllegalStateException if not in the Executing state. 3659 * @throws MediaCodec.CodecException upon codec error. 3660 */ releaseOutputBuffer(int index, long renderTimestampNs)3661 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 3662 releaseOutputBufferInternal( 3663 index, true /* render */, true /* updatePTS */, renderTimestampNs); 3664 } 3665 releaseOutputBufferInternal( int index, boolean render, boolean updatePts, long renderTimestampNs)3666 private void releaseOutputBufferInternal( 3667 int index, boolean render, boolean updatePts, long renderTimestampNs) { 3668 BufferInfo info = null; 3669 synchronized(mBufferLock) { 3670 switch (mBufferMode) { 3671 case BUFFER_MODE_LEGACY: 3672 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 3673 mDequeuedOutputBuffers.remove(index); 3674 if (mHasSurface || mCachedOutputBuffers == null) { 3675 info = mDequeuedOutputInfos.remove(index); 3676 } 3677 break; 3678 case BUFFER_MODE_BLOCK: 3679 OutputFrame frame = mOutputFrames.get(index); 3680 frame.setAccessible(false); 3681 frame.clear(); 3682 break; 3683 default: 3684 throw new IllegalStateException( 3685 "Unrecognized buffer mode: " + mBufferMode); 3686 } 3687 } 3688 releaseOutputBuffer( 3689 index, render, updatePts, renderTimestampNs); 3690 } 3691 3692 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)3693 private native final void releaseOutputBuffer( 3694 int index, boolean render, boolean updatePTS, long timeNs); 3695 3696 /** 3697 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 3698 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 3699 * encoders receiving input from a Surface created by {@link #createInputSurface}. 3700 * @throws IllegalStateException if not in the Executing state. 3701 * @throws MediaCodec.CodecException upon codec error. 3702 */ signalEndOfInputStream()3703 public native final void signalEndOfInputStream(); 3704 3705 /** 3706 * Call this after dequeueOutputBuffer signals a format change by returning 3707 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 3708 * You can also call this after {@link #configure} returns 3709 * successfully to get the output format initially configured 3710 * for the codec. Do this to determine what optional 3711 * configuration parameters were supported by the codec. 3712 * 3713 * @throws IllegalStateException if not in the Executing or 3714 * Configured state. 3715 * @throws MediaCodec.CodecException upon codec error. 3716 */ 3717 @NonNull getOutputFormat()3718 public final MediaFormat getOutputFormat() { 3719 return new MediaFormat(getFormatNative(false /* input */)); 3720 } 3721 3722 /** 3723 * Call this after {@link #configure} returns successfully to 3724 * get the input format accepted by the codec. Do this to 3725 * determine what optional configuration parameters were 3726 * supported by the codec. 3727 * 3728 * @throws IllegalStateException if not in the Executing or 3729 * Configured state. 3730 * @throws MediaCodec.CodecException upon codec error. 3731 */ 3732 @NonNull getInputFormat()3733 public final MediaFormat getInputFormat() { 3734 return new MediaFormat(getFormatNative(true /* input */)); 3735 } 3736 3737 /** 3738 * Returns the output format for a specific output buffer. 3739 * 3740 * @param index The index of a client-owned input buffer previously 3741 * returned from a call to {@link #dequeueInputBuffer}. 3742 * 3743 * @return the format for the output buffer, or null if the index 3744 * is not a dequeued output buffer. 3745 */ 3746 @NonNull getOutputFormat(int index)3747 public final MediaFormat getOutputFormat(int index) { 3748 return new MediaFormat(getOutputFormatNative(index)); 3749 } 3750 3751 @NonNull getFormatNative(boolean input)3752 private native final Map<String, Object> getFormatNative(boolean input); 3753 3754 @NonNull getOutputFormatNative(int index)3755 private native final Map<String, Object> getOutputFormatNative(int index); 3756 3757 // used to track dequeued buffers 3758 private static class BufferMap { 3759 // various returned representations of the codec buffer 3760 private static class CodecBuffer { 3761 private Image mImage; 3762 private ByteBuffer mByteBuffer; 3763 free()3764 public void free() { 3765 if (mByteBuffer != null) { 3766 // all of our ByteBuffers are direct 3767 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 3768 mByteBuffer = null; 3769 } 3770 if (mImage != null) { 3771 mImage.close(); 3772 mImage = null; 3773 } 3774 } 3775 setImage(@ullable Image image)3776 public void setImage(@Nullable Image image) { 3777 free(); 3778 mImage = image; 3779 } 3780 setByteBuffer(@ullable ByteBuffer buffer)3781 public void setByteBuffer(@Nullable ByteBuffer buffer) { 3782 free(); 3783 mByteBuffer = buffer; 3784 } 3785 } 3786 3787 private final Map<Integer, CodecBuffer> mMap = 3788 new HashMap<Integer, CodecBuffer>(); 3789 remove(int index)3790 public void remove(int index) { 3791 CodecBuffer buffer = mMap.get(index); 3792 if (buffer != null) { 3793 buffer.free(); 3794 mMap.remove(index); 3795 } 3796 } 3797 put(int index, @Nullable ByteBuffer newBuffer)3798 public void put(int index, @Nullable ByteBuffer newBuffer) { 3799 CodecBuffer buffer = mMap.get(index); 3800 if (buffer == null) { // likely 3801 buffer = new CodecBuffer(); 3802 mMap.put(index, buffer); 3803 } 3804 buffer.setByteBuffer(newBuffer); 3805 } 3806 put(int index, @Nullable Image newImage)3807 public void put(int index, @Nullable Image newImage) { 3808 CodecBuffer buffer = mMap.get(index); 3809 if (buffer == null) { // likely 3810 buffer = new CodecBuffer(); 3811 mMap.put(index, buffer); 3812 } 3813 buffer.setImage(newImage); 3814 } 3815 clear()3816 public void clear() { 3817 for (CodecBuffer buffer: mMap.values()) { 3818 buffer.free(); 3819 } 3820 mMap.clear(); 3821 } 3822 } 3823 3824 private ByteBuffer[] mCachedInputBuffers; 3825 private ByteBuffer[] mCachedOutputBuffers; 3826 private BitSet mValidInputIndices = new BitSet(); 3827 private BitSet mValidOutputIndices = new BitSet(); 3828 3829 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 3830 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 3831 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 3832 new HashMap<Integer, BufferInfo>(); 3833 final private Object mBufferLock; 3834 invalidateByteBufferLocked( @ullable ByteBuffer[] buffers, int index, boolean input)3835 private void invalidateByteBufferLocked( 3836 @Nullable ByteBuffer[] buffers, int index, boolean input) { 3837 if (buffers == null) { 3838 if (index >= 0) { 3839 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 3840 indices.clear(index); 3841 } 3842 } else if (index >= 0 && index < buffers.length) { 3843 ByteBuffer buffer = buffers[index]; 3844 if (buffer != null) { 3845 buffer.setAccessible(false); 3846 } 3847 } 3848 } 3849 validateInputByteBufferLocked( @ullable ByteBuffer[] buffers, int index)3850 private void validateInputByteBufferLocked( 3851 @Nullable ByteBuffer[] buffers, int index) { 3852 if (buffers == null) { 3853 if (index >= 0) { 3854 mValidInputIndices.set(index); 3855 } 3856 } else if (index >= 0 && index < buffers.length) { 3857 ByteBuffer buffer = buffers[index]; 3858 if (buffer != null) { 3859 buffer.setAccessible(true); 3860 buffer.clear(); 3861 } 3862 } 3863 } 3864 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index, boolean input)3865 private void revalidateByteBuffer( 3866 @Nullable ByteBuffer[] buffers, int index, boolean input) { 3867 synchronized(mBufferLock) { 3868 if (buffers == null) { 3869 if (index >= 0) { 3870 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 3871 indices.set(index); 3872 } 3873 } else if (index >= 0 && index < buffers.length) { 3874 ByteBuffer buffer = buffers[index]; 3875 if (buffer != null) { 3876 buffer.setAccessible(true); 3877 } 3878 } 3879 } 3880 } 3881 validateOutputByteBufferLocked( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)3882 private void validateOutputByteBufferLocked( 3883 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 3884 if (buffers == null) { 3885 if (index >= 0) { 3886 mValidOutputIndices.set(index); 3887 } 3888 } else if (index >= 0 && index < buffers.length) { 3889 ByteBuffer buffer = buffers[index]; 3890 if (buffer != null) { 3891 buffer.setAccessible(true); 3892 buffer.limit(info.offset + info.size).position(info.offset); 3893 } 3894 } 3895 } 3896 invalidateByteBuffersLocked(@ullable ByteBuffer[] buffers)3897 private void invalidateByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 3898 if (buffers != null) { 3899 for (ByteBuffer buffer: buffers) { 3900 if (buffer != null) { 3901 buffer.setAccessible(false); 3902 } 3903 } 3904 } 3905 } 3906 freeByteBufferLocked(@ullable ByteBuffer buffer)3907 private void freeByteBufferLocked(@Nullable ByteBuffer buffer) { 3908 if (buffer != null /* && buffer.isDirect() */) { 3909 // all of our ByteBuffers are direct 3910 java.nio.NioUtils.freeDirectBuffer(buffer); 3911 } 3912 } 3913 freeByteBuffersLocked(@ullable ByteBuffer[] buffers)3914 private void freeByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 3915 if (buffers != null) { 3916 for (ByteBuffer buffer: buffers) { 3917 freeByteBufferLocked(buffer); 3918 } 3919 } 3920 } 3921 freeAllTrackedBuffers()3922 private void freeAllTrackedBuffers() { 3923 synchronized(mBufferLock) { 3924 freeByteBuffersLocked(mCachedInputBuffers); 3925 freeByteBuffersLocked(mCachedOutputBuffers); 3926 mCachedInputBuffers = null; 3927 mCachedOutputBuffers = null; 3928 mValidInputIndices.clear(); 3929 mValidOutputIndices.clear(); 3930 mDequeuedInputBuffers.clear(); 3931 mDequeuedOutputBuffers.clear(); 3932 mQueueRequests.clear(); 3933 mOutputFrames.clear(); 3934 } 3935 } 3936 cacheBuffersLocked(boolean input)3937 private void cacheBuffersLocked(boolean input) { 3938 ByteBuffer[] buffers = null; 3939 try { 3940 buffers = getBuffers(input); 3941 invalidateByteBuffersLocked(buffers); 3942 } catch (IllegalStateException e) { 3943 // we don't get buffers in async mode 3944 } 3945 if (buffers != null) { 3946 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 3947 for (int i = 0; i < buffers.length; ++i) { 3948 ByteBuffer buffer = buffers[i]; 3949 if (buffer == null || !indices.get(i)) { 3950 continue; 3951 } 3952 buffer.setAccessible(true); 3953 if (!input) { 3954 BufferInfo info = mDequeuedOutputInfos.get(i); 3955 if (info != null) { 3956 buffer.limit(info.offset + info.size).position(info.offset); 3957 } 3958 } 3959 } 3960 indices.clear(); 3961 } 3962 if (input) { 3963 mCachedInputBuffers = buffers; 3964 } else { 3965 mCachedOutputBuffers = buffers; 3966 } 3967 } 3968 3969 /** 3970 * Retrieve the set of input buffers. Call this after start() 3971 * returns. After calling this method, any ByteBuffers 3972 * previously returned by an earlier call to this method MUST no 3973 * longer be used. 3974 * 3975 * @deprecated Use the new {@link #getInputBuffer} method instead 3976 * each time an input buffer is dequeued. 3977 * 3978 * <b>Note:</b> As of API 21, dequeued input buffers are 3979 * automatically {@link java.nio.Buffer#clear cleared}. 3980 * 3981 * <em>Do not use this method if using an input surface.</em> 3982 * 3983 * @throws IllegalStateException if not in the Executing state, 3984 * or codec is configured in asynchronous mode. 3985 * @throws MediaCodec.CodecException upon codec error. 3986 */ 3987 @NonNull getInputBuffers()3988 public ByteBuffer[] getInputBuffers() { 3989 synchronized (mBufferLock) { 3990 if (mBufferMode == BUFFER_MODE_BLOCK) { 3991 throw new IncompatibleWithBlockModelException("getInputBuffers() " 3992 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3993 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 3994 + "objects and attach to QueueRequest objects."); 3995 } 3996 if (mCachedInputBuffers == null) { 3997 cacheBuffersLocked(true /* input */); 3998 } 3999 if (mCachedInputBuffers == null) { 4000 throw new IllegalStateException(); 4001 } 4002 // FIXME: check codec status 4003 return mCachedInputBuffers; 4004 } 4005 } 4006 4007 /** 4008 * Retrieve the set of output buffers. Call this after start() 4009 * returns and whenever dequeueOutputBuffer signals an output 4010 * buffer change by returning {@link 4011 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 4012 * ByteBuffers previously returned by an earlier call to this 4013 * method MUST no longer be used. 4014 * 4015 * @deprecated Use the new {@link #getOutputBuffer} method instead 4016 * each time an output buffer is dequeued. This method is not 4017 * supported if codec is configured in asynchronous mode. 4018 * 4019 * <b>Note:</b> As of API 21, the position and limit of output 4020 * buffers that are dequeued will be set to the valid data 4021 * range. 4022 * 4023 * <em>Do not use this method if using an output surface.</em> 4024 * 4025 * @throws IllegalStateException if not in the Executing state, 4026 * or codec is configured in asynchronous mode. 4027 * @throws MediaCodec.CodecException upon codec error. 4028 */ 4029 @NonNull getOutputBuffers()4030 public ByteBuffer[] getOutputBuffers() { 4031 synchronized (mBufferLock) { 4032 if (mBufferMode == BUFFER_MODE_BLOCK) { 4033 throw new IncompatibleWithBlockModelException("getOutputBuffers() " 4034 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4035 + "Please use getOutputFrame to get output frames."); 4036 } 4037 if (mCachedOutputBuffers == null) { 4038 cacheBuffersLocked(false /* input */); 4039 } 4040 if (mCachedOutputBuffers == null) { 4041 throw new IllegalStateException(); 4042 } 4043 // FIXME: check codec status 4044 return mCachedOutputBuffers; 4045 } 4046 } 4047 4048 /** 4049 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 4050 * object for a dequeued input buffer index to contain the input data. 4051 * 4052 * After calling this method any ByteBuffer or Image object 4053 * previously returned for the same input index MUST no longer 4054 * be used. 4055 * 4056 * @param index The index of a client-owned input buffer previously 4057 * returned from a call to {@link #dequeueInputBuffer}, 4058 * or received via an onInputBufferAvailable callback. 4059 * 4060 * @return the input buffer, or null if the index is not a dequeued 4061 * input buffer, or if the codec is configured for surface input. 4062 * 4063 * @throws IllegalStateException if not in the Executing state. 4064 * @throws MediaCodec.CodecException upon codec error. 4065 */ 4066 @Nullable getInputBuffer(int index)4067 public ByteBuffer getInputBuffer(int index) { 4068 synchronized (mBufferLock) { 4069 if (mBufferMode == BUFFER_MODE_BLOCK) { 4070 throw new IncompatibleWithBlockModelException("getInputBuffer() " 4071 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4072 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4073 + "objects and attach to QueueRequest objects."); 4074 } 4075 } 4076 ByteBuffer newBuffer = getBuffer(true /* input */, index); 4077 synchronized (mBufferLock) { 4078 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4079 mDequeuedInputBuffers.put(index, newBuffer); 4080 } 4081 return newBuffer; 4082 } 4083 4084 /** 4085 * Returns a writable Image object for a dequeued input buffer 4086 * index to contain the raw input video frame. 4087 * 4088 * After calling this method any ByteBuffer or Image object 4089 * previously returned for the same input index MUST no longer 4090 * be used. 4091 * 4092 * @param index The index of a client-owned input buffer previously 4093 * returned from a call to {@link #dequeueInputBuffer}, 4094 * or received via an onInputBufferAvailable callback. 4095 * 4096 * @return the input image, or null if the index is not a 4097 * dequeued input buffer, or not a ByteBuffer that contains a 4098 * raw image. 4099 * 4100 * @throws IllegalStateException if not in the Executing state. 4101 * @throws MediaCodec.CodecException upon codec error. 4102 */ 4103 @Nullable getInputImage(int index)4104 public Image getInputImage(int index) { 4105 synchronized (mBufferLock) { 4106 if (mBufferMode == BUFFER_MODE_BLOCK) { 4107 throw new IncompatibleWithBlockModelException("getInputImage() " 4108 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4109 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4110 + "objects and attach to QueueRequest objects."); 4111 } 4112 } 4113 Image newImage = getImage(true /* input */, index); 4114 synchronized (mBufferLock) { 4115 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4116 mDequeuedInputBuffers.put(index, newImage); 4117 } 4118 return newImage; 4119 } 4120 4121 /** 4122 * Returns a read-only ByteBuffer for a dequeued output buffer 4123 * index. The position and limit of the returned buffer are set 4124 * to the valid output data. 4125 * 4126 * After calling this method, any ByteBuffer or Image object 4127 * previously returned for the same output index MUST no longer 4128 * be used. 4129 * 4130 * @param index The index of a client-owned output buffer previously 4131 * returned from a call to {@link #dequeueOutputBuffer}, 4132 * or received via an onOutputBufferAvailable callback. 4133 * 4134 * @return the output buffer, or null if the index is not a dequeued 4135 * output buffer, or the codec is configured with an output surface. 4136 * 4137 * @throws IllegalStateException if not in the Executing state. 4138 * @throws MediaCodec.CodecException upon codec error. 4139 */ 4140 @Nullable getOutputBuffer(int index)4141 public ByteBuffer getOutputBuffer(int index) { 4142 synchronized (mBufferLock) { 4143 if (mBufferMode == BUFFER_MODE_BLOCK) { 4144 throw new IncompatibleWithBlockModelException("getOutputBuffer() " 4145 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4146 + "Please use getOutputFrame() to get output frames."); 4147 } 4148 } 4149 ByteBuffer newBuffer = getBuffer(false /* input */, index); 4150 synchronized (mBufferLock) { 4151 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4152 mDequeuedOutputBuffers.put(index, newBuffer); 4153 } 4154 return newBuffer; 4155 } 4156 4157 /** 4158 * Returns a read-only Image object for a dequeued output buffer 4159 * index that contains the raw video frame. 4160 * 4161 * After calling this method, any ByteBuffer or Image object previously 4162 * returned for the same output index MUST no longer be used. 4163 * 4164 * @param index The index of a client-owned output buffer previously 4165 * returned from a call to {@link #dequeueOutputBuffer}, 4166 * or received via an onOutputBufferAvailable callback. 4167 * 4168 * @return the output image, or null if the index is not a 4169 * dequeued output buffer, not a raw video frame, or if the codec 4170 * was configured with an output surface. 4171 * 4172 * @throws IllegalStateException if not in the Executing state. 4173 * @throws MediaCodec.CodecException upon codec error. 4174 */ 4175 @Nullable getOutputImage(int index)4176 public Image getOutputImage(int index) { 4177 synchronized (mBufferLock) { 4178 if (mBufferMode == BUFFER_MODE_BLOCK) { 4179 throw new IncompatibleWithBlockModelException("getOutputImage() " 4180 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4181 + "Please use getOutputFrame() to get output frames."); 4182 } 4183 } 4184 Image newImage = getImage(false /* input */, index); 4185 synchronized (mBufferLock) { 4186 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4187 mDequeuedOutputBuffers.put(index, newImage); 4188 } 4189 return newImage; 4190 } 4191 4192 /** 4193 * A single output frame and its associated metadata. 4194 */ 4195 public static final class OutputFrame { 4196 // No public constructor OutputFrame(int index)4197 OutputFrame(int index) { 4198 mIndex = index; 4199 } 4200 4201 /** 4202 * Returns the output linear block, or null if this frame is empty. 4203 * 4204 * @throws IllegalStateException if this output frame is not linear. 4205 */ getLinearBlock()4206 public @Nullable LinearBlock getLinearBlock() { 4207 if (mHardwareBuffer != null) { 4208 throw new IllegalStateException("This output frame is not linear"); 4209 } 4210 return mLinearBlock; 4211 } 4212 4213 /** 4214 * Returns the output hardware graphic buffer, or null if this frame is empty. 4215 * 4216 * @throws IllegalStateException if this output frame is not graphic. 4217 */ getHardwareBuffer()4218 public @Nullable HardwareBuffer getHardwareBuffer() { 4219 if (mLinearBlock != null) { 4220 throw new IllegalStateException("This output frame is not graphic"); 4221 } 4222 return mHardwareBuffer; 4223 } 4224 4225 /** 4226 * Returns the presentation timestamp in microseconds. 4227 */ getPresentationTimeUs()4228 public long getPresentationTimeUs() { 4229 return mPresentationTimeUs; 4230 } 4231 4232 /** 4233 * Returns the buffer flags. 4234 */ getFlags()4235 public @BufferFlag int getFlags() { 4236 return mFlags; 4237 } 4238 4239 /** 4240 * Returns a read-only {@link MediaFormat} for this frame. The returned 4241 * object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}. 4242 */ getFormat()4243 public @NonNull MediaFormat getFormat() { 4244 return mFormat; 4245 } 4246 4247 /** 4248 * Returns an unmodifiable set of the names of entries that has changed from 4249 * the previous frame. The entries may have been removed/changed/added. 4250 * Client can find out what the change is by querying {@link MediaFormat} 4251 * object returned from {@link #getFormat}. 4252 */ getChangedKeys()4253 public @NonNull Set<String> getChangedKeys() { 4254 if (mKeySet.isEmpty() && !mChangedKeys.isEmpty()) { 4255 mKeySet.addAll(mChangedKeys); 4256 } 4257 return Collections.unmodifiableSet(mKeySet); 4258 } 4259 clear()4260 void clear() { 4261 mLinearBlock = null; 4262 mHardwareBuffer = null; 4263 mFormat = null; 4264 mChangedKeys.clear(); 4265 mKeySet.clear(); 4266 mLoaded = false; 4267 } 4268 isAccessible()4269 boolean isAccessible() { 4270 return mAccessible; 4271 } 4272 setAccessible(boolean accessible)4273 void setAccessible(boolean accessible) { 4274 mAccessible = accessible; 4275 } 4276 setBufferInfo(MediaCodec.BufferInfo info)4277 void setBufferInfo(MediaCodec.BufferInfo info) { 4278 mPresentationTimeUs = info.presentationTimeUs; 4279 mFlags = info.flags; 4280 } 4281 isLoaded()4282 boolean isLoaded() { 4283 return mLoaded; 4284 } 4285 setLoaded(boolean loaded)4286 void setLoaded(boolean loaded) { 4287 mLoaded = loaded; 4288 } 4289 4290 private final int mIndex; 4291 private LinearBlock mLinearBlock = null; 4292 private HardwareBuffer mHardwareBuffer = null; 4293 private long mPresentationTimeUs = 0; 4294 private @BufferFlag int mFlags = 0; 4295 private MediaFormat mFormat = null; 4296 private final ArrayList<String> mChangedKeys = new ArrayList<>(); 4297 private final Set<String> mKeySet = new HashSet<>(); 4298 private boolean mAccessible = false; 4299 private boolean mLoaded = false; 4300 } 4301 4302 private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); 4303 4304 /** 4305 * Returns an {@link OutputFrame} object. 4306 * 4307 * @param index output buffer index from 4308 * {@link Callback#onOutputBufferAvailable} 4309 * @return {@link OutputFrame} object describing the output buffer 4310 * @throws IllegalStateException if not using block model 4311 * @throws IllegalArgumentException if the output buffer is not available or 4312 * the index is out of range 4313 */ getOutputFrame(int index)4314 public @NonNull OutputFrame getOutputFrame(int index) { 4315 synchronized (mBufferLock) { 4316 if (mBufferMode != BUFFER_MODE_BLOCK) { 4317 throw new IllegalStateException("The codec is not configured for block model"); 4318 } 4319 if (index < 0 || index >= mOutputFrames.size()) { 4320 throw new IndexOutOfBoundsException("Expected range of index: [0," 4321 + (mQueueRequests.size() - 1) + "]; actual: " + index); 4322 } 4323 OutputFrame frame = mOutputFrames.get(index); 4324 if (frame == null) { 4325 throw new IllegalArgumentException("Unavailable index: " + index); 4326 } 4327 if (!frame.isAccessible()) { 4328 throw new IllegalArgumentException( 4329 "The output frame is stale at index " + index); 4330 } 4331 if (!frame.isLoaded()) { 4332 native_getOutputFrame(frame, index); 4333 frame.setLoaded(true); 4334 } 4335 return frame; 4336 } 4337 } 4338 native_getOutputFrame(OutputFrame frame, int index)4339 private native void native_getOutputFrame(OutputFrame frame, int index); 4340 4341 /** 4342 * The content is scaled to the surface dimensions 4343 */ 4344 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 4345 4346 /** 4347 * The content is scaled, maintaining its aspect ratio, the whole 4348 * surface area is used, content may be cropped. 4349 * <p class=note> 4350 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 4351 * configure the pixel aspect ratio for a {@link Surface}. 4352 * <p class=note> 4353 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 4354 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 4355 */ 4356 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 4357 4358 /** @hide */ 4359 @IntDef({ 4360 VIDEO_SCALING_MODE_SCALE_TO_FIT, 4361 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 4362 }) 4363 @Retention(RetentionPolicy.SOURCE) 4364 public @interface VideoScalingMode {} 4365 4366 /** 4367 * If a surface has been specified in a previous call to {@link #configure} 4368 * specifies the scaling mode to use. The default is "scale to fit". 4369 * <p class=note> 4370 * The scaling mode may be reset to the <strong>default</strong> each time an 4371 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 4372 * must call this method after every buffer change event (and before the first output buffer is 4373 * released for rendering) to ensure consistent scaling mode. 4374 * <p class=note> 4375 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 4376 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 4377 * 4378 * @throws IllegalArgumentException if mode is not recognized. 4379 * @throws IllegalStateException if in the Released state. 4380 */ setVideoScalingMode(@ideoScalingMode int mode)4381 public native final void setVideoScalingMode(@VideoScalingMode int mode); 4382 4383 /** 4384 * Sets the audio presentation. 4385 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 4386 */ setAudioPresentation(@onNull AudioPresentation presentation)4387 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 4388 if (presentation == null) { 4389 throw new NullPointerException("audio presentation is null"); 4390 } 4391 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 4392 } 4393 native_setAudioPresentation(int presentationId, int programId)4394 private native void native_setAudioPresentation(int presentationId, int programId); 4395 4396 /** 4397 * Retrieve the codec name. 4398 * 4399 * If the codec was created by createDecoderByType or createEncoderByType, what component is 4400 * chosen is not known beforehand. This method returns the name of the codec that was 4401 * selected by the platform. 4402 * 4403 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 4404 * names) for the same underlying codec, any of which can be used to instantiate the same 4405 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 4406 * name used to create the codec in this case. 4407 * 4408 * @throws IllegalStateException if in the Released state. 4409 */ 4410 @NonNull getName()4411 public final String getName() { 4412 // get canonical name to handle exception 4413 String canonicalName = getCanonicalName(); 4414 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 4415 } 4416 4417 /** 4418 * Retrieve the underlying codec name. 4419 * 4420 * This method is similar to {@link #getName}, except that it returns the underlying component 4421 * name even if an alias was used to create this MediaCodec object by name, 4422 * 4423 * @throws IllegalStateException if in the Released state. 4424 */ 4425 @NonNull getCanonicalName()4426 public native final String getCanonicalName(); 4427 4428 /** 4429 * Return Metrics data about the current codec instance. 4430 * 4431 * @return a {@link PersistableBundle} containing the set of attributes and values 4432 * available for the media being handled by this instance of MediaCodec 4433 * The attributes are descibed in {@link MetricsConstants}. 4434 * 4435 * Additional vendor-specific fields may also be present in 4436 * the return value. 4437 */ getMetrics()4438 public PersistableBundle getMetrics() { 4439 PersistableBundle bundle = native_getMetrics(); 4440 return bundle; 4441 } 4442 native_getMetrics()4443 private native PersistableBundle native_getMetrics(); 4444 4445 /** 4446 * Change a video encoder's target bitrate on the fly. The value is an 4447 * Integer object containing the new bitrate in bps. 4448 * 4449 * @see #setParameters(Bundle) 4450 */ 4451 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 4452 4453 /** 4454 * Temporarily suspend/resume encoding of input data. While suspended 4455 * input data is effectively discarded instead of being fed into the 4456 * encoder. This parameter really only makes sense to use with an encoder 4457 * in "surface-input" mode, as the client code has no control over the 4458 * input-side of the encoder in that case. 4459 * The value is an Integer object containing the value 1 to suspend 4460 * or the value 0 to resume. 4461 * 4462 * @see #setParameters(Bundle) 4463 */ 4464 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 4465 4466 /** 4467 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 4468 * optionally use this key to specify the timestamp (in micro-second) 4469 * at which the suspend/resume operation takes effect. 4470 * 4471 * Note that the specified timestamp must be greater than or equal to the 4472 * timestamp of any previously queued suspend/resume operations. 4473 * 4474 * The value is a long int, indicating the timestamp to suspend/resume. 4475 * 4476 * @see #setParameters(Bundle) 4477 */ 4478 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 4479 4480 /** 4481 * Specify an offset (in micro-second) to be added on top of the timestamps 4482 * onward. A typical use case is to apply an adjust to the timestamps after 4483 * a period of pause by the user. 4484 * 4485 * This parameter can only be used on an encoder in "surface-input" mode. 4486 * 4487 * The value is a long int, indicating the timestamp offset to be applied. 4488 * 4489 * @see #setParameters(Bundle) 4490 */ 4491 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 4492 4493 /** 4494 * Request that the encoder produce a sync frame "soon". 4495 * Provide an Integer with the value 0. 4496 * 4497 * @see #setParameters(Bundle) 4498 */ 4499 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 4500 4501 /** 4502 * Set the HDR10+ metadata on the next queued input frame. 4503 * 4504 * Provide a byte array of data that's conforming to the 4505 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 4506 *<p> 4507 * For decoders: 4508 *<p> 4509 * When a decoder is configured for one of the HDR10+ profiles that uses 4510 * out-of-band metadata (such as {@link 4511 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 4512 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 4513 * parameter sets the HDR10+ metadata on the next input buffer queued 4514 * to the decoder. A decoder supporting these profiles must propagate 4515 * the metadata to the format of the output buffer corresponding to this 4516 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 4517 * The metadata should be applied to that output buffer and the buffers 4518 * following it (in display order), until the next output buffer (in 4519 * display order) upon which an HDR10+ metadata is set. 4520 *<p> 4521 * This parameter shouldn't be set if the decoder is not configured for 4522 * an HDR10+ profile that uses out-of-band metadata. In particular, 4523 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 4524 * where the metadata is embedded in the input buffers, for example 4525 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 4526 *<p> 4527 * For encoders: 4528 *<p> 4529 * When an encoder is configured for one of the HDR10+ profiles and the 4530 * operates in byte buffer input mode (instead of surface input mode), 4531 * this parameter sets the HDR10+ metadata on the next input buffer queued 4532 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 4533 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 4534 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 4535 * the metadata must be propagated to the format of the output buffer 4536 * corresponding to this particular input buffer (under key {@link 4537 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 4538 * in-band metadata (such as {@link 4539 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 4540 * metadata info must be embedded in the corresponding output buffer itself. 4541 *<p> 4542 * This parameter shouldn't be set if the encoder is not configured for 4543 * an HDR10+ profile, or if it's operating in surface input mode. 4544 *<p> 4545 * 4546 * @see MediaFormat#KEY_HDR10_PLUS_INFO 4547 */ 4548 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 4549 4550 /** 4551 * Enable/disable low latency decoding mode. 4552 * When enabled, the decoder doesn't hold input and output data more than 4553 * required by the codec standards. 4554 * The value is an Integer object containing the value 1 to enable 4555 * or the value 0 to disable. 4556 * 4557 * @see #setParameters(Bundle) 4558 * @see MediaFormat#KEY_LOW_LATENCY 4559 */ 4560 public static final String PARAMETER_KEY_LOW_LATENCY = 4561 MediaFormat.KEY_LOW_LATENCY; 4562 4563 /** 4564 * Control video peek of the first frame when a codec is configured for tunnel mode with 4565 * {@link MediaFormat#KEY_AUDIO_SESSION_ID} while the {@link AudioTrack} is paused. 4566 *<p> 4567 * When disabled (1) after a {@link #flush} or {@link #start}, (2) while the corresponding 4568 * {@link AudioTrack} is paused and (3) before any buffers are queued, the first frame is not to 4569 * be rendered until either this parameter is enabled or the corresponding {@link AudioTrack} 4570 * has begun playback. Once the frame is decoded and ready to be rendered, 4571 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called but the frame is 4572 * not rendered. The surface continues to show the previously-rendered content, or black if the 4573 * surface is new. A subsequent call to {@link AudioTrack#play} renders this frame and triggers 4574 * a callback to {@link OnFrameRenderedListener#onFrameRendered}, and video playback begins. 4575 *<p> 4576 * <b>Note</b>: To clear any previously rendered content and show black, configure the 4577 * MediaCodec with {@code KEY_PUSH_BLANK_BUFFERS_ON_STOP(1)}, and call {@link #stop} before 4578 * pushing new video frames to the codec. 4579 *<p> 4580 * When enabled (1) after a {@link #flush} or {@link #start} and (2) while the corresponding 4581 * {@link AudioTrack} is paused, the first frame is rendered as soon as it is decoded, or 4582 * immediately, if it has already been decoded. If not already decoded, when the frame is 4583 * decoded and ready to be rendered, 4584 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called. The frame is then 4585 * immediately rendered and {@link OnFrameRenderedListener#onFrameRendered} is subsequently 4586 * called. 4587 *<p> 4588 * The value is an Integer object containing the value 1 to enable or the value 0 to disable. 4589 *<p> 4590 * The default for this parameter is <b>enabled</b>. Once a frame has been rendered, changing 4591 * this parameter has no effect until a subsequent {@link #flush} or 4592 * {@link #stop}/{@link #start}. 4593 * 4594 * @see #setParameters(Bundle) 4595 */ 4596 public static final String PARAMETER_KEY_TUNNEL_PEEK = "tunnel-peek"; 4597 4598 /** 4599 * Communicate additional parameter changes to the component instance. 4600 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 4601 * 4602 * @param params The bundle of parameters to set. 4603 * @throws IllegalStateException if in the Released state. 4604 */ setParameters(@ullable Bundle params)4605 public final void setParameters(@Nullable Bundle params) { 4606 if (params == null) { 4607 return; 4608 } 4609 4610 String[] keys = new String[params.size()]; 4611 Object[] values = new Object[params.size()]; 4612 4613 int i = 0; 4614 for (final String key: params.keySet()) { 4615 if (key.equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 4616 int sessionId = 0; 4617 try { 4618 sessionId = (Integer)params.get(key); 4619 } catch (Exception e) { 4620 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 4621 } 4622 keys[i] = "audio-hw-sync"; 4623 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 4624 } else { 4625 keys[i] = key; 4626 Object value = params.get(key); 4627 4628 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 4629 if (value instanceof byte[]) { 4630 values[i] = ByteBuffer.wrap((byte[])value); 4631 } else { 4632 values[i] = value; 4633 } 4634 } 4635 ++i; 4636 } 4637 4638 setParameters(keys, values); 4639 } 4640 4641 /** 4642 * Sets an asynchronous callback for actionable MediaCodec events. 4643 * 4644 * If the client intends to use the component in asynchronous mode, 4645 * a valid callback should be provided before {@link #configure} is called. 4646 * 4647 * When asynchronous callback is enabled, the client should not call 4648 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 4649 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 4650 * <p> 4651 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 4652 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 4653 * even if an input surface was created. 4654 * 4655 * @param cb The callback that will run. Use {@code null} to clear a previously 4656 * set callback (before {@link #configure configure} is called and run 4657 * in synchronous mode). 4658 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 4659 * callbacks are done on the default thread (the caller's thread or the 4660 * main thread.) 4661 */ setCallback(@ullable Callback cb, @Nullable Handler handler)4662 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 4663 if (cb != null) { 4664 synchronized (mListenerLock) { 4665 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 4666 // NOTE: there are no callbacks on the handler at this time, but check anyways 4667 // even if we were to extend this to be callable dynamically, it must 4668 // be called when codec is flushed, so no messages are pending. 4669 if (newHandler != mCallbackHandler) { 4670 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4671 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4672 mCallbackHandler = newHandler; 4673 } 4674 } 4675 } else if (mCallbackHandler != null) { 4676 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4677 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4678 } 4679 4680 if (mCallbackHandler != null) { 4681 // set java callback on main handler 4682 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 4683 mCallbackHandler.sendMessage(msg); 4684 4685 // set native handler here, don't post to handler because 4686 // it may cause the callback to be delayed and set in a wrong state. 4687 // Note that native codec may start sending events to the callback 4688 // handler after this returns. 4689 native_setCallback(cb); 4690 } 4691 } 4692 4693 /** 4694 * Sets an asynchronous callback for actionable MediaCodec events on the default 4695 * looper. 4696 * <p> 4697 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 4698 * @param cb The callback that will run. Use {@code null} to clear a previously 4699 * set callback (before {@link #configure configure} is called and run 4700 * in synchronous mode). 4701 * @see #setCallback(Callback, Handler) 4702 */ setCallback(@ullable Callback cb)4703 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 4704 setCallback(cb, null /* handler */); 4705 } 4706 4707 /** 4708 * Listener to be called when the first output frame has been decoded 4709 * and is ready to be rendered for a codec configured for tunnel mode with 4710 * {@code KEY_AUDIO_SESSION_ID}. 4711 * 4712 * @see MediaCodec#setOnFirstTunnelFrameReadyListener 4713 */ 4714 public interface OnFirstTunnelFrameReadyListener { 4715 4716 /** 4717 * Called when the first output frame has been decoded and is ready to be 4718 * rendered. 4719 */ onFirstTunnelFrameReady(@onNull MediaCodec codec)4720 void onFirstTunnelFrameReady(@NonNull MediaCodec codec); 4721 } 4722 4723 /** 4724 * Registers a callback to be invoked when the first output frame has been decoded 4725 * and is ready to be rendered on a codec configured for tunnel mode with {@code 4726 * KEY_AUDIO_SESSION_ID}. 4727 * 4728 * @param handler the callback will be run on the handler's thread. If {@code 4729 * null}, the callback will be run on the default thread, which is the looper from 4730 * which the codec was created, or a new thread if there was none. 4731 * 4732 * @param listener the callback that will be run. If {@code null}, clears any registered 4733 * listener. 4734 */ setOnFirstTunnelFrameReadyListener( @ullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener)4735 public void setOnFirstTunnelFrameReadyListener( 4736 @Nullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener) { 4737 synchronized (mListenerLock) { 4738 mOnFirstTunnelFrameReadyListener = listener; 4739 if (listener != null) { 4740 EventHandler newHandler = getEventHandlerOn( 4741 handler, 4742 mOnFirstTunnelFrameReadyHandler); 4743 if (newHandler != mOnFirstTunnelFrameReadyHandler) { 4744 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 4745 } 4746 mOnFirstTunnelFrameReadyHandler = newHandler; 4747 } else if (mOnFirstTunnelFrameReadyHandler != null) { 4748 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 4749 } 4750 native_enableOnFirstTunnelFrameReadyListener(listener != null); 4751 } 4752 } 4753 native_enableOnFirstTunnelFrameReadyListener(boolean enable)4754 private native void native_enableOnFirstTunnelFrameReadyListener(boolean enable); 4755 4756 /** 4757 * Listener to be called when an output frame has rendered on the output surface 4758 * 4759 * @see MediaCodec#setOnFrameRenderedListener 4760 */ 4761 public interface OnFrameRenderedListener { 4762 4763 /** 4764 * Called when an output frame has rendered on the output surface. 4765 * <p> 4766 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4767 * render timing samples, and can be significantly delayed and batched. Some frames may have 4768 * been rendered even if there was no callback generated. 4769 * 4770 * @param codec the MediaCodec instance 4771 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 4772 * This is usually the same as specified in {@link #queueInputBuffer}; however, 4773 * some codecs may alter the media time by applying some time-based transformation, 4774 * such as frame rate conversion. In that case, presentation time corresponds 4775 * to the actual output frame rendered. 4776 * @param nanoTime The system time when the frame was rendered. 4777 * 4778 * @see System#nanoTime 4779 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)4780 public void onFrameRendered( 4781 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 4782 } 4783 4784 /** 4785 * Registers a callback to be invoked when an output frame is rendered on the output surface. 4786 * <p> 4787 * This method can be called in any codec state, but will only have an effect in the 4788 * Executing state for codecs that render buffers to the output surface. 4789 * <p> 4790 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4791 * render timing samples, and can be significantly delayed and batched. Some frames may have 4792 * been rendered even if there was no callback generated. 4793 * 4794 * @param listener the callback that will be run 4795 * @param handler the callback will be run on the handler's thread. If {@code null}, 4796 * the callback will be run on the default thread, which is the looper 4797 * from which the codec was created, or a new thread if there was none. 4798 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)4799 public void setOnFrameRenderedListener( 4800 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 4801 synchronized (mListenerLock) { 4802 mOnFrameRenderedListener = listener; 4803 if (listener != null) { 4804 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 4805 if (newHandler != mOnFrameRenderedHandler) { 4806 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4807 } 4808 mOnFrameRenderedHandler = newHandler; 4809 } else if (mOnFrameRenderedHandler != null) { 4810 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4811 } 4812 native_enableOnFrameRenderedListener(listener != null); 4813 } 4814 } 4815 native_enableOnFrameRenderedListener(boolean enable)4816 private native void native_enableOnFrameRenderedListener(boolean enable); 4817 4818 /** 4819 * Returns a list of vendor parameter names. 4820 * <p> 4821 * This method can be called in any codec state except for released state. 4822 * 4823 * @return a list containing supported vendor parameters; an empty 4824 * list if no vendor parameters are supported. The order of the 4825 * parameters is arbitrary. 4826 * @throws IllegalStateException if in the Released state. 4827 */ 4828 @NonNull getSupportedVendorParameters()4829 public List<String> getSupportedVendorParameters() { 4830 return native_getSupportedVendorParameters(); 4831 } 4832 4833 @NonNull native_getSupportedVendorParameters()4834 private native List<String> native_getSupportedVendorParameters(); 4835 4836 /** 4837 * Contains description of a parameter. 4838 */ 4839 public static class ParameterDescriptor { ParameterDescriptor()4840 private ParameterDescriptor() {} 4841 4842 /** 4843 * Returns the name of the parameter. 4844 */ 4845 @NonNull getName()4846 public String getName() { 4847 return mName; 4848 } 4849 4850 /** 4851 * Returns the type of the parameter. 4852 * {@link MediaFormat#TYPE_NULL} is never returned. 4853 */ 4854 @MediaFormat.Type getType()4855 public int getType() { 4856 return mType; 4857 } 4858 4859 @Override equals(Object o)4860 public boolean equals(Object o) { 4861 if (o == null) { 4862 return false; 4863 } 4864 if (!(o instanceof ParameterDescriptor)) { 4865 return false; 4866 } 4867 ParameterDescriptor other = (ParameterDescriptor) o; 4868 return this.mName.equals(other.mName) && this.mType == other.mType; 4869 } 4870 4871 @Override hashCode()4872 public int hashCode() { 4873 return Arrays.asList( 4874 (Object) mName, 4875 (Object) Integer.valueOf(mType)).hashCode(); 4876 } 4877 4878 private String mName; 4879 private @MediaFormat.Type int mType; 4880 } 4881 4882 /** 4883 * Describe a parameter with the name. 4884 * <p> 4885 * This method can be called in any codec state except for released state. 4886 * 4887 * @param name name of the parameter to describe, typically one from 4888 * {@link #getSupportedVendorParameters}. 4889 * @return {@link ParameterDescriptor} object that describes the parameter. 4890 * {@code null} if unrecognized / not able to describe. 4891 * @throws IllegalStateException if in the Released state. 4892 */ 4893 @Nullable getParameterDescriptor(@onNull String name)4894 public ParameterDescriptor getParameterDescriptor(@NonNull String name) { 4895 return native_getParameterDescriptor(name); 4896 } 4897 4898 @Nullable native_getParameterDescriptor(@onNull String name)4899 private native ParameterDescriptor native_getParameterDescriptor(@NonNull String name); 4900 4901 /** 4902 * Subscribe to vendor parameters, so that these parameters will be present in 4903 * {@link #getOutputFormat} and changes to these parameters generate 4904 * output format change event. 4905 * <p> 4906 * Unrecognized parameter names or standard (non-vendor) parameter names will be ignored. 4907 * {@link #reset} also resets the list of subscribed parameters. 4908 * If a parameter in {@code names} is already subscribed, it will remain subscribed. 4909 * <p> 4910 * This method can be called in any codec state except for released state. When called in 4911 * running state with newly subscribed parameters, it takes effect no later than the 4912 * processing of the subsequently queued buffer. For the new parameters, the codec will generate 4913 * output format change event. 4914 * <p> 4915 * Note that any vendor parameters set in a {@link #configure} or 4916 * {@link #setParameters} call are automatically subscribed. 4917 * <p> 4918 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 4919 * for output format change events. 4920 * 4921 * @param names names of the vendor parameters to subscribe. This may be an empty list, 4922 * and in that case this method will not change the list of subscribed parameters. 4923 * @throws IllegalStateException if in the Released state. 4924 */ subscribeToVendorParameters(@onNull List<String> names)4925 public void subscribeToVendorParameters(@NonNull List<String> names) { 4926 native_subscribeToVendorParameters(names); 4927 } 4928 native_subscribeToVendorParameters(@onNull List<String> names)4929 private native void native_subscribeToVendorParameters(@NonNull List<String> names); 4930 4931 /** 4932 * Unsubscribe from vendor parameters, so that these parameters will not be present in 4933 * {@link #getOutputFormat} and changes to these parameters no longer generate 4934 * output format change event. 4935 * <p> 4936 * Unrecognized parameter names, standard (non-vendor) parameter names will be ignored. 4937 * {@link #reset} also resets the list of subscribed parameters. 4938 * If a parameter in {@code names} is already unsubscribed, it will remain unsubscribed. 4939 * <p> 4940 * This method can be called in any codec state except for released state. When called in 4941 * running state with newly unsubscribed parameters, it takes effect no later than the 4942 * processing of the subsequently queued buffer. For the removed parameters, the codec will 4943 * generate output format change event. 4944 * <p> 4945 * Note that any vendor parameters set in a {@link #configure} or 4946 * {@link #setParameters} call are automatically subscribed, and with this method 4947 * they can be unsubscribed. 4948 * <p> 4949 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 4950 * for output format change events. 4951 * 4952 * @param names names of the vendor parameters to unsubscribe. This may be an empty list, 4953 * and in that case this method will not change the list of subscribed parameters. 4954 * @throws IllegalStateException if in the Released state. 4955 */ unsubscribeFromVendorParameters(@onNull List<String> names)4956 public void unsubscribeFromVendorParameters(@NonNull List<String> names) { 4957 native_unsubscribeFromVendorParameters(names); 4958 } 4959 native_unsubscribeFromVendorParameters(@onNull List<String> names)4960 private native void native_unsubscribeFromVendorParameters(@NonNull List<String> names); 4961 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)4962 private EventHandler getEventHandlerOn( 4963 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 4964 if (handler == null) { 4965 return mEventHandler; 4966 } else { 4967 Looper looper = handler.getLooper(); 4968 if (lastHandler.getLooper() == looper) { 4969 return lastHandler; 4970 } else { 4971 return new EventHandler(this, looper); 4972 } 4973 } 4974 } 4975 4976 /** 4977 * MediaCodec callback interface. Used to notify the user asynchronously 4978 * of various MediaCodec events. 4979 */ 4980 public static abstract class Callback { 4981 /** 4982 * Called when an input buffer becomes available. 4983 * 4984 * @param codec The MediaCodec object. 4985 * @param index The index of the available input buffer. 4986 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)4987 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 4988 4989 /** 4990 * Called when an output buffer becomes available. 4991 * 4992 * @param codec The MediaCodec object. 4993 * @param index The index of the available output buffer. 4994 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 4995 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)4996 public abstract void onOutputBufferAvailable( 4997 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 4998 4999 /** 5000 * Called when the MediaCodec encountered an error 5001 * 5002 * @param codec The MediaCodec object. 5003 * @param e The {@link MediaCodec.CodecException} object describing the error. 5004 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)5005 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 5006 5007 /** 5008 * Called when the output format has changed 5009 * 5010 * @param codec The MediaCodec object. 5011 * @param format The new output format. 5012 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)5013 public abstract void onOutputFormatChanged( 5014 @NonNull MediaCodec codec, @NonNull MediaFormat format); 5015 } 5016 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)5017 private void postEventFromNative( 5018 int what, int arg1, int arg2, @Nullable Object obj) { 5019 synchronized (mListenerLock) { 5020 EventHandler handler = mEventHandler; 5021 if (what == EVENT_CALLBACK) { 5022 handler = mCallbackHandler; 5023 } else if (what == EVENT_FIRST_TUNNEL_FRAME_READY) { 5024 handler = mOnFirstTunnelFrameReadyHandler; 5025 } else if (what == EVENT_FRAME_RENDERED) { 5026 handler = mOnFrameRenderedHandler; 5027 } 5028 if (handler != null) { 5029 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 5030 handler.sendMessage(msg); 5031 } 5032 } 5033 } 5034 5035 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)5036 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 5037 5038 /** 5039 * Get the codec info. If the codec was created by createDecoderByType 5040 * or createEncoderByType, what component is chosen is not known beforehand, 5041 * and thus the caller does not have the MediaCodecInfo. 5042 * @throws IllegalStateException if in the Released state. 5043 */ 5044 @NonNull getCodecInfo()5045 public MediaCodecInfo getCodecInfo() { 5046 // Get the codec name first. If the codec is already released, 5047 // IllegalStateException will be thrown here. 5048 String name = getName(); 5049 synchronized (mCodecInfoLock) { 5050 if (mCodecInfo == null) { 5051 // Get the codec info for this codec itself first. Only initialize 5052 // the full codec list if this somehow fails because it can be slow. 5053 mCodecInfo = getOwnCodecInfo(); 5054 if (mCodecInfo == null) { 5055 mCodecInfo = MediaCodecList.getInfoFor(name); 5056 } 5057 } 5058 return mCodecInfo; 5059 } 5060 } 5061 5062 @NonNull getOwnCodecInfo()5063 private native final MediaCodecInfo getOwnCodecInfo(); 5064 5065 @NonNull 5066 @UnsupportedAppUsage getBuffers(boolean input)5067 private native final ByteBuffer[] getBuffers(boolean input); 5068 5069 @Nullable getBuffer(boolean input, int index)5070 private native final ByteBuffer getBuffer(boolean input, int index); 5071 5072 @Nullable getImage(boolean input, int index)5073 private native final Image getImage(boolean input, int index); 5074 native_init()5075 private static native final void native_init(); 5076 native_setup( @onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)5077 private native final void native_setup( 5078 @NonNull String name, boolean nameIsType, boolean encoder, int pid, int uid); 5079 native_finalize()5080 private native final void native_finalize(); 5081 5082 static { 5083 System.loadLibrary("media_jni"); native_init()5084 native_init(); 5085 } 5086 5087 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 5088 private long mNativeContext = 0; 5089 private final Lock mNativeContextLock = new ReentrantLock(); 5090 lockAndGetContext()5091 private final long lockAndGetContext() { 5092 mNativeContextLock.lock(); 5093 return mNativeContext; 5094 } 5095 setAndUnlockContext(long context)5096 private final void setAndUnlockContext(long context) { 5097 mNativeContext = context; 5098 mNativeContextLock.unlock(); 5099 } 5100 5101 /** @hide */ 5102 public static class MediaImage extends Image { 5103 private final boolean mIsReadOnly; 5104 private final int mWidth; 5105 private final int mHeight; 5106 private final int mFormat; 5107 private long mTimestamp; 5108 private final Plane[] mPlanes; 5109 private final ByteBuffer mBuffer; 5110 private final ByteBuffer mInfo; 5111 private final int mXOffset; 5112 private final int mYOffset; 5113 private final long mBufferContext; 5114 5115 private final static int TYPE_YUV = 1; 5116 5117 private final int mTransform = 0; //Default no transform 5118 private final int mScalingMode = 0; //Default frozen scaling mode 5119 5120 @Override getFormat()5121 public int getFormat() { 5122 throwISEIfImageIsInvalid(); 5123 return mFormat; 5124 } 5125 5126 @Override getHeight()5127 public int getHeight() { 5128 throwISEIfImageIsInvalid(); 5129 return mHeight; 5130 } 5131 5132 @Override getWidth()5133 public int getWidth() { 5134 throwISEIfImageIsInvalid(); 5135 return mWidth; 5136 } 5137 5138 @Override getTransform()5139 public int getTransform() { 5140 throwISEIfImageIsInvalid(); 5141 return mTransform; 5142 } 5143 5144 @Override getScalingMode()5145 public int getScalingMode() { 5146 throwISEIfImageIsInvalid(); 5147 return mScalingMode; 5148 } 5149 5150 @Override getTimestamp()5151 public long getTimestamp() { 5152 throwISEIfImageIsInvalid(); 5153 return mTimestamp; 5154 } 5155 5156 @Override 5157 @NonNull getPlanes()5158 public Plane[] getPlanes() { 5159 throwISEIfImageIsInvalid(); 5160 return Arrays.copyOf(mPlanes, mPlanes.length); 5161 } 5162 5163 @Override close()5164 public void close() { 5165 if (mIsImageValid) { 5166 if (mBuffer != null) { 5167 java.nio.NioUtils.freeDirectBuffer(mBuffer); 5168 } 5169 if (mBufferContext != 0) { 5170 native_closeMediaImage(mBufferContext); 5171 } 5172 mIsImageValid = false; 5173 } 5174 } 5175 5176 /** 5177 * Set the crop rectangle associated with this frame. 5178 * <p> 5179 * The crop rectangle specifies the region of valid pixels in the image, 5180 * using coordinates in the largest-resolution plane. 5181 */ 5182 @Override setCropRect(@ullable Rect cropRect)5183 public void setCropRect(@Nullable Rect cropRect) { 5184 if (mIsReadOnly) { 5185 throw new ReadOnlyBufferException(); 5186 } 5187 super.setCropRect(cropRect); 5188 } 5189 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)5190 public MediaImage( 5191 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 5192 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 5193 mTimestamp = timestamp; 5194 mIsImageValid = true; 5195 mIsReadOnly = buffer.isReadOnly(); 5196 mBuffer = buffer.duplicate(); 5197 5198 // save offsets and info 5199 mXOffset = xOffset; 5200 mYOffset = yOffset; 5201 mInfo = info; 5202 5203 mBufferContext = 0; 5204 5205 int cbPlaneOffset = -1; 5206 int crPlaneOffset = -1; 5207 int planeOffsetInc = -1; 5208 int pixelStride = -1; 5209 5210 // read media-info. See MediaImage2 5211 if (info.remaining() == 104) { 5212 int type = info.getInt(); 5213 if (type != TYPE_YUV) { 5214 throw new UnsupportedOperationException("unsupported type: " + type); 5215 } 5216 int numPlanes = info.getInt(); 5217 if (numPlanes != 3) { 5218 throw new RuntimeException("unexpected number of planes: " + numPlanes); 5219 } 5220 mWidth = info.getInt(); 5221 mHeight = info.getInt(); 5222 if (mWidth < 1 || mHeight < 1) { 5223 throw new UnsupportedOperationException( 5224 "unsupported size: " + mWidth + "x" + mHeight); 5225 } 5226 int bitDepth = info.getInt(); 5227 if (bitDepth != 8 && bitDepth != 10) { 5228 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 5229 } 5230 int bitDepthAllocated = info.getInt(); 5231 if (bitDepthAllocated != 8 && bitDepthAllocated != 16) { 5232 throw new UnsupportedOperationException( 5233 "unsupported allocated bit depth: " + bitDepthAllocated); 5234 } 5235 if (bitDepth == 8 && bitDepthAllocated == 8) { 5236 mFormat = ImageFormat.YUV_420_888; 5237 planeOffsetInc = 1; 5238 pixelStride = 2; 5239 } else if (bitDepth == 10 && bitDepthAllocated == 16) { 5240 mFormat = ImageFormat.YCBCR_P010; 5241 planeOffsetInc = 2; 5242 pixelStride = 4; 5243 } else { 5244 throw new UnsupportedOperationException("couldn't infer ImageFormat" 5245 + " bitDepth: " + bitDepth + " bitDepthAllocated: " + bitDepthAllocated); 5246 } 5247 5248 mPlanes = new MediaPlane[numPlanes]; 5249 for (int ix = 0; ix < numPlanes; ix++) { 5250 int planeOffset = info.getInt(); 5251 int colInc = info.getInt(); 5252 int rowInc = info.getInt(); 5253 int horiz = info.getInt(); 5254 int vert = info.getInt(); 5255 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 5256 throw new UnsupportedOperationException("unexpected subsampling: " 5257 + horiz + "x" + vert + " on plane " + ix); 5258 } 5259 if (colInc < 1 || rowInc < 1) { 5260 throw new UnsupportedOperationException("unexpected strides: " 5261 + colInc + " pixel, " + rowInc + " row on plane " + ix); 5262 } 5263 buffer.clear(); 5264 buffer.position(mBuffer.position() + planeOffset 5265 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 5266 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 5267 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 5268 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 5269 if ((mFormat == ImageFormat.YUV_420_888 || mFormat == ImageFormat.YCBCR_P010) 5270 && ix == 1) { 5271 cbPlaneOffset = planeOffset; 5272 } else if ((mFormat == ImageFormat.YUV_420_888 5273 || mFormat == ImageFormat.YCBCR_P010) && ix == 2) { 5274 crPlaneOffset = planeOffset; 5275 } 5276 } 5277 } else { 5278 throw new UnsupportedOperationException( 5279 "unsupported info length: " + info.remaining()); 5280 } 5281 5282 // Validate chroma semiplanerness. 5283 if (mFormat == ImageFormat.YCBCR_P010) { 5284 if (crPlaneOffset != cbPlaneOffset + planeOffsetInc) { 5285 throw new UnsupportedOperationException("Invalid plane offsets" 5286 + " cbPlaneOffset: " + cbPlaneOffset + " crPlaneOffset: " + crPlaneOffset); 5287 } 5288 if (mPlanes[1].getPixelStride() != pixelStride 5289 || mPlanes[2].getPixelStride() != pixelStride) { 5290 throw new UnsupportedOperationException("Invalid pixelStride"); 5291 } 5292 } 5293 5294 if (cropRect == null) { 5295 cropRect = new Rect(0, 0, mWidth, mHeight); 5296 } 5297 cropRect.offset(-xOffset, -yOffset); 5298 super.setCropRect(cropRect); 5299 } 5300 MediaImage( @onNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, int width, int height, int format, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context)5301 public MediaImage( 5302 @NonNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, 5303 int width, int height, int format, boolean readOnly, 5304 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context) { 5305 if (buffers.length != rowStrides.length || buffers.length != pixelStrides.length) { 5306 throw new IllegalArgumentException( 5307 "buffers, rowStrides and pixelStrides should have the same length"); 5308 } 5309 mWidth = width; 5310 mHeight = height; 5311 mFormat = format; 5312 mTimestamp = timestamp; 5313 mIsImageValid = true; 5314 mIsReadOnly = readOnly; 5315 mBuffer = null; 5316 mInfo = null; 5317 mPlanes = new MediaPlane[buffers.length]; 5318 for (int i = 0; i < buffers.length; ++i) { 5319 mPlanes[i] = new MediaPlane(buffers[i], rowStrides[i], pixelStrides[i]); 5320 } 5321 5322 // save offsets and info 5323 mXOffset = xOffset; 5324 mYOffset = yOffset; 5325 5326 if (cropRect == null) { 5327 cropRect = new Rect(0, 0, mWidth, mHeight); 5328 } 5329 cropRect.offset(-xOffset, -yOffset); 5330 super.setCropRect(cropRect); 5331 5332 mBufferContext = context; 5333 } 5334 5335 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)5336 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 5337 mData = buffer; 5338 mRowInc = rowInc; 5339 mColInc = colInc; 5340 } 5341 5342 @Override getRowStride()5343 public int getRowStride() { 5344 throwISEIfImageIsInvalid(); 5345 return mRowInc; 5346 } 5347 5348 @Override getPixelStride()5349 public int getPixelStride() { 5350 throwISEIfImageIsInvalid(); 5351 return mColInc; 5352 } 5353 5354 @Override 5355 @NonNull getBuffer()5356 public ByteBuffer getBuffer() { 5357 throwISEIfImageIsInvalid(); 5358 return mData; 5359 } 5360 5361 private final int mRowInc; 5362 private final int mColInc; 5363 private final ByteBuffer mData; 5364 } 5365 } 5366 5367 public final static class MetricsConstants 5368 { MetricsConstants()5369 private MetricsConstants() {} 5370 5371 /** 5372 * Key to extract the codec being used 5373 * from the {@link MediaCodec#getMetrics} return value. 5374 * The value is a String. 5375 */ 5376 public static final String CODEC = "android.media.mediacodec.codec"; 5377 5378 /** 5379 * Key to extract the MIME type 5380 * from the {@link MediaCodec#getMetrics} return value. 5381 * The value is a String. 5382 */ 5383 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 5384 5385 /** 5386 * Key to extract what the codec mode 5387 * from the {@link MediaCodec#getMetrics} return value. 5388 * The value is a String. Values will be one of the constants 5389 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 5390 */ 5391 public static final String MODE = "android.media.mediacodec.mode"; 5392 5393 /** 5394 * The value returned for the key {@link #MODE} when the 5395 * codec is a audio codec. 5396 */ 5397 public static final String MODE_AUDIO = "audio"; 5398 5399 /** 5400 * The value returned for the key {@link #MODE} when the 5401 * codec is a video codec. 5402 */ 5403 public static final String MODE_VIDEO = "video"; 5404 5405 /** 5406 * Key to extract the flag indicating whether the codec is running 5407 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 5408 * The value is an integer. 5409 * A 0 indicates decoder; 1 indicates encoder. 5410 */ 5411 public static final String ENCODER = "android.media.mediacodec.encoder"; 5412 5413 /** 5414 * Key to extract the flag indicating whether the codec is running 5415 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 5416 * The value is an integer. 5417 */ 5418 public static final String SECURE = "android.media.mediacodec.secure"; 5419 5420 /** 5421 * Key to extract the width (in pixels) of the video track 5422 * from the {@link MediaCodec#getMetrics} return value. 5423 * The value is an integer. 5424 */ 5425 public static final String WIDTH = "android.media.mediacodec.width"; 5426 5427 /** 5428 * Key to extract the height (in pixels) of the video track 5429 * from the {@link MediaCodec#getMetrics} return value. 5430 * The value is an integer. 5431 */ 5432 public static final String HEIGHT = "android.media.mediacodec.height"; 5433 5434 /** 5435 * Key to extract the rotation (in degrees) to properly orient the video 5436 * from the {@link MediaCodec#getMetrics} return. 5437 * The value is a integer. 5438 */ 5439 public static final String ROTATION = "android.media.mediacodec.rotation"; 5440 5441 } 5442 } 5443
