1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import static android.media.codec.Flags.FLAG_CODEC_AVAILABILITY; 20 import static android.media.codec.Flags.FLAG_NULL_OUTPUT_SURFACE; 21 import static android.media.codec.Flags.FLAG_REGION_OF_INTEREST; 22 import static android.media.codec.Flags.FLAG_SUBSESSION_METRICS; 23 import static android.media.tv.flags.Flags.applyPictureProfiles; 24 import static android.media.tv.flags.Flags.mediaQualityFw; 25 26 import static com.android.media.codec.flags.Flags.FLAG_LARGE_AUDIO_FRAME; 27 28 import android.Manifest; 29 import android.annotation.FlaggedApi; 30 import android.annotation.IntDef; 31 import android.annotation.NonNull; 32 import android.annotation.Nullable; 33 import android.annotation.RequiresPermission; 34 import android.annotation.SystemApi; 35 import android.annotation.TestApi; 36 import android.compat.annotation.UnsupportedAppUsage; 37 import android.graphics.ImageFormat; 38 import android.graphics.Rect; 39 import android.graphics.SurfaceTexture; 40 import android.hardware.HardwareBuffer; 41 import android.media.MediaCodecInfo.CodecCapabilities; 42 import android.media.quality.PictureProfile; 43 import android.media.quality.PictureProfileHandle; 44 import android.os.Build; 45 import android.os.Bundle; 46 import android.os.Handler; 47 import android.os.IHwBinder; 48 import android.os.Looper; 49 import android.os.Message; 50 import android.os.PersistableBundle; 51 import android.os.Trace; 52 import android.view.Surface; 53 import android.util.Log; 54 55 import java.io.IOException; 56 import java.lang.annotation.Retention; 57 import java.lang.annotation.RetentionPolicy; 58 import java.nio.ByteBuffer; 59 import java.nio.ByteOrder; 60 import java.nio.ReadOnlyBufferException; 61 import java.util.ArrayDeque; 62 import java.util.ArrayList; 63 import java.util.Arrays; 64 import java.util.BitSet; 65 import java.util.Collections; 66 import java.util.HashMap; 67 import java.util.HashSet; 68 import java.util.List; 69 import java.util.Map; 70 import java.util.Objects; 71 import java.util.Optional; 72 import java.util.Set; 73 import java.util.concurrent.BlockingQueue; 74 import java.util.concurrent.LinkedBlockingQueue; 75 import java.util.concurrent.locks.Lock; 76 import java.util.concurrent.locks.ReentrantLock; 77 import java.util.function.Supplier; 78 79 /** 80 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 81 It is part of the Android low-level multimedia support infrastructure (normally used together 82 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 83 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 84 <p> 85 <center> 86 <img src="../../../images/media/mediacodec_buffers.svg" style="width: 540px; height: 205px" 87 alt="MediaCodec buffer flow diagram"> 88 </center> 89 <p> 90 In broad terms, a codec processes input data to generate output data. It processes data 91 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 92 (or receive) an empty input buffer, fill it up with data and send it to the codec for 93 processing. The codec uses up the data and transforms it into one of its empty output buffers. 94 Finally, you request (or receive) a filled output buffer, consume its contents and release it 95 back to the codec. 96 97 <h3 id=qualityFloor><a name="qualityFloor">Minimum Quality Floor for Video Encoding</h3> 98 <p> 99 Beginning with {@link android.os.Build.VERSION_CODES#S}, Android's Video MediaCodecs enforce a 100 minimum quality floor. The intent is to eliminate poor quality video encodings. This quality 101 floor is applied when the codec is in Variable Bitrate (VBR) mode; it is not applied when 102 the codec is in Constant Bitrate (CBR) mode. The quality floor enforcement is also restricted 103 to a particular size range; this size range is currently for video resolutions 104 larger than 320x240 up through 1920x1080. 105 106 <p> 107 When this quality floor is in effect, the codec and supporting framework code will work to 108 ensure that the generated video is of at least a "fair" or "good" quality. The metric 109 used to choose these targets is the VMAF (Video Multi-method Assessment Function) with a 110 target score of 70 for selected test sequences. 111 112 <p> 113 The typical effect is that 114 some videos will generate a higher bitrate than originally configured. This will be most 115 notable for videos which were configured with very low bitrates; the codec will use a bitrate 116 that is determined to be more likely to generate an "fair" or "good" quality video. Another 117 situation is where a video includes very complicated content (lots of motion and detail); 118 in such configurations, the codec will use extra bitrate as needed to avoid losing all of 119 the content's finer detail. 120 121 <p> 122 This quality floor will not impact content captured at high bitrates (a high bitrate should 123 already provide the codec with sufficient capacity to encode all of the detail). 124 The quality floor does not operate on CBR encodings. 125 The quality floor currently does not operate on resolutions of 320x240 or lower, nor on 126 videos with resolution above 1920x1080. 127 128 <h3>Data Types</h3> 129 <p> 130 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 131 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 132 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 133 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 134 You normally cannot access the raw video data when using a Surface, but you can use the 135 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 136 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 137 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 138 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 139 OutputImage(int)}. 140 141 <h4>Compressed Buffers</h4> 142 <p> 143 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 144 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 145 compressed video frame. For audio data this is normally a single access unit (an encoded audio 146 segment typically containing a few milliseconds of audio as dictated by the format type), but 147 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 148 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 149 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 150 151 <h4>Raw Audio Buffers</h4> 152 <p> 153 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 154 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 155 in native byte order. 156 Raw audio buffers in the float PCM encoding are only possible 157 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 158 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 159 {@link #configure configure(…)} 160 and confirmed by {@link #getOutputFormat} for decoders 161 or {@link #getInputFormat} for encoders. 162 A sample method to check for float PCM in the MediaFormat is as follows: 163 164 <pre class=prettyprint> 165 static boolean isPcmFloat(MediaFormat format) { 166 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 167 == AudioFormat.ENCODING_PCM_FLOAT; 168 }</pre> 169 170 In order to extract, in a short array, 171 one channel of a buffer containing 16 bit signed integer audio data, 172 the following code may be used: 173 174 <pre class=prettyprint> 175 // Assumes the buffer PCM encoding is 16 bit. 176 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 177 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 178 MediaFormat format = codec.getOutputFormat(bufferId); 179 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 180 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 181 if (channelIx < 0 || channelIx >= numChannels) { 182 return null; 183 } 184 short[] res = new short[samples.remaining() / numChannels]; 185 for (int i = 0; i < res.length; ++i) { 186 res[i] = samples.get(i * numChannels + channelIx); 187 } 188 return res; 189 }</pre> 190 191 <h4>Raw Video Buffers</h4> 192 <p> 193 In ByteBuffer mode video buffers are laid out according to their {@linkplain 194 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 195 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 196 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 197 Video codecs may support three kinds of color formats: 198 <ul> 199 <li><strong>native raw video format:</strong> This is marked by {@link 200 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 201 <li><strong>flexible YUV buffers</strong> (such as {@link 202 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 203 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 204 OutputImage(int)}.</li> 205 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 206 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 207 For color formats that are equivalent to a flexible format, you can still use {@link 208 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 209 </ul> 210 <p> 211 All video codecs support flexible YUV 4:2:0 buffers since {@link 212 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 213 214 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 215 <p> 216 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 217 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 218 values to understand the layout of the raw output buffers. 219 <p class=note> 220 Note that on some devices the slice-height is advertised as 0. This could mean either that the 221 slice-height is the same as the frame height, or that the slice-height is the frame height 222 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 223 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 224 plane in planar formats is also not specified or defined, though usually it is half of the slice 225 height. 226 <p> 227 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 228 video frames; however, for most encondings the video (picture) only occupies a portion of the 229 video frame. This is represented by the 'crop rectangle'. 230 <p> 231 You need to use the following keys to get the crop rectangle of raw output images from the 232 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 233 entire video frame.The crop rectangle is understood in the context of the output frame 234 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 235 <table style="width: 0%"> 236 <thead> 237 <tr> 238 <th>Format Key</th> 239 <th>Type</th> 240 <th>Description</th> 241 </tr> 242 </thead> 243 <tbody> 244 <tr> 245 <td>{@link MediaFormat#KEY_CROP_LEFT}</td> 246 <td>Integer</td> 247 <td>The left-coordinate (x) of the crop rectangle</td> 248 </tr><tr> 249 <td>{@link MediaFormat#KEY_CROP_TOP}</td> 250 <td>Integer</td> 251 <td>The top-coordinate (y) of the crop rectangle</td> 252 </tr><tr> 253 <td>{@link MediaFormat#KEY_CROP_RIGHT}</td> 254 <td>Integer</td> 255 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 256 </tr><tr> 257 <td>{@link MediaFormat#KEY_CROP_BOTTOM}</td> 258 <td>Integer</td> 259 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 260 </tr><tr> 261 <td colspan=3> 262 The right and bottom coordinates can be understood as the coordinates of the right-most 263 valid column/bottom-most valid row of the cropped output image. 264 </td> 265 </tr> 266 </tbody> 267 </table> 268 <p> 269 The size of the video frame (before rotation) can be calculated as such: 270 <pre class=prettyprint> 271 MediaFormat format = decoder.getOutputFormat(…); 272 int width = format.getInteger(MediaFormat.KEY_WIDTH); 273 if (format.containsKey(MediaFormat.KEY_CROP_LEFT) 274 && format.containsKey(MediaFormat.KEY_CROP_RIGHT)) { 275 width = format.getInteger(MediaFormat.KEY_CROP_RIGHT) + 1 276 - format.getInteger(MediaFormat.KEY_CROP_LEFT); 277 } 278 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 279 if (format.containsKey(MediaFormat.KEY_CROP_TOP) 280 && format.containsKey(MediaFormat.KEY_CROP_BOTTOM)) { 281 height = format.getInteger(MediaFormat.KEY_CROP_BOTTOM) + 1 282 - format.getInteger(MediaFormat.KEY_CROP_TOP); 283 } 284 </pre> 285 <p class=note> 286 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 287 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 288 most devices it pointed to the top-left pixel of the entire frame. 289 290 <h3>States</h3> 291 <p> 292 During its life a codec conceptually exists in one of three states: Stopped, Executing or 293 Released. The Stopped collective state is actually the conglomeration of three states: 294 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 295 three sub-states: Flushed, Running and End-of-Stream. 296 <p> 297 <center> 298 <img src="../../../images/media/mediacodec_states.svg" style="width: 519px; height: 356px" 299 alt="MediaCodec state diagram"> 300 </center> 301 <p> 302 When you create a codec using one of the factory methods, the codec is in the Uninitialized 303 state. First, you need to configure it via {@link #configure configure(…)}, which brings 304 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 305 state you can process data through the buffer queue manipulation described above. 306 <p> 307 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 308 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 309 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 310 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 311 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 312 codec no longer accepts further input buffers, but still generates output buffers until the 313 end-of-stream is reached on the output. For decoders, you can move back to the Flushed sub-state 314 at any time while in the Executing state using {@link #flush}. 315 <p class=note> 316 <strong>Note:</strong> Going back to Flushed state is only supported for decoders, and may not 317 work for encoders (the behavior is undefined). 318 <p> 319 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 320 again. When you are done using a codec, you must release it by calling {@link #release}. 321 <p> 322 On rare occasions the codec may encounter an error and move to the Error state. This is 323 communicated using an invalid return value from a queuing operation, or sometimes via an 324 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 325 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 326 terminal Released state. 327 328 <h3>Creation</h3> 329 <p> 330 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 331 decoding a file or a stream, you can get the desired format from {@link 332 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 333 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 334 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 335 name of a codec that can handle that specific media format. Finally, create the codec using 336 {@link #createByCodecName}. 337 <p class=note> 338 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 339 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 340 MediaFormat#KEY_FRAME_RATE frame rate}. Use 341 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 342 to clear any existing frame rate setting in the format. 343 <p> 344 You can also create the preferred codec for a specific MIME type using {@link 345 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 346 This, however, cannot be used to inject features, and may create a codec that cannot handle the 347 specific desired media format. 348 349 <h4>Creating secure decoders</h4> 350 <p> 351 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 352 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 353 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 354 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 355 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 356 <p> 357 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 358 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 359 360 <h3>Initialization</h3> 361 <p> 362 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 363 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 364 specific media format. This is when you can specify the output {@link Surface} for video 365 producers – codecs that generate raw video data (e.g. video decoders). This is also when 366 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 367 some codecs can operate in multiple modes, you must specify whether you want it to work as a 368 decoder or an encoder. 369 <p> 370 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 371 output format in the Configured state. You can use this to verify the resulting configuration, 372 e.g. color formats, before starting the codec. 373 <p> 374 If you want to process raw input video buffers natively with a video consumer – a codec 375 that processes raw video input, such as a video encoder – create a destination Surface for 376 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 377 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 378 surface} by calling {@link #setInputSurface}. 379 380 <h4 id=EncoderProfiles><a name="EncoderProfiles"></a>Encoder Profiles</h4> 381 <p> 382 When using an encoder, it is recommended to set the desired codec {@link MediaFormat#KEY_PROFILE 383 profile} during {@link #configure configure()}. (This is only meaningful for 384 {@link MediaFormat#KEY_MIME media formats} for which profiles are defined.) 385 <p> 386 If a profile is not specified during {@code configure}, the encoder will choose a profile for the 387 session based on the available information. We will call this value the <i>default profile</i>. 388 The selection of the default profile is device specific and may not be deterministic 389 (could be ad hoc or even experimental). The encoder may choose a default profile that is not 390 suitable for the intended encoding session, which may result in the encoder ultimately rejecting 391 the session. 392 <p> 393 The encoder may reject the encoding session if the configured (or default if unspecified) profile 394 does not support the codec input (mainly the {@link MediaFormat#KEY_COLOR_FORMAT color format} for 395 video/image codecs, or the {@link MediaFormat#KEY_PCM_ENCODING sample encoding} and the {@link 396 MediaFormat#KEY_CHANNEL_COUNT number of channels} for audio codecs, but also possibly 397 {@link MediaFormat#KEY_WIDTH width}, {@link MediaFormat#KEY_HEIGHT height}, 398 {@link MediaFormat#KEY_FRAME_RATE frame rate}, {@link MediaFormat#KEY_BIT_RATE bitrate} or 399 {@link MediaFormat#KEY_SAMPLE_RATE sample rate}.) 400 Alternatively, the encoder may choose to (but is not required to) convert the input to support the 401 selected (or default) profile - or adjust the chosen profile based on the presumed or detected 402 input format - to ensure a successful encoding session. <b>Note</b>: Converting the input to match 403 an incompatible profile will in most cases result in decreased codec performance. 404 <p> 405 To ensure backward compatibility, the following guarantees are provided by Android: 406 <ul> 407 <li>The default video encoder profile always supports 8-bit YUV 4:2:0 color format ({@link 408 CodecCapabilities#COLOR_FormatYUV420Flexible COLOR_FormatYUV420Flexible} and equivalent 409 {@link CodecCapabilities#colorFormats supported formats}) for both Surface and ByteBuffer modes. 410 <li>The default video encoder profile always supports the default 8-bit RGBA color format in 411 Surface mode even if no such formats are enumerated in the {@link CodecCapabilities#colorFormats 412 supported formats}. 413 </ul> 414 <p class=note> 415 <b>Note</b>: the accepted profile can be queried through the {@link #getOutputFormat output 416 format} of the encoder after {@code configure} to allow applications to set up their 417 codec input to a format supported by the encoder profile. 418 <p> 419 <b>Implication:</b> 420 <ul> 421 <li>Applications that want to encode 4:2:2, 4:4:4, 10+ bit or HDR video input <b>MUST</b> configure 422 a suitable profile for encoders. 423 </ul> 424 425 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 426 <p> 427 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 428 to be prefixed by a number of buffers containing setup data, or codec specific data. When 429 processing such compressed formats, this data must be submitted to the codec after {@link 430 #start} and before any frame data. Such data must be marked using the flag {@link 431 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 432 <p> 433 Codec-specific data can also be included in the format passed to {@link #configure configure} in 434 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 435 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 436 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 437 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 438 specific data, you can choose to submit it using the specified number of buffers in the correct 439 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 440 codec-specific data and submit it as a single codec-config buffer. 441 <p> 442 Android uses the following codec-specific data buffers. These are also required to be set in 443 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 444 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 445 {@code "\x00\x00\x00\x01"}. 446 <p> 447 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 448 <table> 449 <thead> 450 <th>Format</th> 451 <th>CSD buffer #0</th> 452 <th>CSD buffer #1</th> 453 <th>CSD buffer #2</th> 454 </thead> 455 <tbody class=mid> 456 <tr> 457 <td>AAC</td> 458 <td>Decoder-specific information from ESDS<sup>*</sup></td> 459 <td class=NA>Not Used</td> 460 <td class=NA>Not Used</td> 461 </tr> 462 <tr> 463 <td>VORBIS</td> 464 <td>Identification header</td> 465 <td>Setup header</td> 466 <td class=NA>Not Used</td> 467 </tr> 468 <tr> 469 <td>OPUS</td> 470 <td>Identification header</td> 471 <td>Pre-skip in nanosecs<br> 472 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 473 This overrides the pre-skip value in the identification header.</td> 474 <td>Seek Pre-roll in nanosecs<br> 475 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 476 </tr> 477 <tr> 478 <td>FLAC</td> 479 <td>"fLaC", the FLAC stream marker in ASCII,<br> 480 followed by the STREAMINFO block (the mandatory metadata block),<br> 481 optionally followed by any number of other metadata blocks</td> 482 <td class=NA>Not Used</td> 483 <td class=NA>Not Used</td> 484 </tr> 485 <tr> 486 <td>MPEG-4</td> 487 <td>Decoder-specific information from ESDS<sup>*</sup></td> 488 <td class=NA>Not Used</td> 489 <td class=NA>Not Used</td> 490 </tr> 491 <tr> 492 <td>H.264 AVC</td> 493 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 494 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 495 <td class=NA>Not Used</td> 496 </tr> 497 <tr> 498 <td>H.265 HEVC</td> 499 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 500 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 501 PPS (Picture Parameter Sets<sup>*</sup>)</td> 502 <td class=NA>Not Used</td> 503 <td class=NA>Not Used</td> 504 </tr> 505 <tr> 506 <td>VP9</td> 507 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 508 (optional)</td> 509 <td class=NA>Not Used</td> 510 <td class=NA>Not Used</td> 511 </tr> 512 <tr> 513 <td>AV1</td> 514 <td>AV1 <a href="https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-syntax"> 515 AV1CodecConfigurationRecord</a> Data (optional) 516 </td> 517 <td class=NA>Not Used</td> 518 <td class=NA>Not Used</td> 519 </tr> 520 </tbody> 521 </table> 522 523 <p class=note> 524 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 525 after start, before any output buffer or output format change has been returned, as the codec 526 specific data may be lost during the flush. You must resubmit the data using buffers marked with 527 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 528 <p> 529 Encoders (or codecs that generate compressed data) will create and return the codec specific data 530 before any valid output buffer in output buffers marked with the {@linkplain 531 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 532 meaningful timestamps. 533 534 <h3>Data Processing</h3> 535 <p> 536 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 537 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 538 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 539 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 540 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 541 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 542 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 543 <p> 544 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 545 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 546 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 547 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 548 <p> 549 The codec in turn will return a read-only output buffer via the {@link 550 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 551 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 552 output buffer has been processed, call one of the {@link #releaseOutputBuffer 553 releaseOutputBuffer} methods to return the buffer to the codec. 554 <p> 555 While you are not required to resubmit/release buffers immediately to the codec, holding onto 556 input and/or output buffers may stall the codec, and this behavior is device dependent. 557 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 558 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 559 hold onto to available buffers as little as possible. 560 <p> 561 Depending on the API version, you can process data in three ways: 562 <table> 563 <thead> 564 <tr> 565 <th>Processing Mode</th> 566 <th>API version <= 20<br>Jelly Bean/KitKat</th> 567 <th>API version >= 21<br>Lollipop and later</th> 568 </tr> 569 </thead> 570 <tbody> 571 <tr> 572 <td>Synchronous API using buffer arrays</td> 573 <td>Supported</td> 574 <td>Deprecated</td> 575 </tr> 576 <tr> 577 <td>Synchronous API using buffers</td> 578 <td class=NA>Not Available</td> 579 <td>Supported</td> 580 </tr> 581 <tr> 582 <td>Asynchronous API using buffers</td> 583 <td class=NA>Not Available</td> 584 <td>Supported</td> 585 </tr> 586 </tbody> 587 </table> 588 589 <h4>Asynchronous Processing using Buffers</h4> 590 <p> 591 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 592 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 593 mode changes the state transitions slightly, because you must call {@link #start} after {@link 594 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 595 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 596 sub-state and start passing available input buffers via the callback. 597 <p> 598 <center> 599 <img src="../../../images/media/mediacodec_async_states.svg" style="width: 516px; height: 353px" 600 alt="MediaCodec state diagram for asynchronous operation"> 601 </center> 602 <p> 603 MediaCodec is typically used like this in asynchronous mode: 604 <pre class=prettyprint> 605 MediaCodec codec = MediaCodec.createByCodecName(name); 606 MediaFormat mOutputFormat; // member variable 607 codec.setCallback(new MediaCodec.Callback() { 608 {@literal @Override} 609 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 610 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 611 // fill inputBuffer with valid data 612 … 613 codec.queueInputBuffer(inputBufferId, …); 614 } 615 616 {@literal @Override} 617 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 618 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 619 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 620 // bufferFormat is equivalent to mOutputFormat 621 // outputBuffer is ready to be processed or rendered. 622 … 623 codec.releaseOutputBuffer(outputBufferId, …); 624 } 625 626 {@literal @Override} 627 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 628 // Subsequent data will conform to new format. 629 // Can ignore if using getOutputFormat(outputBufferId) 630 mOutputFormat = format; // option B 631 } 632 633 {@literal @Override} 634 void onError(…) { 635 … 636 } 637 {@literal @Override} 638 void onCryptoError(…) { 639 … 640 } 641 }); 642 codec.configure(format, …); 643 mOutputFormat = codec.getOutputFormat(); // option B 644 codec.start(); 645 // wait for processing to complete 646 codec.stop(); 647 codec.release();</pre> 648 649 <h4>Synchronous Processing using Buffers</h4> 650 <p> 651 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 652 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 653 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 654 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 655 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 656 getInput}/{@link #getOutputBuffers OutputBuffers()}. 657 658 <p class=note> 659 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 660 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 661 #start} or after having dequeued an output buffer ID with the value of {@link 662 #INFO_OUTPUT_FORMAT_CHANGED}. 663 <p> 664 MediaCodec is typically used like this in synchronous mode: 665 <pre> 666 MediaCodec codec = MediaCodec.createByCodecName(name); 667 codec.configure(format, …); 668 MediaFormat outputFormat = codec.getOutputFormat(); // option B 669 codec.start(); 670 for (;;) { 671 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 672 if (inputBufferId >= 0) { 673 ByteBuffer inputBuffer = codec.getInputBuffer(…); 674 // fill inputBuffer with valid data 675 … 676 codec.queueInputBuffer(inputBufferId, …); 677 } 678 int outputBufferId = codec.dequeueOutputBuffer(…); 679 if (outputBufferId >= 0) { 680 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 681 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 682 // bufferFormat is identical to outputFormat 683 // outputBuffer is ready to be processed or rendered. 684 … 685 codec.releaseOutputBuffer(outputBufferId, …); 686 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 687 // Subsequent data will conform to new format. 688 // Can ignore if using getOutputFormat(outputBufferId) 689 outputFormat = codec.getOutputFormat(); // option B 690 } 691 } 692 codec.stop(); 693 codec.release();</pre> 694 695 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 696 <p> 697 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 698 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 699 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 700 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 701 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 702 between the size of the arrays and the number of input and output buffers used by the system, 703 although the array size provides an upper bound. 704 <pre> 705 MediaCodec codec = MediaCodec.createByCodecName(name); 706 codec.configure(format, …); 707 codec.start(); 708 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 709 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 710 for (;;) { 711 int inputBufferId = codec.dequeueInputBuffer(…); 712 if (inputBufferId >= 0) { 713 // fill inputBuffers[inputBufferId] with valid data 714 … 715 codec.queueInputBuffer(inputBufferId, …); 716 } 717 int outputBufferId = codec.dequeueOutputBuffer(…); 718 if (outputBufferId >= 0) { 719 // outputBuffers[outputBufferId] is ready to be processed or rendered. 720 … 721 codec.releaseOutputBuffer(outputBufferId, …); 722 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 723 outputBuffers = codec.getOutputBuffers(); 724 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 725 // Subsequent data will conform to new format. 726 MediaFormat format = codec.getOutputFormat(); 727 } 728 } 729 codec.stop(); 730 codec.release();</pre> 731 732 <h4>End-of-stream Handling</h4> 733 <p> 734 When you reach the end of the input data, you must signal it to the codec by specifying the 735 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 736 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 737 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 738 be ignored. 739 <p> 740 The codec will continue to return output buffers until it eventually signals the end of the 741 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 742 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 743 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 744 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 745 <p> 746 Do not submit additional input buffers after signaling the end of the input stream, unless the 747 codec has been flushed, or stopped and restarted. 748 749 <h4>Using an Output Surface</h4> 750 <p> 751 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 752 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 753 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 754 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 755 null}-s. 756 <p> 757 When using an output Surface, you can select whether or not to render each output buffer on the 758 surface. You have three choices: 759 <ul> 760 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 761 releaseOutputBuffer(bufferId, false)}.</li> 762 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 763 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 764 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 765 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 766 </ul> 767 <p> 768 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 769 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 770 It was not defined prior to that. 771 <p> 772 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 773 dynamically using {@link #setOutputSurface setOutputSurface}. 774 <p> 775 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 776 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 777 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 778 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 779 was undefined, with the exception that View surfaces (SurfaceView or TextureView) always dropped 780 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 781 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 782 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 783 setting the key {@link MediaFormat#KEY_ALLOW_FRAME_DROP} to {@code 0} 784 in their configure format. 785 786 <h4>Transformations When Rendering onto Surface</h4> 787 788 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 789 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 790 mode} will be automatically applied with one exception: 791 <p class=note> 792 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 793 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 794 and simple way to identify software decoders, or if they apply the rotation other than by trying 795 it out. 796 <p> 797 There are also some caveats. 798 <p class=note> 799 Note that the pixel aspect ratio is not considered when displaying the output onto the 800 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 801 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 802 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 803 square pixels (pixel aspect ratio or 1:1). 804 <p class=note> 805 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 806 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 807 by 90 or 270 degrees. 808 <p class=note> 809 When setting the video scaling mode, note that it must be reset after each time the output 810 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 811 do this after each time the output format changes. 812 813 <h4>Using an Input Surface</h4> 814 <p> 815 When using an input Surface, there are no accessible input buffers, as buffers are automatically 816 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 817 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 818 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 819 <p> 820 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 821 submitting data to the codec immediately after this call. 822 <p> 823 824 <h3>Seeking & Adaptive Playback Support</h3> 825 <p> 826 Video decoders (and in general codecs that consume compressed video data) behave differently 827 regarding seek and format change whether or not they support and are configured for adaptive 828 playback. You can check if a decoder supports {@linkplain 829 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 830 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 831 playback support for video decoders is only activated if you configure the codec to decode onto a 832 {@link Surface}. 833 834 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 835 <p> 836 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 837 stream boundary: the first frame must be a key frame. A <em>key frame</em> can be decoded 838 completely on its own (for most codecs this means an I-frame), and no frames that are to be 839 displayed after a key frame refer to frames before the key frame. 840 <p> 841 The following table summarizes suitable key frames for various video formats. 842 <table> 843 <thead> 844 <tr> 845 <th>Format</th> 846 <th>Suitable key frame</th> 847 </tr> 848 </thead> 849 <tbody class=mid> 850 <tr> 851 <td>VP9/VP8</td> 852 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 853 <i>(There is no specific name for such key frame.)</i></td> 854 </tr> 855 <tr> 856 <td>H.265 HEVC</td> 857 <td>IDR or CRA</td> 858 </tr> 859 <tr> 860 <td>H.264 AVC</td> 861 <td>IDR</td> 862 </tr> 863 <tr> 864 <td>MPEG-4<br>H.263<br>MPEG-2</td> 865 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 866 <i>(There is no specific name for such key frame.)</td> 867 </tr> 868 </tbody> 869 </table> 870 871 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 872 Surface)</h4> 873 <p> 874 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 875 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 876 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 877 before you call {@code flush}. It is important that the input data after a flush starts at a 878 suitable stream boundary/key frame. 879 <p class=note> 880 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 881 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 882 #configure configure(…)} - {@link #start} cycle is necessary. 883 884 <p class=note> 885 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 886 generally, before the first output buffer or output format change is received – you 887 will need to resubmit the codec-specific-data to the codec. See the <a 888 href="#CSD">codec-specific-data section</a> for more info. 889 890 <h4>For decoders that support and are configured for adaptive playback</h4> 891 <p> 892 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 893 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 894 discontinuity must start at a suitable stream boundary/key frame. 895 <p> 896 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 897 picture size or configuration mid-stream. To do this you must package the entire new 898 codec-specific configuration data together with the key frame into a single buffer (including 899 any start codes), and submit it as a <strong>regular</strong> input buffer. 900 <p> 901 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 902 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputFormatChanged 903 onOutputFormatChanged} callback just after the picture-size change takes place and before any 904 frames with the new size have been returned. 905 <p class=note> 906 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 907 {@link #flush} shortly after you have changed the picture size. If you have not received 908 confirmation of the picture size change, you will need to repeat the request for the new picture 909 size. 910 911 <h3>Error handling</h3> 912 <p> 913 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 914 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 915 which you must catch or declare to pass up. MediaCodec methods throw {@code 916 IllegalStateException} when the method is called from a codec state that does not allow it; this 917 is typically due to incorrect application API usage. Methods involving secure buffers may throw 918 {@link CryptoException}, which has further error information obtainable from {@link 919 CryptoException#getErrorCode}. 920 <p> 921 Internal codec errors result in a {@link CodecException}, which may be due to media content 922 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 923 correctly using the API. The recommended action when receiving a {@code CodecException} 924 can be determined by calling {@link CodecException#isRecoverable} and {@link 925 CodecException#isTransient}: 926 <ul> 927 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 928 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 929 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 930 temporarily unavailable and the method may be retried at a later time.</li> 931 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 932 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 933 reset} or {@linkplain #release released}.</li> 934 </ul> 935 <p> 936 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 937 938 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 939 <p> 940 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 941 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 942 943 <style> 944 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 945 .api > tr > th { vertical-align: bottom; } 946 .api > tr > td { vertical-align: middle; } 947 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 948 .fn { text-align: left; } 949 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 950 .deg45 { 951 white-space: nowrap; background: none; border: none; vertical-align: bottom; 952 width: 30px; height: 83px; 953 } 954 .deg45 > div { 955 transform: skew(-45deg, 0deg) translate(1px, -67px); 956 transform-origin: bottom left 0; 957 width: 30px; height: 20px; 958 } 959 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 960 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 961 </style> 962 963 <table align="right" style="width: 0%"> 964 <thead> 965 <tr><th>Symbol</th><th>Meaning</th></tr> 966 </thead> 967 <tbody class=sml> 968 <tr><td>●</td><td>Supported</td></tr> 969 <tr><td>⁕</td><td>Semantics changed</td></tr> 970 <tr><td>○</td><td>Experimental support</td></tr> 971 <tr><td>[ ]</td><td>Deprecated</td></tr> 972 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 973 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 974 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 975 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 976 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 977 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 978 </tbody> 979 </table> 980 981 <table style="width: 100%;"> 982 <thead class=api> 983 <tr> 984 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 985 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 986 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 987 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 988 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 989 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 990 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 991 <th></th> 992 <th colspan="8">SDK Version</th> 993 </tr> 994 <tr> 995 <th colspan="7">State</th> 996 <th>Method</th> 997 <th>16</th> 998 <th>17</th> 999 <th>18</th> 1000 <th>19</th> 1001 <th>20</th> 1002 <th>21</th> 1003 <th>22</th> 1004 <th>23</th> 1005 </tr> 1006 </thead> 1007 <tbody class=api> 1008 <tr> 1009 <td></td> 1010 <td></td> 1011 <td></td> 1012 <td></td> 1013 <td></td> 1014 <td></td> 1015 <td></td> 1016 <td class=fn>{@link #createByCodecName createByCodecName}</td> 1017 <td>●</td> 1018 <td>●</td> 1019 <td>●</td> 1020 <td>●</td> 1021 <td>●</td> 1022 <td>●</td> 1023 <td>●</td> 1024 <td>●</td> 1025 </tr> 1026 <tr> 1027 <td></td> 1028 <td></td> 1029 <td></td> 1030 <td></td> 1031 <td></td> 1032 <td></td> 1033 <td></td> 1034 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 1035 <td>●</td> 1036 <td>●</td> 1037 <td>●</td> 1038 <td>●</td> 1039 <td>●</td> 1040 <td>●</td> 1041 <td>●</td> 1042 <td>●</td> 1043 </tr> 1044 <tr> 1045 <td></td> 1046 <td></td> 1047 <td></td> 1048 <td></td> 1049 <td></td> 1050 <td></td> 1051 <td></td> 1052 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 1053 <td>●</td> 1054 <td>●</td> 1055 <td>●</td> 1056 <td>●</td> 1057 <td>●</td> 1058 <td>●</td> 1059 <td>●</td> 1060 <td>●</td> 1061 </tr> 1062 <tr> 1063 <td></td> 1064 <td></td> 1065 <td></td> 1066 <td></td> 1067 <td></td> 1068 <td></td> 1069 <td></td> 1070 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 1071 <td></td> 1072 <td></td> 1073 <td></td> 1074 <td></td> 1075 <td></td> 1076 <td></td> 1077 <td></td> 1078 <td>●</td> 1079 </tr> 1080 <tr> 1081 <td>16+</td> 1082 <td>-</td> 1083 <td>-</td> 1084 <td>-</td> 1085 <td>-</td> 1086 <td>-</td> 1087 <td>-</td> 1088 <td class=fn>{@link #configure configure}</td> 1089 <td>●</td> 1090 <td>●</td> 1091 <td>●</td> 1092 <td>●</td> 1093 <td>●</td> 1094 <td>⁕</td> 1095 <td>●</td> 1096 <td>●</td> 1097 </tr> 1098 <tr> 1099 <td>-</td> 1100 <td>18+</td> 1101 <td>-</td> 1102 <td>-</td> 1103 <td>-</td> 1104 <td>-</td> 1105 <td>-</td> 1106 <td class=fn>{@link #createInputSurface createInputSurface}</td> 1107 <td></td> 1108 <td></td> 1109 <td>⎋</td> 1110 <td>⎋</td> 1111 <td>⎋</td> 1112 <td>⎋</td> 1113 <td>⎋</td> 1114 <td>⎋</td> 1115 </tr> 1116 <tr> 1117 <td>-</td> 1118 <td>-</td> 1119 <td>16+</td> 1120 <td>16+</td> 1121 <td>(16+)</td> 1122 <td>-</td> 1123 <td>-</td> 1124 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 1125 <td>●</td> 1126 <td>●</td> 1127 <td>▧</td> 1128 <td>▧</td> 1129 <td>▧</td> 1130 <td>⁕▧↩</td> 1131 <td>▧↩</td> 1132 <td>▧↩</td> 1133 </tr> 1134 <tr> 1135 <td>-</td> 1136 <td>-</td> 1137 <td>16+</td> 1138 <td>16+</td> 1139 <td>16+</td> 1140 <td>-</td> 1141 <td>-</td> 1142 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1143 <td>●</td> 1144 <td>●</td> 1145 <td>●</td> 1146 <td>●</td> 1147 <td>●</td> 1148 <td>⁕↩</td> 1149 <td>↩</td> 1150 <td>↩</td> 1151 </tr> 1152 <tr> 1153 <td>-</td> 1154 <td>-</td> 1155 <td>16+</td> 1156 <td>16+</td> 1157 <td>16+</td> 1158 <td>-</td> 1159 <td>-</td> 1160 <td class=fn>{@link #flush flush}</td> 1161 <td>●</td> 1162 <td>●</td> 1163 <td>●</td> 1164 <td>●</td> 1165 <td>●</td> 1166 <td>●</td> 1167 <td>●</td> 1168 <td>●</td> 1169 </tr> 1170 <tr> 1171 <td>18+</td> 1172 <td>18+</td> 1173 <td>18+</td> 1174 <td>18+</td> 1175 <td>18+</td> 1176 <td>18+</td> 1177 <td>-</td> 1178 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1179 <td></td> 1180 <td></td> 1181 <td>●</td> 1182 <td>●</td> 1183 <td>●</td> 1184 <td>●</td> 1185 <td>●</td> 1186 <td>●</td> 1187 </tr> 1188 <tr> 1189 <td>-</td> 1190 <td>-</td> 1191 <td>(21+)</td> 1192 <td>21+</td> 1193 <td>(21+)</td> 1194 <td>-</td> 1195 <td>-</td> 1196 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1197 <td></td> 1198 <td></td> 1199 <td></td> 1200 <td></td> 1201 <td></td> 1202 <td>●</td> 1203 <td>●</td> 1204 <td>●</td> 1205 </tr> 1206 <tr> 1207 <td>-</td> 1208 <td>-</td> 1209 <td>16+</td> 1210 <td>(16+)</td> 1211 <td>(16+)</td> 1212 <td>-</td> 1213 <td>-</td> 1214 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1215 <td>●</td> 1216 <td>●</td> 1217 <td>●</td> 1218 <td>●</td> 1219 <td>●</td> 1220 <td>[⁕↩]</td> 1221 <td>[↩]</td> 1222 <td>[↩]</td> 1223 </tr> 1224 <tr> 1225 <td>-</td> 1226 <td>21+</td> 1227 <td>(21+)</td> 1228 <td>(21+)</td> 1229 <td>(21+)</td> 1230 <td>-</td> 1231 <td>-</td> 1232 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1233 <td></td> 1234 <td></td> 1235 <td></td> 1236 <td></td> 1237 <td></td> 1238 <td>●</td> 1239 <td>●</td> 1240 <td>●</td> 1241 </tr> 1242 <tr> 1243 <td>-</td> 1244 <td>-</td> 1245 <td>(21+)</td> 1246 <td>21+</td> 1247 <td>(21+)</td> 1248 <td>-</td> 1249 <td>-</td> 1250 <td class=fn>{@link #getInputImage getInputImage}</td> 1251 <td></td> 1252 <td></td> 1253 <td></td> 1254 <td></td> 1255 <td></td> 1256 <td>○</td> 1257 <td>●</td> 1258 <td>●</td> 1259 </tr> 1260 <tr> 1261 <td>18+</td> 1262 <td>18+</td> 1263 <td>18+</td> 1264 <td>18+</td> 1265 <td>18+</td> 1266 <td>18+</td> 1267 <td>-</td> 1268 <td class=fn>{@link #getName getName}</td> 1269 <td></td> 1270 <td></td> 1271 <td>●</td> 1272 <td>●</td> 1273 <td>●</td> 1274 <td>●</td> 1275 <td>●</td> 1276 <td>●</td> 1277 </tr> 1278 <tr> 1279 <td>-</td> 1280 <td>-</td> 1281 <td>(21+)</td> 1282 <td>21+</td> 1283 <td>21+</td> 1284 <td>-</td> 1285 <td>-</td> 1286 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1287 <td></td> 1288 <td></td> 1289 <td></td> 1290 <td></td> 1291 <td></td> 1292 <td>●</td> 1293 <td>●</td> 1294 <td>●</td> 1295 </tr> 1296 <tr> 1297 <td>-</td> 1298 <td>-</td> 1299 <td>16+</td> 1300 <td>16+</td> 1301 <td>16+</td> 1302 <td>-</td> 1303 <td>-</td> 1304 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1305 <td>●</td> 1306 <td>●</td> 1307 <td>●</td> 1308 <td>●</td> 1309 <td>●</td> 1310 <td>[⁕↩]</td> 1311 <td>[↩]</td> 1312 <td>[↩]</td> 1313 </tr> 1314 <tr> 1315 <td>-</td> 1316 <td>21+</td> 1317 <td>16+</td> 1318 <td>16+</td> 1319 <td>16+</td> 1320 <td>-</td> 1321 <td>-</td> 1322 <td class=fn>{@link #getOutputFormat()}</td> 1323 <td>●</td> 1324 <td>●</td> 1325 <td>●</td> 1326 <td>●</td> 1327 <td>●</td> 1328 <td>●</td> 1329 <td>●</td> 1330 <td>●</td> 1331 </tr> 1332 <tr> 1333 <td>-</td> 1334 <td>-</td> 1335 <td>(21+)</td> 1336 <td>21+</td> 1337 <td>21+</td> 1338 <td>-</td> 1339 <td>-</td> 1340 <td class=fn>{@link #getOutputFormat(int)}</td> 1341 <td></td> 1342 <td></td> 1343 <td></td> 1344 <td></td> 1345 <td></td> 1346 <td>●</td> 1347 <td>●</td> 1348 <td>●</td> 1349 </tr> 1350 <tr> 1351 <td>-</td> 1352 <td>-</td> 1353 <td>(21+)</td> 1354 <td>21+</td> 1355 <td>21+</td> 1356 <td>-</td> 1357 <td>-</td> 1358 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1359 <td></td> 1360 <td></td> 1361 <td></td> 1362 <td></td> 1363 <td></td> 1364 <td>○</td> 1365 <td>●</td> 1366 <td>●</td> 1367 </tr> 1368 <tr> 1369 <td>-</td> 1370 <td>-</td> 1371 <td>-</td> 1372 <td>16+</td> 1373 <td>(16+)</td> 1374 <td>-</td> 1375 <td>-</td> 1376 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1377 <td>●</td> 1378 <td>●</td> 1379 <td>●</td> 1380 <td>●</td> 1381 <td>●</td> 1382 <td>⁕</td> 1383 <td>●</td> 1384 <td>●</td> 1385 </tr> 1386 <tr> 1387 <td>-</td> 1388 <td>-</td> 1389 <td>-</td> 1390 <td>16+</td> 1391 <td>(16+)</td> 1392 <td>-</td> 1393 <td>-</td> 1394 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1395 <td>●</td> 1396 <td>●</td> 1397 <td>●</td> 1398 <td>●</td> 1399 <td>●</td> 1400 <td>⁕</td> 1401 <td>●</td> 1402 <td>●</td> 1403 </tr> 1404 <tr> 1405 <td>16+</td> 1406 <td>16+</td> 1407 <td>16+</td> 1408 <td>16+</td> 1409 <td>16+</td> 1410 <td>16+</td> 1411 <td>16+</td> 1412 <td class=fn>{@link #release release}</td> 1413 <td>●</td> 1414 <td>●</td> 1415 <td>●</td> 1416 <td>●</td> 1417 <td>●</td> 1418 <td>●</td> 1419 <td>●</td> 1420 <td>●</td> 1421 </tr> 1422 <tr> 1423 <td>-</td> 1424 <td>-</td> 1425 <td>-</td> 1426 <td>16+</td> 1427 <td>16+</td> 1428 <td>-</td> 1429 <td>-</td> 1430 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1431 <td>●</td> 1432 <td>●</td> 1433 <td>●</td> 1434 <td>●</td> 1435 <td>●</td> 1436 <td>⁕</td> 1437 <td>●</td> 1438 <td>⁕</td> 1439 </tr> 1440 <tr> 1441 <td>-</td> 1442 <td>-</td> 1443 <td>-</td> 1444 <td>21+</td> 1445 <td>21+</td> 1446 <td>-</td> 1447 <td>-</td> 1448 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1449 <td></td> 1450 <td></td> 1451 <td></td> 1452 <td></td> 1453 <td></td> 1454 <td>⎆</td> 1455 <td>⎆</td> 1456 <td>⎆</td> 1457 </tr> 1458 <tr> 1459 <td>21+</td> 1460 <td>21+</td> 1461 <td>21+</td> 1462 <td>21+</td> 1463 <td>21+</td> 1464 <td>21+</td> 1465 <td>-</td> 1466 <td class=fn>{@link #reset reset}</td> 1467 <td></td> 1468 <td></td> 1469 <td></td> 1470 <td></td> 1471 <td></td> 1472 <td>●</td> 1473 <td>●</td> 1474 <td>●</td> 1475 </tr> 1476 <tr> 1477 <td>21+</td> 1478 <td>-</td> 1479 <td>-</td> 1480 <td>-</td> 1481 <td>-</td> 1482 <td>-</td> 1483 <td>-</td> 1484 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1485 <td></td> 1486 <td></td> 1487 <td></td> 1488 <td></td> 1489 <td></td> 1490 <td>●</td> 1491 <td>●</td> 1492 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1493 </tr> 1494 <tr> 1495 <td>-</td> 1496 <td>23+</td> 1497 <td>-</td> 1498 <td>-</td> 1499 <td>-</td> 1500 <td>-</td> 1501 <td>-</td> 1502 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1503 <td></td> 1504 <td></td> 1505 <td></td> 1506 <td></td> 1507 <td></td> 1508 <td></td> 1509 <td></td> 1510 <td>⎋</td> 1511 </tr> 1512 <tr> 1513 <td>23+</td> 1514 <td>23+</td> 1515 <td>23+</td> 1516 <td>23+</td> 1517 <td>23+</td> 1518 <td>(23+)</td> 1519 <td>(23+)</td> 1520 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1521 <td></td> 1522 <td></td> 1523 <td></td> 1524 <td></td> 1525 <td></td> 1526 <td></td> 1527 <td></td> 1528 <td>○ ⎆</td> 1529 </tr> 1530 <tr> 1531 <td>-</td> 1532 <td>23+</td> 1533 <td>23+</td> 1534 <td>23+</td> 1535 <td>23+</td> 1536 <td>-</td> 1537 <td>-</td> 1538 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1539 <td></td> 1540 <td></td> 1541 <td></td> 1542 <td></td> 1543 <td></td> 1544 <td></td> 1545 <td></td> 1546 <td>⎆</td> 1547 </tr> 1548 <tr> 1549 <td>19+</td> 1550 <td>19+</td> 1551 <td>19+</td> 1552 <td>19+</td> 1553 <td>19+</td> 1554 <td>(19+)</td> 1555 <td>-</td> 1556 <td class=fn>{@link #setParameters setParameters}</td> 1557 <td></td> 1558 <td></td> 1559 <td></td> 1560 <td>●</td> 1561 <td>●</td> 1562 <td>●</td> 1563 <td>●</td> 1564 <td>●</td> 1565 </tr> 1566 <tr> 1567 <td>-</td> 1568 <td>(16+)</td> 1569 <td>(16+)</td> 1570 <td>16+</td> 1571 <td>(16+)</td> 1572 <td>(16+)</td> 1573 <td>-</td> 1574 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1575 <td>⎆</td> 1576 <td>⎆</td> 1577 <td>⎆</td> 1578 <td>⎆</td> 1579 <td>⎆</td> 1580 <td>⎆</td> 1581 <td>⎆</td> 1582 <td>⎆</td> 1583 </tr> 1584 <tr> 1585 <td>(29+)</td> 1586 <td>29+</td> 1587 <td>29+</td> 1588 <td>29+</td> 1589 <td>(29+)</td> 1590 <td>(29+)</td> 1591 <td>-</td> 1592 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1593 <td></td> 1594 <td></td> 1595 <td></td> 1596 <td></td> 1597 <td></td> 1598 <td></td> 1599 <td></td> 1600 <td></td> 1601 </tr> 1602 <tr> 1603 <td>-</td> 1604 <td>-</td> 1605 <td>18+</td> 1606 <td>18+</td> 1607 <td>-</td> 1608 <td>-</td> 1609 <td>-</td> 1610 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1611 <td></td> 1612 <td></td> 1613 <td>⎋</td> 1614 <td>⎋</td> 1615 <td>⎋</td> 1616 <td>⎋</td> 1617 <td>⎋</td> 1618 <td>⎋</td> 1619 </tr> 1620 <tr> 1621 <td>-</td> 1622 <td>16+</td> 1623 <td>21+(⇄)</td> 1624 <td>-</td> 1625 <td>-</td> 1626 <td>-</td> 1627 <td>-</td> 1628 <td class=fn>{@link #start start}</td> 1629 <td>●</td> 1630 <td>●</td> 1631 <td>●</td> 1632 <td>●</td> 1633 <td>●</td> 1634 <td>⁕</td> 1635 <td>●</td> 1636 <td>●</td> 1637 </tr> 1638 <tr> 1639 <td>-</td> 1640 <td>-</td> 1641 <td>16+</td> 1642 <td>16+</td> 1643 <td>16+</td> 1644 <td>-</td> 1645 <td>-</td> 1646 <td class=fn>{@link #stop stop}</td> 1647 <td>●</td> 1648 <td>●</td> 1649 <td>●</td> 1650 <td>●</td> 1651 <td>●</td> 1652 <td>●</td> 1653 <td>●</td> 1654 <td>●</td> 1655 </tr> 1656 </tbody> 1657 </table> 1658 */ 1659 final public class MediaCodec { 1660 private static final String TAG = "MediaCodec"; 1661 1662 /** 1663 * Per buffer metadata includes an offset and size specifying 1664 * the range of valid data in the associated codec (output) buffer. 1665 */ 1666 public final static class BufferInfo { 1667 /** 1668 * Update the buffer metadata information. 1669 * 1670 * @param newOffset the start-offset of the data in the buffer. 1671 * @param newSize the amount of data (in bytes) in the buffer. 1672 * @param newTimeUs the presentation timestamp in microseconds. 1673 * @param newFlags buffer flags associated with the buffer. This 1674 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1675 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1676 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1677 public void set( 1678 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1679 offset = newOffset; 1680 size = newSize; 1681 presentationTimeUs = newTimeUs; 1682 flags = newFlags; 1683 } 1684 1685 /** 1686 * The start-offset of the data in the buffer. 1687 */ 1688 public int offset; 1689 1690 /** 1691 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1692 * the buffer has no data in it and can be discarded. The only 1693 * use of a 0-size buffer is to carry the end-of-stream marker. 1694 */ 1695 public int size; 1696 1697 /** 1698 * The presentation timestamp in microseconds for the buffer. 1699 * This is derived from the presentation timestamp passed in 1700 * with the corresponding input buffer. This should be ignored for 1701 * a 0-sized buffer. 1702 */ 1703 public long presentationTimeUs; 1704 1705 /** 1706 * Buffer flags associated with the buffer. A combination of 1707 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1708 * 1709 * <p>Encoded buffers that are key frames are marked with 1710 * {@link #BUFFER_FLAG_KEY_FRAME}. 1711 * 1712 * <p>The last output buffer corresponding to the input buffer 1713 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1714 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1715 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1716 * marker. 1717 */ 1718 @BufferFlag 1719 public int flags; 1720 1721 /** @hide */ 1722 @NonNull dup()1723 public BufferInfo dup() { 1724 BufferInfo copy = new BufferInfo(); 1725 copy.set(offset, size, presentationTimeUs, flags); 1726 return copy; 1727 } 1728 }; 1729 1730 // The follow flag constants MUST stay in sync with their equivalents 1731 // in MediaCodec.h ! 1732 1733 /** 1734 * This indicates that the (encoded) buffer marked as such contains 1735 * the data for a key frame. 1736 * 1737 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1738 */ 1739 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1740 1741 /** 1742 * This indicates that the (encoded) buffer marked as such contains 1743 * the data for a key frame. 1744 */ 1745 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1746 1747 /** 1748 * This indicated that the buffer marked as such contains codec 1749 * initialization / codec specific data instead of media data. 1750 */ 1751 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1752 1753 /** 1754 * This signals the end of stream, i.e. no buffers will be available 1755 * after this, unless of course, {@link #flush} follows. 1756 */ 1757 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1758 1759 /** 1760 * This indicates that the buffer only contains part of a frame, 1761 * and the decoder should batch the data until a buffer without 1762 * this flag appears before decoding the frame. 1763 */ 1764 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1765 1766 /** 1767 * This indicates that the buffer contains non-media data for the 1768 * muxer to process. 1769 * 1770 * All muxer data should start with a FOURCC header that determines the type of data. 1771 * 1772 * For example, when it contains Exif data sent to a MediaMuxer track of 1773 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1774 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1775 * 1776 * @hide 1777 */ 1778 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1779 1780 /** 1781 * This indicates that the buffer is decoded and updates the internal state of the decoder, 1782 * but does not produce any output buffer. 1783 * 1784 * When a buffer has this flag set, 1785 * {@link OnFrameRenderedListener#onFrameRendered(MediaCodec, long, long)} and 1786 * {@link Callback#onOutputBufferAvailable(MediaCodec, int, BufferInfo)} will not be called for 1787 * that given buffer. 1788 * 1789 * For example, when seeking to a certain frame, that frame may need to reference previous 1790 * frames in order for it to produce output. The preceding frames can be marked with this flag 1791 * so that they are only decoded and their data is used when decoding the latter frame that 1792 * should be initially displayed post-seek. 1793 * Another example would be trick play, trick play is when a video is fast-forwarded and only a 1794 * subset of the frames is to be rendered on the screen. The frames not to be rendered can be 1795 * marked with this flag for the same reason as the above one. 1796 * Marking frames with this flag improves the overall performance of playing a video stream as 1797 * fewer frames need to be passed back to the app. 1798 * 1799 * In {@link CodecCapabilities#FEATURE_TunneledPlayback}, buffers marked with this flag 1800 * are not rendered on the output surface. 1801 * 1802 * A frame should not be marked with this flag and {@link #BUFFER_FLAG_END_OF_STREAM} 1803 * simultaneously, doing so will produce a {@link InvalidBufferFlagsException} 1804 */ 1805 public static final int BUFFER_FLAG_DECODE_ONLY = 32; 1806 1807 /** @hide */ 1808 @IntDef( 1809 flag = true, 1810 value = { 1811 BUFFER_FLAG_SYNC_FRAME, 1812 BUFFER_FLAG_KEY_FRAME, 1813 BUFFER_FLAG_CODEC_CONFIG, 1814 BUFFER_FLAG_END_OF_STREAM, 1815 BUFFER_FLAG_PARTIAL_FRAME, 1816 BUFFER_FLAG_MUXER_DATA, 1817 BUFFER_FLAG_DECODE_ONLY, 1818 }) 1819 @Retention(RetentionPolicy.SOURCE) 1820 public @interface BufferFlag {} 1821 1822 private EventHandler mEventHandler; 1823 private EventHandler mOnFirstTunnelFrameReadyHandler; 1824 private EventHandler mOnFrameRenderedHandler; 1825 private EventHandler mCallbackHandler; 1826 private Callback mCallback; 1827 private OnFirstTunnelFrameReadyListener mOnFirstTunnelFrameReadyListener; 1828 private OnFrameRenderedListener mOnFrameRenderedListener; 1829 private final Object mListenerLock = new Object(); 1830 private MediaCodecInfo mCodecInfo; 1831 private final Object mCodecInfoLock = new Object(); 1832 private MediaCrypto mCrypto; 1833 1834 private static final int EVENT_CALLBACK = 1; 1835 private static final int EVENT_SET_CALLBACK = 2; 1836 private static final int EVENT_FRAME_RENDERED = 3; 1837 private static final int EVENT_FIRST_TUNNEL_FRAME_READY = 4; 1838 1839 private static final int CB_INPUT_AVAILABLE = 1; 1840 private static final int CB_OUTPUT_AVAILABLE = 2; 1841 private static final int CB_ERROR = 3; 1842 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1843 private static final String EOS_AND_DECODE_ONLY_ERROR_MESSAGE = "An input buffer cannot have " 1844 + "both BUFFER_FLAG_END_OF_STREAM and BUFFER_FLAG_DECODE_ONLY flags"; 1845 private static final int CB_CRYPTO_ERROR = 6; 1846 private static final int CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7; 1847 1848 /** 1849 * Callback ID for when the metrics for this codec have been flushed due to 1850 * the start of a new subsession. The associated Java Message object will 1851 * contain the flushed metrics as a PersistentBundle in the obj field. 1852 */ 1853 private static final int CB_METRICS_FLUSHED = 8; 1854 1855 /** 1856 * Callback ID to notify the change in resource requirement 1857 * for the codec component. 1858 */ 1859 private static final int CB_REQUIRED_RESOURCES_CHANGE = 9; 1860 1861 private class EventHandler extends Handler { 1862 private MediaCodec mCodec; 1863 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1864 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1865 super(looper); 1866 mCodec = codec; 1867 } 1868 1869 @Override handleMessage(@onNull Message msg)1870 public void handleMessage(@NonNull Message msg) { 1871 switch (msg.what) { 1872 case EVENT_CALLBACK: 1873 { 1874 handleCallback(msg); 1875 break; 1876 } 1877 case EVENT_SET_CALLBACK: 1878 { 1879 mCallback = (MediaCodec.Callback) msg.obj; 1880 break; 1881 } 1882 case EVENT_FRAME_RENDERED: 1883 Map<String, Object> map = (Map<String, Object>)msg.obj; 1884 for (int i = 0; ; ++i) { 1885 Object mediaTimeUs = map.get(i + "-media-time-us"); 1886 Object systemNano = map.get(i + "-system-nano"); 1887 OnFrameRenderedListener onFrameRenderedListener; 1888 synchronized (mListenerLock) { 1889 onFrameRenderedListener = mOnFrameRenderedListener; 1890 } 1891 if (mediaTimeUs == null || systemNano == null 1892 || onFrameRenderedListener == null) { 1893 break; 1894 } 1895 onFrameRenderedListener.onFrameRendered( 1896 mCodec, (long)mediaTimeUs, (long)systemNano); 1897 } 1898 break; 1899 case EVENT_FIRST_TUNNEL_FRAME_READY: 1900 OnFirstTunnelFrameReadyListener onFirstTunnelFrameReadyListener; 1901 synchronized (mListenerLock) { 1902 onFirstTunnelFrameReadyListener = mOnFirstTunnelFrameReadyListener; 1903 } 1904 if (onFirstTunnelFrameReadyListener == null) { 1905 break; 1906 } 1907 onFirstTunnelFrameReadyListener.onFirstTunnelFrameReady(mCodec); 1908 break; 1909 default: 1910 { 1911 break; 1912 } 1913 } 1914 } 1915 handleCallback(@onNull Message msg)1916 private void handleCallback(@NonNull Message msg) { 1917 if (mCallback == null) { 1918 return; 1919 } 1920 1921 switch (msg.arg1) { 1922 case CB_INPUT_AVAILABLE: 1923 { 1924 int index = msg.arg2; 1925 synchronized(mBufferLock) { 1926 switch (mBufferMode) { 1927 case BUFFER_MODE_LEGACY: 1928 validateInputByteBufferLocked(mCachedInputBuffers, index); 1929 break; 1930 case BUFFER_MODE_BLOCK: 1931 while (mQueueRequests.size() <= index) { 1932 mQueueRequests.add(null); 1933 } 1934 QueueRequest request = mQueueRequests.get(index); 1935 if (request == null) { 1936 request = new QueueRequest(mCodec, index); 1937 mQueueRequests.set(index, request); 1938 } 1939 request.setAccessible(true); 1940 break; 1941 default: 1942 throw new IllegalStateException( 1943 "Unrecognized buffer mode: " + mBufferMode); 1944 } 1945 } 1946 mCallback.onInputBufferAvailable(mCodec, index); 1947 break; 1948 } 1949 1950 case CB_OUTPUT_AVAILABLE: 1951 { 1952 int index = msg.arg2; 1953 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1954 synchronized(mBufferLock) { 1955 switch (mBufferMode) { 1956 case BUFFER_MODE_LEGACY: 1957 validateOutputByteBufferLocked(mCachedOutputBuffers, index, info); 1958 break; 1959 case BUFFER_MODE_BLOCK: 1960 while (mOutputFrames.size() <= index) { 1961 mOutputFrames.add(null); 1962 } 1963 OutputFrame frame = mOutputFrames.get(index); 1964 if (frame == null) { 1965 frame = new OutputFrame(index); 1966 mOutputFrames.set(index, frame); 1967 } 1968 frame.setBufferInfo(info); 1969 frame.setAccessible(true); 1970 break; 1971 default: 1972 throw new IllegalStateException( 1973 "Unrecognized buffer mode: " + mBufferMode); 1974 } 1975 } 1976 mCallback.onOutputBufferAvailable( 1977 mCodec, index, info); 1978 break; 1979 } 1980 1981 case CB_LARGE_FRAME_OUTPUT_AVAILABLE: 1982 { 1983 int index = msg.arg2; 1984 ArrayDeque<BufferInfo> infos = (ArrayDeque<BufferInfo>)msg.obj; 1985 synchronized(mBufferLock) { 1986 switch (mBufferMode) { 1987 case BUFFER_MODE_LEGACY: 1988 validateOutputByteBuffersLocked(mCachedOutputBuffers, 1989 index, infos); 1990 break; 1991 case BUFFER_MODE_BLOCK: 1992 while (mOutputFrames.size() <= index) { 1993 mOutputFrames.add(null); 1994 } 1995 OutputFrame frame = mOutputFrames.get(index); 1996 if (frame == null) { 1997 frame = new OutputFrame(index); 1998 mOutputFrames.set(index, frame); 1999 } 2000 frame.setBufferInfos(infos); 2001 frame.setAccessible(true); 2002 break; 2003 default: 2004 throw new IllegalArgumentException( 2005 "Unrecognized buffer mode: for large frame output"); 2006 } 2007 } 2008 mCallback.onOutputBuffersAvailable( 2009 mCodec, index, infos); 2010 2011 break; 2012 } 2013 2014 case CB_ERROR: 2015 { 2016 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 2017 break; 2018 } 2019 2020 case CB_CRYPTO_ERROR: 2021 { 2022 mCallback.onCryptoError(mCodec, (MediaCodec.CryptoException) msg.obj); 2023 break; 2024 } 2025 2026 case CB_OUTPUT_FORMAT_CHANGE: 2027 { 2028 mCallback.onOutputFormatChanged(mCodec, 2029 new MediaFormat((Map<String, Object>) msg.obj)); 2030 break; 2031 } 2032 2033 case CB_METRICS_FLUSHED: 2034 { 2035 if (GetFlag(() -> android.media.codec.Flags.subsessionMetrics())) { 2036 mCallback.onMetricsFlushed(mCodec, (PersistableBundle)msg.obj); 2037 } 2038 break; 2039 } 2040 2041 case CB_REQUIRED_RESOURCES_CHANGE: { 2042 if (android.media.codec.Flags.codecAvailability()) { 2043 mCallback.onRequiredResourcesChanged(mCodec); 2044 } 2045 break; 2046 } 2047 2048 default: 2049 { 2050 break; 2051 } 2052 } 2053 } 2054 } 2055 2056 // HACKY(b/325389296): aconfig flag accessors may not work in all contexts where MediaCodec API 2057 // is used, so allow accessors to fail. In those contexts use a default value, normally false. 2058 2059 /* package private */ GetFlag(Supplier<Boolean> flagValueSupplier)2060 static boolean GetFlag(Supplier<Boolean> flagValueSupplier) { 2061 return GetFlag(flagValueSupplier, false /* defaultValue */); 2062 } 2063 2064 /* package private */ GetFlag(Supplier<Boolean> flagValueSupplier, boolean defaultValue)2065 static boolean GetFlag(Supplier<Boolean> flagValueSupplier, boolean defaultValue) { 2066 try { 2067 return flagValueSupplier.get(); 2068 } catch (java.lang.RuntimeException e) { 2069 return defaultValue; 2070 } 2071 } 2072 2073 private boolean mHasSurface = false; 2074 2075 /** 2076 * Instantiate the preferred decoder supporting input data of the given mime type. 2077 * 2078 * The following is a partial list of defined mime types and their semantics: 2079 * <ul> 2080 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 2081 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 2082 * <li>"video/avc" - H.264/AVC video 2083 * <li>"video/hevc" - H.265/HEVC video 2084 * <li>"video/mp4v-es" - MPEG4 video 2085 * <li>"video/3gpp" - H.263 video 2086 * <li>"audio/3gpp" - AMR narrowband audio 2087 * <li>"audio/amr-wb" - AMR wideband audio 2088 * <li>"audio/mpeg" - MPEG1/2 audio layer III 2089 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 2090 * <li>"audio/vorbis" - vorbis audio 2091 * <li>"audio/g711-alaw" - G.711 alaw audio 2092 * <li>"audio/g711-mlaw" - G.711 ulaw audio 2093 * </ul> 2094 * 2095 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 2096 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 2097 * given format. 2098 * 2099 * @param type The mime type of the input data. 2100 * @throws IOException if the codec cannot be created. 2101 * @throws IllegalArgumentException if type is not a valid mime type. 2102 * @throws NullPointerException if type is null. 2103 */ 2104 @NonNull createDecoderByType(@onNull String type)2105 public static MediaCodec createDecoderByType(@NonNull String type) 2106 throws IOException { 2107 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 2108 } 2109 2110 /** 2111 * Instantiate the preferred encoder supporting output data of the given mime type. 2112 * 2113 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 2114 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 2115 * given format. 2116 * 2117 * @param type The desired mime type of the output data. 2118 * @throws IOException if the codec cannot be created. 2119 * @throws IllegalArgumentException if type is not a valid mime type. 2120 * @throws NullPointerException if type is null. 2121 */ 2122 @NonNull createEncoderByType(@onNull String type)2123 public static MediaCodec createEncoderByType(@NonNull String type) 2124 throws IOException { 2125 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 2126 } 2127 2128 /** 2129 * If you know the exact name of the component you want to instantiate 2130 * use this method to instantiate it. Use with caution. 2131 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 2132 * @param name The name of the codec to be instantiated. 2133 * @throws IOException if the codec cannot be created. 2134 * @throws IllegalArgumentException if name is not valid. 2135 * @throws NullPointerException if name is null. 2136 */ 2137 @NonNull createByCodecName(@onNull String name)2138 public static MediaCodec createByCodecName(@NonNull String name) 2139 throws IOException { 2140 return new MediaCodec(name, false /* nameIsType */, false /* encoder */); 2141 } 2142 2143 /** 2144 * This is the same as createByCodecName, but allows for instantiating a codec on behalf of a 2145 * client process. This is used for system apps or system services that create MediaCodecs on 2146 * behalf of other processes and will reclaim resources as necessary from processes with lower 2147 * priority than the client process, rather than processes with lower priority than the system 2148 * app or system service. Likely to be used with information obtained from 2149 * {@link android.media.MediaCodecList}. 2150 * @param name 2151 * @param clientPid 2152 * @param clientUid 2153 * @throws IOException if the codec cannot be created. 2154 * @throws IllegalArgumentException if name is not valid. 2155 * @throws NullPointerException if name is null. 2156 * @throws SecurityException if the MEDIA_RESOURCE_OVERRIDE_PID permission is not granted. 2157 * 2158 * @hide 2159 */ 2160 @NonNull 2161 @SystemApi 2162 @RequiresPermission(Manifest.permission.MEDIA_RESOURCE_OVERRIDE_PID) createByCodecNameForClient(@onNull String name, int clientPid, int clientUid)2163 public static MediaCodec createByCodecNameForClient(@NonNull String name, int clientPid, 2164 int clientUid) throws IOException { 2165 return new MediaCodec(name, false /* nameIsType */, false /* encoder */, clientPid, 2166 clientUid); 2167 } 2168 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder)2169 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder) { 2170 this(name, nameIsType, encoder, -1 /* pid */, -1 /* uid */); 2171 } 2172 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)2173 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder, int pid, 2174 int uid) { 2175 Looper looper; 2176 if ((looper = Looper.myLooper()) != null) { 2177 mEventHandler = new EventHandler(this, looper); 2178 } else if ((looper = Looper.getMainLooper()) != null) { 2179 mEventHandler = new EventHandler(this, looper); 2180 } else { 2181 mEventHandler = null; 2182 } 2183 mCallbackHandler = mEventHandler; 2184 mOnFirstTunnelFrameReadyHandler = mEventHandler; 2185 mOnFrameRenderedHandler = mEventHandler; 2186 2187 mBufferLock = new Object(); 2188 2189 // save name used at creation 2190 mNameAtCreation = nameIsType ? null : name; 2191 2192 native_setup(name, nameIsType, encoder, pid, uid); 2193 } 2194 2195 private String mNameAtCreation; 2196 2197 @Override finalize()2198 protected void finalize() { 2199 native_finalize(); 2200 mCrypto = null; 2201 } 2202 2203 /** 2204 * Returns the codec to its initial (Uninitialized) state. 2205 * 2206 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 2207 * error has occured to reset the codec to its initial state after creation. 2208 * 2209 * @throws CodecException if an unrecoverable error has occured and the codec 2210 * could not be reset. 2211 * @throws IllegalStateException if in the Released state. 2212 */ reset()2213 public final void reset() { 2214 freeAllTrackedBuffers(); // free buffers first 2215 native_reset(); 2216 mCrypto = null; 2217 } 2218 native_reset()2219 private native final void native_reset(); 2220 2221 /** 2222 * Free up resources used by the codec instance. 2223 * 2224 * Make sure you call this when you're done to free up any opened 2225 * component instance instead of relying on the garbage collector 2226 * to do this for you at some point in the future. 2227 */ release()2228 public final void release() { 2229 freeAllTrackedBuffers(); // free buffers first 2230 native_release(); 2231 mCrypto = null; 2232 } 2233 native_release()2234 private native final void native_release(); 2235 2236 /** 2237 * If this codec is to be used as an encoder, pass this flag. 2238 */ 2239 public static final int CONFIGURE_FLAG_ENCODE = 1; 2240 2241 /** 2242 * If this codec is to be used with {@link LinearBlock} and/or {@link 2243 * HardwareBuffer}, pass this flag. 2244 * <p> 2245 * When this flag is set, the following APIs throw {@link IncompatibleWithBlockModelException}. 2246 * <ul> 2247 * <li>{@link #getInputBuffer} 2248 * <li>{@link #getInputImage} 2249 * <li>{@link #getInputBuffers} 2250 * <li>{@link #getOutputBuffer} 2251 * <li>{@link #getOutputImage} 2252 * <li>{@link #getOutputBuffers} 2253 * <li>{@link #queueInputBuffer} 2254 * <li>{@link #queueSecureInputBuffer} 2255 * <li>{@link #dequeueInputBuffer} 2256 * <li>{@link #dequeueOutputBuffer} 2257 * </ul> 2258 */ 2259 public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; 2260 2261 /** 2262 * This flag should be used on a secure decoder only. MediaCodec configured with this 2263 * flag does decryption in a separate thread. The flag requires MediaCodec to operate 2264 * asynchronously and will throw CryptoException if any, in the onCryptoError() 2265 * callback. Applications should override the default implementation of 2266 * onCryptoError() and access the associated CryptoException. 2267 * 2268 * CryptoException thrown will contain {@link MediaCodec.CryptoInfo} 2269 * This can be accessed using getCryptoInfo() 2270 */ 2271 public static final int CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4; 2272 2273 /** 2274 * Configure the codec with a detached output surface. 2275 * <p> 2276 * This flag is only defined for a video decoder. MediaCodec 2277 * configured with this flag will be in Surface mode even though 2278 * the surface parameter is null. 2279 * 2280 * @see detachOutputSurface 2281 */ 2282 @FlaggedApi(FLAG_NULL_OUTPUT_SURFACE) 2283 public static final int CONFIGURE_FLAG_DETACHED_SURFACE = 8; 2284 2285 /** @hide */ 2286 @IntDef( 2287 flag = true, 2288 value = { 2289 CONFIGURE_FLAG_ENCODE, 2290 CONFIGURE_FLAG_USE_BLOCK_MODEL, 2291 CONFIGURE_FLAG_USE_CRYPTO_ASYNC, 2292 }) 2293 @Retention(RetentionPolicy.SOURCE) 2294 public @interface ConfigureFlag {} 2295 2296 /** 2297 * Thrown when the codec is configured for block model and an incompatible API is called. 2298 */ 2299 public class IncompatibleWithBlockModelException extends RuntimeException { IncompatibleWithBlockModelException()2300 IncompatibleWithBlockModelException() { } 2301 IncompatibleWithBlockModelException(String message)2302 IncompatibleWithBlockModelException(String message) { 2303 super(message); 2304 } 2305 IncompatibleWithBlockModelException(String message, Throwable cause)2306 IncompatibleWithBlockModelException(String message, Throwable cause) { 2307 super(message, cause); 2308 } 2309 IncompatibleWithBlockModelException(Throwable cause)2310 IncompatibleWithBlockModelException(Throwable cause) { 2311 super(cause); 2312 } 2313 } 2314 2315 /** 2316 * Thrown when a buffer is marked with an invalid combination of flags 2317 * (e.g. both {@link #BUFFER_FLAG_END_OF_STREAM} and {@link #BUFFER_FLAG_DECODE_ONLY}) 2318 */ 2319 public class InvalidBufferFlagsException extends RuntimeException { InvalidBufferFlagsException(String message)2320 InvalidBufferFlagsException(String message) { 2321 super(message); 2322 } 2323 } 2324 2325 /** 2326 * @hide 2327 * Abstraction for the Global Codec resources. 2328 * This encapsulates all the available codec resources on the device. 2329 * 2330 * To be able to enforce and test the implementation of codec availability hal APIs, 2331 * globally available codec resources are exposed only as TestApi. 2332 * This will be tracked and verified through cts. 2333 */ 2334 @FlaggedApi(FLAG_CODEC_AVAILABILITY) 2335 @TestApi 2336 public static final class GlobalResourceInfo { 2337 /** 2338 * Identifier for the Resource type. 2339 */ 2340 String mName; 2341 /** 2342 * Total count/capacity of resources of this type. 2343 */ 2344 long mCapacity; 2345 /** 2346 * Available count of this resource type. 2347 */ 2348 long mAvailable; 2349 2350 @NonNull getName()2351 public String getName() { 2352 return mName; 2353 } 2354 getCapacity()2355 public long getCapacity() { 2356 return mCapacity; 2357 } 2358 getAvailable()2359 public long getAvailable() { 2360 return mAvailable; 2361 } 2362 }; 2363 2364 /** 2365 * @hide 2366 * Get a list of globally available codec resources. 2367 * 2368 * To be able to enforce and test the implementation of codec availability hal APIs, 2369 * it is exposed only as TestApi. 2370 * This will be tracked and verified through cts. 2371 * 2372 * This returns a {@link java.util.List} list of codec resources. 2373 * For every {@link GlobalResourceInfo} in the list, it encapsulates the 2374 * information about each resources available globaly on device. 2375 * 2376 * @return A list of available device codec resources; an empty list if no 2377 * device codec resources are available. 2378 * @throws UnsupportedOperationException if not implemented. 2379 */ 2380 @FlaggedApi(FLAG_CODEC_AVAILABILITY) 2381 @TestApi getGloballyAvailableResources()2382 public static @NonNull List<GlobalResourceInfo> getGloballyAvailableResources() { 2383 return native_getGloballyAvailableResources(); 2384 } 2385 2386 @NonNull native_getGloballyAvailableResources()2387 private static native List<GlobalResourceInfo> native_getGloballyAvailableResources(); 2388 2389 /** 2390 * Configures a component. 2391 * 2392 * @param format The format of the input data (decoder) or the desired 2393 * format of the output data (encoder). Passing {@code null} 2394 * as {@code format} is equivalent to passing an 2395 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2396 * @param surface Specify a surface on which to render the output of this 2397 * decoder. Pass {@code null} as {@code surface} if the 2398 * codec does not generate raw video output (e.g. not a video 2399 * decoder) and/or if you want to configure the codec for 2400 * {@link ByteBuffer} output. 2401 * @param crypto Specify a crypto object to facilitate secure decryption 2402 * of the media data. Pass {@code null} as {@code crypto} for 2403 * non-secure codecs. 2404 * Please note that {@link MediaCodec} does NOT take ownership 2405 * of the {@link MediaCrypto} object; it is the application's 2406 * responsibility to properly cleanup the {@link MediaCrypto} object 2407 * when not in use. 2408 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2409 * component as an encoder. 2410 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2411 * or the format is unacceptable (e.g. missing a mandatory key), 2412 * or the flags are not set properly 2413 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2414 * @throws IllegalStateException if not in the Uninitialized state. 2415 * @throws CryptoException upon DRM error. 2416 * @throws CodecException upon codec error. 2417 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)2418 public void configure( 2419 @Nullable MediaFormat format, 2420 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2421 @ConfigureFlag int flags) { 2422 configure(format, surface, crypto, null, flags); 2423 } 2424 2425 /** 2426 * Configure a component to be used with a descrambler. 2427 * @param format The format of the input data (decoder) or the desired 2428 * format of the output data (encoder). Passing {@code null} 2429 * as {@code format} is equivalent to passing an 2430 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2431 * @param surface Specify a surface on which to render the output of this 2432 * decoder. Pass {@code null} as {@code surface} if the 2433 * codec does not generate raw video output (e.g. not a video 2434 * decoder) and/or if you want to configure the codec for 2435 * {@link ByteBuffer} output. 2436 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2437 * component as an encoder. 2438 * @param descrambler Specify a descrambler object to facilitate secure 2439 * descrambling of the media data, or null for non-secure codecs. 2440 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2441 * or the format is unacceptable (e.g. missing a mandatory key), 2442 * or the flags are not set properly 2443 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2444 * @throws IllegalStateException if not in the Uninitialized state. 2445 * @throws CryptoException upon DRM error. 2446 * @throws CodecException upon codec error. 2447 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)2448 public void configure( 2449 @Nullable MediaFormat format, @Nullable Surface surface, 2450 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 2451 configure(format, surface, null, 2452 descrambler != null ? descrambler.getBinder() : null, flags); 2453 } 2454 2455 private static final int BUFFER_MODE_INVALID = -1; 2456 private static final int BUFFER_MODE_LEGACY = 0; 2457 private static final int BUFFER_MODE_BLOCK = 1; 2458 private int mBufferMode = BUFFER_MODE_INVALID; 2459 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2460 private void configure( 2461 @Nullable MediaFormat format, @Nullable Surface surface, 2462 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 2463 @ConfigureFlag int flags) { 2464 if (crypto != null && descramblerBinder != null) { 2465 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 2466 } 2467 2468 // at the moment no codecs support detachable surface 2469 boolean canDetach = GetFlag(() -> android.media.codec.Flags.nullOutputSurfaceSupport()); 2470 if (GetFlag(() -> android.media.codec.Flags.nullOutputSurface())) { 2471 // Detached surface flag is only meaningful if surface is null. Otherwise, it is 2472 // ignored. 2473 if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0 && !canDetach) { 2474 throw new IllegalArgumentException("Codec does not support detached surface"); 2475 } 2476 } else { 2477 // don't allow detaching if API is disabled 2478 canDetach = false; 2479 } 2480 2481 String[] keys = null; 2482 Object[] values = null; 2483 2484 if (format != null) { 2485 Map<String, Object> formatMap = format.getMap(); 2486 keys = new String[formatMap.size()]; 2487 values = new Object[formatMap.size()]; 2488 2489 int i = 0; 2490 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2491 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2492 int sessionId = 0; 2493 try { 2494 sessionId = (Integer)entry.getValue(); 2495 } 2496 catch (Exception e) { 2497 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2498 } 2499 keys[i] = "audio-hw-sync"; 2500 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2501 } else if (applyPictureProfiles() && mediaQualityFw() 2502 && entry.getKey().equals(MediaFormat.KEY_PICTURE_PROFILE_INSTANCE)) { 2503 PictureProfile pictureProfile = null; 2504 try { 2505 pictureProfile = (PictureProfile) entry.getValue(); 2506 } catch (ClassCastException e) { 2507 throw new IllegalArgumentException( 2508 "Cannot cast the instance parameter to PictureProfile!"); 2509 } catch (Exception e) { 2510 Log.e(TAG, Log.getStackTraceString(e)); 2511 throw new IllegalArgumentException("Unexpected exception when casting the " 2512 + "instance parameter to PictureProfile!"); 2513 } 2514 if (pictureProfile == null) { 2515 throw new IllegalArgumentException( 2516 "Picture profile instance parameter is null!"); 2517 } 2518 PictureProfileHandle handle = pictureProfile.getHandle(); 2519 if (handle != PictureProfileHandle.NONE) { 2520 keys[i] = PARAMETER_KEY_PICTURE_PROFILE_HANDLE; 2521 values[i] = Long.valueOf(handle.getId()); 2522 } 2523 } else if (applyPictureProfiles() && mediaQualityFw() 2524 && entry.getKey().equals(MediaFormat.KEY_PICTURE_PROFILE_ID)) { 2525 String pictureProfileId = null; 2526 try { 2527 pictureProfileId = (String) entry.getValue(); 2528 } catch (ClassCastException e) { 2529 throw new IllegalArgumentException( 2530 "Cannot cast the KEY_PICTURE_PROFILE_ID parameter to String!"); 2531 } catch (Exception e) { 2532 Log.e(TAG, Log.getStackTraceString(e)); 2533 throw new IllegalArgumentException("Unexpected exception when casting the " 2534 + "KEY_PICTURE_PROFILE_ID parameter!"); 2535 } 2536 if (pictureProfileId == null) { 2537 throw new IllegalArgumentException( 2538 "KEY_PICTURE_PROFILE_ID parameter is null!"); 2539 } 2540 if (!pictureProfileId.isEmpty()) { 2541 keys[i] = MediaFormat.KEY_PICTURE_PROFILE_ID; 2542 values[i] = pictureProfileId; 2543 } 2544 } else { 2545 keys[i] = entry.getKey(); 2546 values[i] = entry.getValue(); 2547 } 2548 ++i; 2549 } 2550 } 2551 2552 mHasSurface = surface != null; 2553 mCrypto = crypto; 2554 synchronized (mBufferLock) { 2555 if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { 2556 mBufferMode = BUFFER_MODE_BLOCK; 2557 } else { 2558 mBufferMode = BUFFER_MODE_LEGACY; 2559 } 2560 } 2561 2562 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2563 2564 if (canDetach) { 2565 // If we were able to configure native codec with a detached surface 2566 // we now know that we have a surface. 2567 if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0) { 2568 mHasSurface = true; 2569 } 2570 } 2571 } 2572 2573 /** 2574 * @hide 2575 * Abstraction for the resources associated with a codec instance. 2576 * This encapsulates the required codec resources for a configured codec instance. 2577 * 2578 * To be able to enforce and test the implementation of codec availability hal APIs, 2579 * required codec resources are exposed only as TestApi. 2580 * This will be tracked and verified through cts. 2581 */ 2582 @FlaggedApi(FLAG_CODEC_AVAILABILITY) 2583 @TestApi 2584 public static final class InstanceResourceInfo { 2585 /** 2586 * Identifier for the Resource type. 2587 */ 2588 String mName; 2589 /** 2590 * Required resource count of this type. 2591 */ 2592 long mStaticCount; 2593 /** 2594 * Per frame resource requirement of this resource type. 2595 */ 2596 long mPerFrameCount; 2597 2598 @NonNull getName()2599 public String getName() { 2600 return mName; 2601 } 2602 getStaticCount()2603 public long getStaticCount() { 2604 return mStaticCount; 2605 } 2606 getPerFrameCount()2607 public long getPerFrameCount() { 2608 return mPerFrameCount; 2609 } 2610 }; 2611 2612 /** 2613 * @hide 2614 * Get a list of required codec resources for this configuration. 2615 * 2616 * To be able to enforce and test the implementation of codec availability hal APIs, 2617 * it is exposed only as TestApi. 2618 * This will be tracked and verified through cts. 2619 * 2620 * This returns a {@link java.util.List} list of codec resources. 2621 * For every {@link GlobalResourceInfo} in the list, it encapsulates the 2622 * information about each resources required for the current configuration. 2623 * 2624 * NOTE: This may only be called after {@link #configure}. 2625 * 2626 * @return A list of required device codec resources; an empty list if no 2627 * device codec resources are required. 2628 * @throws IllegalStateException if the codec wasn't configured yet. 2629 * @throws UnsupportedOperationException if not implemented. 2630 */ 2631 @FlaggedApi(FLAG_CODEC_AVAILABILITY) 2632 @TestApi getRequiredResources()2633 public @NonNull List<InstanceResourceInfo> getRequiredResources() { 2634 return native_getRequiredResources(); 2635 } 2636 2637 @NonNull native_getRequiredResources()2638 private native List<InstanceResourceInfo> native_getRequiredResources(); 2639 2640 /** 2641 * Dynamically sets the output surface of a codec. 2642 * <p> 2643 * This can only be used if the codec was configured with an output surface. The 2644 * new output surface should have a compatible usage type to the original output surface. 2645 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2646 * to ImageReader (software readable) output. 2647 * @param surface the output surface to use. It must not be {@code null}. 2648 * @throws IllegalStateException if the codec does not support setting the output 2649 * surface in the current state. 2650 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2651 */ setOutputSurface(@onNull Surface surface)2652 public void setOutputSurface(@NonNull Surface surface) { 2653 if (!mHasSurface) { 2654 throw new IllegalStateException("codec was not configured for an output surface"); 2655 } 2656 native_setSurface(surface); 2657 } 2658 native_setSurface(@onNull Surface surface)2659 private native void native_setSurface(@NonNull Surface surface); 2660 2661 /** 2662 * Detach the current output surface of a codec. 2663 * <p> 2664 * Detaches the currently associated output Surface from the 2665 * MediaCodec decoder. This allows the SurfaceView or other 2666 * component holding the Surface to be safely destroyed or 2667 * modified without affecting the decoder's operation. After 2668 * calling this method (and after it returns), the decoder will 2669 * enter detached-Surface mode and will no longer render 2670 * output. 2671 * 2672 * @throws IllegalStateException if the codec was not 2673 * configured in surface mode or if the codec does not support 2674 * detaching the output surface. 2675 * @see CONFIGURE_FLAG_DETACHED_SURFACE 2676 */ 2677 @FlaggedApi(FLAG_NULL_OUTPUT_SURFACE) detachOutputSurface()2678 public void detachOutputSurface() { 2679 if (!mHasSurface) { 2680 throw new IllegalStateException("codec was not configured for an output surface"); 2681 } 2682 2683 // note: we still have a surface in detached mode, so keep mHasSurface 2684 // we also technically allow calling detachOutputSurface multiple times in a row 2685 2686 if (GetFlag(() -> android.media.codec.Flags.nullOutputSurfaceSupport())) { 2687 native_detachOutputSurface(); 2688 } else { 2689 throw new IllegalStateException("codec does not support detaching output surface"); 2690 } 2691 } 2692 native_detachOutputSurface()2693 private native void native_detachOutputSurface(); 2694 2695 /** 2696 * Create a persistent input surface that can be used with codecs that normally have an input 2697 * surface, such as video encoders. A persistent input can be reused by subsequent 2698 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2699 * most one codec or recorder instance concurrently. 2700 * <p> 2701 * The application is responsible for calling release() on the Surface when done. 2702 * 2703 * @return an input surface that can be used with {@link #setInputSurface}. 2704 */ 2705 @NonNull createPersistentInputSurface()2706 public static Surface createPersistentInputSurface() { 2707 return native_createPersistentInputSurface(); 2708 } 2709 2710 static class PersistentSurface extends Surface { 2711 @SuppressWarnings("unused") PersistentSurface()2712 PersistentSurface() {} // used by native 2713 2714 @Override release()2715 public void release() { 2716 native_releasePersistentInputSurface(this); 2717 super.release(); 2718 } 2719 2720 private long mPersistentObject; 2721 }; 2722 2723 /** 2724 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2725 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2726 * lieu of {@link #createInputSurface}. 2727 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2728 * @throws IllegalStateException if not in the Configured state or does not require an input 2729 * surface. 2730 * @throws IllegalArgumentException if the surface was not created by 2731 * {@link #createPersistentInputSurface}. 2732 */ setInputSurface(@onNull Surface surface)2733 public void setInputSurface(@NonNull Surface surface) { 2734 if (!(surface instanceof PersistentSurface)) { 2735 throw new IllegalArgumentException("not a PersistentSurface"); 2736 } 2737 native_setInputSurface(surface); 2738 } 2739 2740 @NonNull native_createPersistentInputSurface()2741 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2742 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2743 private native final void native_setInputSurface(@NonNull Surface surface); 2744 native_setCallback(@ullable Callback cb)2745 private native final void native_setCallback(@Nullable Callback cb); 2746 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2747 private native final void native_configure( 2748 @Nullable String[] keys, @Nullable Object[] values, 2749 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2750 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2751 2752 /** 2753 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2754 * may only be called after {@link #configure} and before {@link #start}. 2755 * <p> 2756 * The application is responsible for calling release() on the Surface when 2757 * done. 2758 * <p> 2759 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2760 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2761 * unexpected results. 2762 * @throws IllegalStateException if not in the Configured state. 2763 */ 2764 @NonNull createInputSurface()2765 public native final Surface createInputSurface(); 2766 2767 /** 2768 * After successfully configuring the component, call {@code start}. 2769 * <p> 2770 * Call {@code start} also if the codec is configured in asynchronous mode, 2771 * and it has just been flushed, to resume requesting input buffers. 2772 * @throws IllegalStateException if not in the Configured state 2773 * or just after {@link #flush} for a codec that is configured 2774 * in asynchronous mode. 2775 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2776 * for start may be attributed to future method calls. 2777 */ start()2778 public final void start() { 2779 native_start(); 2780 } native_start()2781 private native final void native_start(); 2782 2783 /** 2784 * Finish the decode/encode session, note that the codec instance 2785 * remains active and ready to be {@link #start}ed again. 2786 * To ensure that it is available to other client call {@link #release} 2787 * and don't just rely on garbage collection to eventually do this for you. 2788 * @throws IllegalStateException if in the Released state. 2789 */ stop()2790 public final void stop() { 2791 native_stop(); 2792 freeAllTrackedBuffers(); 2793 2794 synchronized (mListenerLock) { 2795 if (mCallbackHandler != null) { 2796 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2797 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2798 } 2799 if (mOnFirstTunnelFrameReadyHandler != null) { 2800 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 2801 } 2802 if (mOnFrameRenderedHandler != null) { 2803 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2804 } 2805 } 2806 } 2807 native_stop()2808 private native final void native_stop(); 2809 2810 /** 2811 * Flush both input and output ports of the component. 2812 * <p> 2813 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2814 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2815 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2816 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2817 * invalid, and all buffers are owned by the codec. 2818 * <p> 2819 * If the codec is configured in asynchronous mode, call {@link #start} 2820 * after {@code flush} has returned to resume codec operations. The codec 2821 * will not request input buffers until this has happened. 2822 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2823 * callbacks that were not handled prior to calling {@code flush}. 2824 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2825 * should be discarded.</strong> 2826 * <p> 2827 * If the codec is configured in synchronous mode, codec will resume 2828 * automatically if it is configured with an input surface. Otherwise, it 2829 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2830 * 2831 * @throws IllegalStateException if not in the Executing state. 2832 * @throws MediaCodec.CodecException upon codec error. 2833 */ flush()2834 public final void flush() { 2835 synchronized(mBufferLock) { 2836 invalidateByteBuffersLocked(mCachedInputBuffers); 2837 invalidateByteBuffersLocked(mCachedOutputBuffers); 2838 mValidInputIndices.clear(); 2839 mValidOutputIndices.clear(); 2840 mDequeuedInputBuffers.clear(); 2841 mDequeuedOutputBuffers.clear(); 2842 } 2843 native_flush(); 2844 } 2845 native_flush()2846 private native final void native_flush(); 2847 2848 /** 2849 * Thrown when an internal codec error occurs. 2850 */ 2851 public final static class CodecException extends IllegalStateException { 2852 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2853 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2854 super(detailMessage); 2855 mErrorCode = errorCode; 2856 mActionCode = actionCode; 2857 2858 // TODO get this from codec 2859 final String sign = errorCode < 0 ? "neg_" : ""; 2860 mDiagnosticInfo = 2861 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2862 } 2863 2864 /** 2865 * Returns true if the codec exception is a transient issue, 2866 * perhaps due to resource constraints, and that the method 2867 * (or encoding/decoding) may be retried at a later time. 2868 */ 2869 public boolean isTransient() { 2870 return mActionCode == ACTION_TRANSIENT; 2871 } 2872 2873 /** 2874 * Returns true if the codec cannot proceed further, 2875 * but can be recovered by stopping, configuring, 2876 * and starting again. 2877 */ 2878 public boolean isRecoverable() { 2879 return mActionCode == ACTION_RECOVERABLE; 2880 } 2881 2882 /** 2883 * Retrieve the error code associated with a CodecException 2884 */ 2885 public int getErrorCode() { 2886 return mErrorCode; 2887 } 2888 2889 /** 2890 * Retrieve a developer-readable diagnostic information string 2891 * associated with the exception. Do not show this to end-users, 2892 * since this string will not be localized or generally 2893 * comprehensible to end-users. 2894 */ 2895 public @NonNull String getDiagnosticInfo() { 2896 return mDiagnosticInfo; 2897 } 2898 2899 /** 2900 * This indicates required resource was not able to be allocated. 2901 */ 2902 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2903 2904 /** 2905 * This indicates the resource manager reclaimed the media resource used by the codec. 2906 * <p> 2907 * With this exception, the codec must be released, as it has moved to terminal state. 2908 */ 2909 public static final int ERROR_RECLAIMED = 1101; 2910 2911 /** @hide */ 2912 @IntDef({ 2913 ERROR_INSUFFICIENT_RESOURCE, 2914 ERROR_RECLAIMED, 2915 }) 2916 @Retention(RetentionPolicy.SOURCE) 2917 public @interface ReasonCode {} 2918 2919 /* Must be in sync with android_media_MediaCodec.cpp */ 2920 private final static int ACTION_TRANSIENT = 1; 2921 private final static int ACTION_RECOVERABLE = 2; 2922 2923 private final String mDiagnosticInfo; 2924 private final int mErrorCode; 2925 private final int mActionCode; 2926 } 2927 2928 /** 2929 * Thrown when a crypto error occurs while queueing a secure input buffer. 2930 */ 2931 public final static class CryptoException extends RuntimeException 2932 implements MediaDrmThrowable { 2933 public CryptoException(int errorCode, @Nullable String detailMessage) { 2934 this(detailMessage, errorCode, 0, 0, 0, null); 2935 } 2936 2937 /** 2938 * @hide 2939 */ 2940 public CryptoException(String message, int errorCode, int vendorError, int oemError, 2941 int errorContext, @Nullable CryptoInfo cryptoInfo) { 2942 super(message); 2943 mErrorCode = errorCode; 2944 mVendorError = vendorError; 2945 mOemError = oemError; 2946 mErrorContext = errorContext; 2947 mCryptoInfo = cryptoInfo; 2948 } 2949 2950 /** 2951 * This indicates that the requested key was not found when trying to 2952 * perform a decrypt operation. The operation can be retried after adding 2953 * the correct decryption key. 2954 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_NO_KEY}. 2955 */ 2956 public static final int ERROR_NO_KEY = MediaDrm.ErrorCodes.ERROR_NO_KEY; 2957 2958 /** 2959 * This indicates that the key used for decryption is no longer 2960 * valid due to license term expiration. The operation can be retried 2961 * after updating the expired keys. 2962 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_KEY_EXPIRED}. 2963 */ 2964 public static final int ERROR_KEY_EXPIRED = MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED; 2965 2966 /** 2967 * This indicates that a required crypto resource was not able to be 2968 * allocated while attempting the requested operation. The operation 2969 * can be retried if the app is able to release resources. 2970 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_RESOURCE_BUSY} 2971 */ 2972 public static final int ERROR_RESOURCE_BUSY = MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY; 2973 2974 /** 2975 * This indicates that the output protection levels supported by the 2976 * device are not sufficient to meet the requirements set by the 2977 * content owner in the license policy. 2978 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_OUTPUT_PROTECTION} 2979 */ 2980 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 2981 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION; 2982 2983 /** 2984 * This indicates that decryption was attempted on a session that is 2985 * not opened, which could be due to a failure to open the session, 2986 * closing the session prematurely, or the session being reclaimed 2987 * by the resource manager. 2988 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_SESSION_NOT_OPENED} 2989 */ 2990 public static final int ERROR_SESSION_NOT_OPENED = 2991 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED; 2992 2993 /** 2994 * This indicates that an operation was attempted that could not be 2995 * supported by the crypto system of the device in its current 2996 * configuration. It may occur when the license policy requires 2997 * device security features that aren't supported by the device, 2998 * or due to an internal error in the crypto system that prevents 2999 * the specified security policy from being met. 3000 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_UNSUPPORTED_OPERATION} 3001 */ 3002 public static final int ERROR_UNSUPPORTED_OPERATION = 3003 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION; 3004 3005 /** 3006 * This indicates that the security level of the device is not 3007 * sufficient to meet the requirements set by the content owner 3008 * in the license policy. 3009 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_SECURITY} 3010 */ 3011 public static final int ERROR_INSUFFICIENT_SECURITY = 3012 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY; 3013 3014 /** 3015 * This indicates that the video frame being decrypted exceeds 3016 * the size of the device's protected output buffers. When 3017 * encountering this error the app should try playing content 3018 * of a lower resolution. 3019 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_FRAME_TOO_LARGE} 3020 */ 3021 public static final int ERROR_FRAME_TOO_LARGE = MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE; 3022 3023 /** 3024 * This error indicates that session state has been 3025 * invalidated. It can occur on devices that are not capable 3026 * of retaining crypto session state across device 3027 * suspend/resume. The session must be closed and a new 3028 * session opened to resume operation. 3029 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_LOST_STATE} 3030 */ 3031 public static final int ERROR_LOST_STATE = MediaDrm.ErrorCodes.ERROR_LOST_STATE; 3032 3033 /** @hide */ 3034 @IntDef({ 3035 MediaDrm.ErrorCodes.ERROR_NO_KEY, 3036 MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED, 3037 MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY, 3038 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 3039 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED, 3040 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION, 3041 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY, 3042 MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE, 3043 MediaDrm.ErrorCodes.ERROR_LOST_STATE, 3044 MediaDrm.ErrorCodes.ERROR_GENERIC_OEM, 3045 MediaDrm.ErrorCodes.ERROR_GENERIC_PLUGIN, 3046 MediaDrm.ErrorCodes.ERROR_LICENSE_PARSE, 3047 MediaDrm.ErrorCodes.ERROR_MEDIA_FRAMEWORK, 3048 MediaDrm.ErrorCodes.ERROR_ZERO_SUBSAMPLES 3049 }) 3050 @Retention(RetentionPolicy.SOURCE) 3051 public @interface CryptoErrorCode {} 3052 3053 /** 3054 * Returns error code associated with this {@link CryptoException}. 3055 * <p> 3056 * Please refer to {@link MediaDrm.ErrorCodes} for the general error 3057 * handling strategy and details about each possible return value. 3058 * 3059 * @return an error code defined in {@link MediaDrm.ErrorCodes}. 3060 */ 3061 @CryptoErrorCode 3062 public int getErrorCode() { 3063 return mErrorCode; 3064 } 3065 3066 /** 3067 * Returns CryptoInfo associated with this {@link CryptoException} 3068 * if any 3069 * 3070 * @return CryptoInfo object if any. {@link MediaCodec.CryptoException} 3071 */ 3072 public @Nullable CryptoInfo getCryptoInfo() { 3073 return mCryptoInfo; 3074 } 3075 3076 @Override 3077 public int getVendorError() { 3078 return mVendorError; 3079 } 3080 3081 @Override 3082 public int getOemError() { 3083 return mOemError; 3084 } 3085 3086 @Override 3087 public int getErrorContext() { 3088 return mErrorContext; 3089 } 3090 3091 private final int mErrorCode, mVendorError, mOemError, mErrorContext; 3092 private CryptoInfo mCryptoInfo; 3093 } 3094 3095 /** 3096 * After filling a range of the input buffer at the specified index 3097 * submit it to the component. Once an input buffer is queued to 3098 * the codec, it MUST NOT be used until it is later retrieved by 3099 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 3100 * return value or a {@link Callback#onInputBufferAvailable} 3101 * callback. 3102 * <p> 3103 * Many decoders require the actual compressed data stream to be 3104 * preceded by "codec specific data", i.e. setup data used to initialize 3105 * the codec such as PPS/SPS in the case of AVC video or code tables 3106 * in the case of vorbis audio. 3107 * The class {@link android.media.MediaExtractor} provides codec 3108 * specific data as part of 3109 * the returned track format in entries named "csd-0", "csd-1" ... 3110 * <p> 3111 * These buffers can be submitted directly after {@link #start} or 3112 * {@link #flush} by specifying the flag {@link 3113 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 3114 * codec with a {@link MediaFormat} containing these keys, they 3115 * will be automatically submitted by MediaCodec directly after 3116 * start. Therefore, the use of {@link 3117 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 3118 * recommended only for advanced users. 3119 * <p> 3120 * To indicate that this is the final piece of input data (or rather that 3121 * no more input data follows unless the decoder is subsequently flushed) 3122 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 3123 * <p class=note> 3124 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 3125 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 3126 * Surface output buffers, and the resulting frame timestamp was undefined. 3127 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 3128 * Similarly, since frame timestamps can be used by the destination surface for rendering 3129 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 3130 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 3131 * SurfaceView specifics}).</strong> 3132 * 3133 * @param index The index of a client-owned input buffer previously returned 3134 * in a call to {@link #dequeueInputBuffer}. 3135 * @param offset The byte offset into the input buffer at which the data starts. 3136 * @param size The number of bytes of valid input data. 3137 * @param presentationTimeUs The presentation timestamp in microseconds for this 3138 * buffer. This is normally the media time at which this 3139 * buffer should be presented (rendered). When using an output 3140 * surface, this will be propagated as the {@link 3141 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3142 * conversion to nanoseconds). 3143 * @param flags A bitmask of flags 3144 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3145 * While not prohibited, most codecs do not use the 3146 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3147 * @throws IllegalStateException if not in the Executing state. 3148 * @throws MediaCodec.CodecException upon codec error. 3149 * @throws CryptoException if a crypto object has been specified in 3150 * {@link #configure} 3151 */ 3152 public final void queueInputBuffer( 3153 int index, 3154 int offset, int size, long presentationTimeUs, int flags) 3155 throws CryptoException { 3156 Trace.traceBegin(Trace.TRACE_TAG_VIDEO, "MediaCodec::queueInputBuffer#java"); 3157 if ((flags & BUFFER_FLAG_DECODE_ONLY) != 0 3158 && (flags & BUFFER_FLAG_END_OF_STREAM) != 0) { 3159 throw new InvalidBufferFlagsException(EOS_AND_DECODE_ONLY_ERROR_MESSAGE); 3160 } 3161 synchronized(mBufferLock) { 3162 if (mBufferMode == BUFFER_MODE_BLOCK) { 3163 throw new IncompatibleWithBlockModelException("queueInputBuffer() " 3164 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3165 + "Please use getQueueRequest() to queue buffers"); 3166 } 3167 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3168 mDequeuedInputBuffers.remove(index); 3169 } 3170 try { 3171 native_queueInputBuffer( 3172 index, offset, size, presentationTimeUs, flags); 3173 } catch (CryptoException | IllegalStateException e) { 3174 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3175 throw e; 3176 } finally { 3177 Trace.traceEnd(Trace.TRACE_TAG_VIDEO); 3178 } 3179 } 3180 3181 /** 3182 * Submit multiple access units to the codec along with multiple 3183 * {@link MediaCodec.BufferInfo} describing the contents of the buffer. This method 3184 * is supported only in asynchronous mode. While this method can be used for all codecs, 3185 * it is meant for buffer batching, which is only supported by codecs that advertise 3186 * FEATURE_MultipleFrames. Other codecs will not output large output buffers via 3187 * onOutputBuffersAvailable, and instead will output single-access-unit output via 3188 * onOutputBufferAvailable. 3189 * <p> 3190 * Output buffer size can be configured using the following MediaFormat keys. 3191 * {@link MediaFormat#KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE} and 3192 * {@link MediaFormat#KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE}. 3193 * Details for each access unit present in the buffer should be described using 3194 * {@link MediaCodec.BufferInfo}. Access units must be laid out contiguously (without any gaps) 3195 * and in order. Multiple access units in the output if present, will be available in 3196 * {@link Callback#onOutputBuffersAvailable} or {@link Callback#onOutputBufferAvailable} 3197 * in case of single-access-unit output or when output does not contain any buffers, 3198 * such as flags. 3199 * <p> 3200 * All other details for populating {@link MediaCodec.BufferInfo} is the same as described in 3201 * {@link #queueInputBuffer}. 3202 * 3203 * @param index The index of a client-owned input buffer previously returned 3204 * in a call to {@link #dequeueInputBuffer}. 3205 * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the 3206 * contents in the buffer. The ArrayDeque and the BufferInfo objects provided 3207 * can be recycled by the caller for re-use. 3208 * @throws IllegalStateException if not in the Executing state or not in asynchronous mode. 3209 * @throws MediaCodec.CodecException upon codec error. 3210 * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the 3211 * access units are not contiguous. 3212 * @throws CryptoException if a crypto object has been specified in 3213 * {@link #configure} 3214 */ 3215 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) 3216 public final void queueInputBuffers( 3217 int index, 3218 @NonNull ArrayDeque<BufferInfo> bufferInfos) { 3219 Trace.traceBegin(Trace.TRACE_TAG_VIDEO, "MediaCodec::queueInputBuffers#java"); 3220 synchronized(mBufferLock) { 3221 if (mBufferMode == BUFFER_MODE_BLOCK) { 3222 throw new IncompatibleWithBlockModelException("queueInputBuffers() " 3223 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3224 + "Please use getQueueRequest() to queue buffers"); 3225 } 3226 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3227 mDequeuedInputBuffers.remove(index); 3228 } 3229 try { 3230 native_queueInputBuffers( 3231 index, bufferInfos.toArray()); 3232 } catch (CryptoException | IllegalStateException | IllegalArgumentException e) { 3233 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3234 throw e; 3235 } finally { 3236 Trace.traceEnd(Trace.TRACE_TAG_VIDEO); 3237 } 3238 } 3239 3240 private native final void native_queueInputBuffer( 3241 int index, 3242 int offset, int size, long presentationTimeUs, int flags) 3243 throws CryptoException; 3244 3245 private native final void native_queueInputBuffers( 3246 int index, 3247 @NonNull Object[] infos) 3248 throws CryptoException, CodecException; 3249 3250 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 3251 public static final int CRYPTO_MODE_AES_CTR = 1; 3252 public static final int CRYPTO_MODE_AES_CBC = 2; 3253 3254 /** 3255 * Metadata describing the structure of an encrypted input sample. 3256 * <p> 3257 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 3258 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 3259 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 3260 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 3261 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 3262 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 3263 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 3264 * <p> 3265 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 3266 * "Common encryption in ISO base media file format files". 3267 * <p> 3268 * <h3>ISO-CENC Schemes</h3> 3269 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 3270 * corresponding to each possible combination of an AES mode with the presence or absence of 3271 * patterned encryption. 3272 * 3273 * <table style="width: 0%"> 3274 * <thead> 3275 * <tr> 3276 * <th> </th> 3277 * <th>AES-CTR</th> 3278 * <th>AES-CBC</th> 3279 * </tr> 3280 * </thead> 3281 * <tbody> 3282 * <tr> 3283 * <th>Without Patterns</th> 3284 * <td>cenc</td> 3285 * <td>cbc1</td> 3286 * </tr><tr> 3287 * <th>With Patterns</th> 3288 * <td>cens</td> 3289 * <td>cbcs</td> 3290 * </tr> 3291 * </tbody> 3292 * </table> 3293 * 3294 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 3295 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 3296 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 3297 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 3298 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 3299 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 3300 * {@link #setPattern} is never called is all zeroes. 3301 * <p> 3302 * <h4>HLS SAMPLE-AES Audio</h4> 3303 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 3304 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 3305 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 3306 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 3307 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 3308 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 3309 * while still decrypting every block. 3310 */ 3311 public final static class CryptoInfo { 3312 /** 3313 * The number of subSamples that make up the buffer's contents. 3314 */ 3315 public int numSubSamples; 3316 /** 3317 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 3318 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 3319 */ 3320 public int[] numBytesOfClearData; 3321 /** 3322 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 3323 * as clear and {@link #numBytesOfClearData} must be specified. 3324 */ 3325 public int[] numBytesOfEncryptedData; 3326 /** 3327 * A 16-byte key id 3328 */ 3329 public byte[] key; 3330 /** 3331 * A 16-byte initialization vector 3332 */ 3333 public byte[] iv; 3334 /** 3335 * The type of encryption that has been applied, 3336 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 3337 * and {@link #CRYPTO_MODE_AES_CBC} 3338 */ 3339 public int mode; 3340 3341 /** 3342 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 3343 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 3344 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 3345 */ 3346 public final static class Pattern { 3347 /** 3348 * Number of blocks to be encrypted in the pattern. If both this and 3349 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 3350 */ 3351 private int mEncryptBlocks; 3352 3353 /** 3354 * Number of blocks to be skipped (left clear) in the pattern. If both this and 3355 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 3356 */ 3357 private int mSkipBlocks; 3358 3359 /** 3360 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 3361 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 3362 */ 3363 public Pattern(int blocksToEncrypt, int blocksToSkip) { 3364 set(blocksToEncrypt, blocksToSkip); 3365 } 3366 3367 /** 3368 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 3369 * parameters are zero, pattern encryption is inoperative. 3370 */ 3371 public void set(int blocksToEncrypt, int blocksToSkip) { 3372 mEncryptBlocks = blocksToEncrypt; 3373 mSkipBlocks = blocksToSkip; 3374 } 3375 3376 /** 3377 * Return the number of blocks to skip in a sample encryption pattern. 3378 */ 3379 public int getSkipBlocks() { 3380 return mSkipBlocks; 3381 } 3382 3383 /** 3384 * Return the number of blocks to encrypt in a sample encryption pattern. 3385 */ 3386 public int getEncryptBlocks() { 3387 return mEncryptBlocks; 3388 } 3389 }; 3390 3391 private static final Pattern ZERO_PATTERN = new Pattern(0, 0); 3392 3393 /** 3394 * The pattern applicable to the protected data in each subsample. 3395 */ 3396 private Pattern mPattern = ZERO_PATTERN; 3397 3398 /** 3399 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 3400 * a {@link MediaCodec.CryptoInfo} instance. 3401 */ 3402 public void set( 3403 int newNumSubSamples, 3404 @NonNull int[] newNumBytesOfClearData, 3405 @NonNull int[] newNumBytesOfEncryptedData, 3406 @NonNull byte[] newKey, 3407 @NonNull byte[] newIV, 3408 int newMode) { 3409 numSubSamples = newNumSubSamples; 3410 numBytesOfClearData = newNumBytesOfClearData; 3411 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 3412 key = newKey; 3413 iv = newIV; 3414 mode = newMode; 3415 mPattern = ZERO_PATTERN; 3416 } 3417 3418 /** 3419 * Returns the {@link Pattern encryption pattern}. 3420 */ 3421 public @NonNull Pattern getPattern() { 3422 return new Pattern(mPattern.getEncryptBlocks(), mPattern.getSkipBlocks()); 3423 } 3424 3425 /** 3426 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 3427 * See {@link Pattern}. 3428 */ 3429 public void setPattern(Pattern newPattern) { 3430 if (newPattern == null) { 3431 newPattern = ZERO_PATTERN; 3432 } 3433 setPattern(newPattern.getEncryptBlocks(), newPattern.getSkipBlocks()); 3434 } 3435 3436 // Accessed from android_media_MediaExtractor.cpp. 3437 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 3438 mPattern = new Pattern(blocksToEncrypt, blocksToSkip); 3439 } 3440 3441 @Override 3442 public String toString() { 3443 StringBuilder builder = new StringBuilder(); 3444 builder.append(numSubSamples + " subsamples, key ["); 3445 String hexdigits = "0123456789abcdef"; 3446 for (int i = 0; i < key.length; i++) { 3447 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 3448 builder.append(hexdigits.charAt(key[i] & 0x0f)); 3449 } 3450 builder.append("], iv ["); 3451 for (int i = 0; i < iv.length; i++) { 3452 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 3453 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 3454 } 3455 builder.append("], clear "); Arrays.toString(numBytesOfClearData)3456 builder.append(Arrays.toString(numBytesOfClearData)); 3457 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)3458 builder.append(Arrays.toString(numBytesOfEncryptedData)); 3459 builder.append(", pattern (encrypt: "); builder.append(mPattern.mEncryptBlocks)3460 builder.append(mPattern.mEncryptBlocks); 3461 builder.append(", skip: "); builder.append(mPattern.mSkipBlocks)3462 builder.append(mPattern.mSkipBlocks); 3463 builder.append(")"); 3464 return builder.toString(); 3465 } 3466 }; 3467 3468 /** 3469 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 3470 * potentially encrypted. 3471 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 3472 * 3473 * @param index The index of a client-owned input buffer previously returned 3474 * in a call to {@link #dequeueInputBuffer}. 3475 * @param offset The byte offset into the input buffer at which the data starts. 3476 * @param info Metadata required to facilitate decryption, the object can be 3477 * reused immediately after this call returns. 3478 * @param presentationTimeUs The presentation timestamp in microseconds for this 3479 * buffer. This is normally the media time at which this 3480 * buffer should be presented (rendered). 3481 * @param flags A bitmask of flags 3482 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3483 * While not prohibited, most codecs do not use the 3484 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3485 * @throws IllegalStateException if not in the Executing state. 3486 * @throws MediaCodec.CodecException upon codec error. 3487 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 3488 * An error code associated with the exception helps identify the 3489 * reason for the failure. 3490 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)3491 public final void queueSecureInputBuffer( 3492 int index, 3493 int offset, 3494 @NonNull CryptoInfo info, 3495 long presentationTimeUs, 3496 int flags) throws CryptoException { 3497 Trace.traceBegin(Trace.TRACE_TAG_VIDEO, "MediaCodec::queueSecureInputBuffer#java"); 3498 if ((flags & BUFFER_FLAG_DECODE_ONLY) != 0 3499 && (flags & BUFFER_FLAG_END_OF_STREAM) != 0) { 3500 throw new InvalidBufferFlagsException(EOS_AND_DECODE_ONLY_ERROR_MESSAGE); 3501 } 3502 synchronized(mBufferLock) { 3503 if (mBufferMode == BUFFER_MODE_BLOCK) { 3504 throw new IncompatibleWithBlockModelException("queueSecureInputBuffer() " 3505 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3506 + "Please use getQueueRequest() to queue buffers"); 3507 } 3508 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3509 mDequeuedInputBuffers.remove(index); 3510 } 3511 try { 3512 native_queueSecureInputBuffer( 3513 index, offset, info, presentationTimeUs, flags); 3514 } catch (CryptoException | IllegalStateException e) { 3515 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3516 throw e; 3517 } finally { 3518 Trace.traceEnd(Trace.TRACE_TAG_VIDEO); 3519 } 3520 } 3521 3522 /** 3523 * Similar to {@link #queueInputBuffers queueInputBuffers} but submits multiple access units 3524 * in a buffer that is potentially encrypted. 3525 * <strong>Check out further notes at {@link #queueInputBuffers queueInputBuffers}.</strong> 3526 * 3527 * @param index The index of a client-owned input buffer previously returned 3528 * in a call to {@link #dequeueInputBuffer}. 3529 * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the 3530 * contents in the buffer. The ArrayDeque and the BufferInfo objects provided 3531 * can be recycled by the caller for re-use. 3532 * @param cryptoInfos ArrayDeque of {@link MediaCodec.CryptoInfo} objects to facilitate the 3533 * decryption of the contents. The ArrayDeque and the CryptoInfo objects 3534 * provided can be reused immediately after the call returns. These objects 3535 * should correspond to bufferInfo objects to ensure correct decryption. 3536 * @throws IllegalStateException if not in the Executing state or not in asynchronous mode. 3537 * @throws MediaCodec.CodecException upon codec error. 3538 * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the 3539 * access units are not contiguous. 3540 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 3541 * An error code associated with the exception helps identify the 3542 * reason for the failure. 3543 */ 3544 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) queueSecureInputBuffers( int index, @NonNull ArrayDeque<BufferInfo> bufferInfos, @NonNull ArrayDeque<CryptoInfo> cryptoInfos)3545 public final void queueSecureInputBuffers( 3546 int index, 3547 @NonNull ArrayDeque<BufferInfo> bufferInfos, 3548 @NonNull ArrayDeque<CryptoInfo> cryptoInfos) { 3549 Trace.traceBegin(Trace.TRACE_TAG_VIDEO, "MediaCodec::queueSecureInputBuffers#java"); 3550 synchronized(mBufferLock) { 3551 if (mBufferMode == BUFFER_MODE_BLOCK) { 3552 throw new IncompatibleWithBlockModelException("queueSecureInputBuffers() " 3553 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3554 + "Please use getQueueRequest() to queue buffers"); 3555 } 3556 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3557 mDequeuedInputBuffers.remove(index); 3558 } 3559 try { 3560 native_queueSecureInputBuffers( 3561 index, bufferInfos.toArray(), cryptoInfos.toArray()); 3562 } catch (CryptoException | IllegalStateException | IllegalArgumentException e) { 3563 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3564 throw e; 3565 } finally { 3566 Trace.traceEnd(Trace.TRACE_TAG_VIDEO); 3567 } 3568 } 3569 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)3570 private native final void native_queueSecureInputBuffer( 3571 int index, 3572 int offset, 3573 @NonNull CryptoInfo info, 3574 long presentationTimeUs, 3575 int flags) throws CryptoException; 3576 native_queueSecureInputBuffers( int index, @NonNull Object[] bufferInfos, @NonNull Object[] cryptoInfos)3577 private native final void native_queueSecureInputBuffers( 3578 int index, 3579 @NonNull Object[] bufferInfos, 3580 @NonNull Object[] cryptoInfos) throws CryptoException, CodecException; 3581 3582 /** 3583 * Returns the index of an input buffer to be filled with valid data 3584 * or -1 if no such buffer is currently available. 3585 * This method will return immediately if timeoutUs == 0, wait indefinitely 3586 * for the availability of an input buffer if timeoutUs < 0 or wait up 3587 * to "timeoutUs" microseconds if timeoutUs > 0. 3588 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3589 * @throws IllegalStateException if not in the Executing state, 3590 * or codec is configured in asynchronous mode. 3591 * @throws MediaCodec.CodecException upon codec error. 3592 */ dequeueInputBuffer(long timeoutUs)3593 public final int dequeueInputBuffer(long timeoutUs) { 3594 Trace.traceBegin(Trace.TRACE_TAG_VIDEO, "MediaCodec::dequeueInputBuffer#java"); 3595 synchronized (mBufferLock) { 3596 if (mBufferMode == BUFFER_MODE_BLOCK) { 3597 throw new IncompatibleWithBlockModelException("dequeueInputBuffer() " 3598 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3599 + "Please use MediaCodec.Callback objectes to get input buffer slots."); 3600 } 3601 } 3602 int res = native_dequeueInputBuffer(timeoutUs); 3603 if (res >= 0) { 3604 synchronized(mBufferLock) { 3605 validateInputByteBufferLocked(mCachedInputBuffers, res); 3606 } 3607 } 3608 Trace.traceEnd(Trace.TRACE_TAG_VIDEO); 3609 return res; 3610 } 3611 native_dequeueInputBuffer(long timeoutUs)3612 private native final int native_dequeueInputBuffer(long timeoutUs); 3613 3614 /** 3615 * Section of memory that represents a linear block. Applications may 3616 * acquire a block via {@link LinearBlock#obtain} and queue all or part 3617 * of the block as an input buffer to a codec, or get a block allocated by 3618 * codec as an output buffer from {@link OutputFrame}. 3619 * 3620 * {@see QueueRequest#setLinearBlock} 3621 * {@see QueueRequest#setEncryptedLinearBlock} 3622 * {@see OutputFrame#getLinearBlock} 3623 */ 3624 public static final class LinearBlock { 3625 // No public constructors. LinearBlock()3626 private LinearBlock() {} 3627 3628 /** 3629 * Returns true if the buffer is mappable. 3630 * @throws IllegalStateException if invalid 3631 */ isMappable()3632 public boolean isMappable() { 3633 synchronized (mLock) { 3634 if (!mValid) { 3635 throw new IllegalStateException("The linear block is invalid"); 3636 } 3637 return mMappable; 3638 } 3639 } 3640 3641 /** 3642 * Map the memory and return the mapped region. 3643 * <p> 3644 * The returned memory region becomes inaccessible after 3645 * {@link #recycle}, or the buffer is queued to the codecs and not 3646 * returned to the client yet. 3647 * 3648 * @return mapped memory region as {@link ByteBuffer} object 3649 * @throws IllegalStateException if not mappable or invalid 3650 */ map()3651 public @NonNull ByteBuffer map() { 3652 synchronized (mLock) { 3653 if (!mValid) { 3654 throw new IllegalStateException("The linear block is invalid"); 3655 } 3656 if (!mMappable) { 3657 throw new IllegalStateException("The linear block is not mappable"); 3658 } 3659 if (mMapped == null) { 3660 mMapped = native_map(); 3661 } 3662 return mMapped; 3663 } 3664 } 3665 native_map()3666 private native ByteBuffer native_map(); 3667 3668 /** 3669 * Mark this block as ready to be recycled by the framework once it is 3670 * no longer in use. All operations to this object after 3671 * this call will cause exceptions, as well as attempt to access the 3672 * previously mapped memory region. Caller should clear all references 3673 * to this object after this call. 3674 * <p> 3675 * To avoid excessive memory consumption, it is recommended that callers 3676 * recycle buffers as soon as they no longer need the buffers 3677 * 3678 * @throws IllegalStateException if invalid 3679 */ recycle()3680 public void recycle() { 3681 synchronized (mLock) { 3682 if (!mValid) { 3683 throw new IllegalStateException("The linear block is invalid"); 3684 } 3685 if (mMapped != null) { 3686 mMapped.setAccessible(false); 3687 mMapped = null; 3688 } 3689 native_recycle(); 3690 mValid = false; 3691 mNativeContext = 0; 3692 } 3693 3694 if (!mInternal) { 3695 sPool.offer(this); 3696 } 3697 } 3698 native_recycle()3699 private native void native_recycle(); 3700 native_obtain(int capacity, String[] codecNames)3701 private native void native_obtain(int capacity, String[] codecNames); 3702 3703 @Override finalize()3704 protected void finalize() { 3705 native_recycle(); 3706 } 3707 3708 /** 3709 * Returns true if it is possible to allocate a linear block that can be 3710 * passed to all listed codecs as input buffers without copying the 3711 * content. 3712 * <p> 3713 * Note that even if this function returns true, {@link #obtain} may 3714 * still throw due to invalid arguments or allocation failure. 3715 * 3716 * @param codecNames list of codecs that the client wants to use a 3717 * linear block without copying. Null entries are 3718 * ignored. 3719 */ isCodecCopyFreeCompatible(@onNull String[] codecNames)3720 public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { 3721 return native_checkCompatible(codecNames); 3722 } 3723 native_checkCompatible(@onNull String[] codecNames)3724 private static native boolean native_checkCompatible(@NonNull String[] codecNames); 3725 3726 /** 3727 * Obtain a linear block object no smaller than {@code capacity}. 3728 * If {@link #isCodecCopyFreeCompatible} with the same 3729 * {@code codecNames} returned true, the returned 3730 * {@link LinearBlock} object can be queued to the listed codecs without 3731 * copying. The returned {@link LinearBlock} object is always 3732 * read/write mappable. 3733 * 3734 * @param capacity requested capacity of the linear block in bytes 3735 * @param codecNames list of codecs that the client wants to use this 3736 * linear block without copying. Null entries are 3737 * ignored. 3738 * @return a linear block object. 3739 * @throws IllegalArgumentException if the capacity is invalid or 3740 * codecNames contains invalid name 3741 * @throws IOException if an error occurred while allocating a buffer 3742 */ obtain( int capacity, @NonNull String[] codecNames)3743 public static @Nullable LinearBlock obtain( 3744 int capacity, @NonNull String[] codecNames) { 3745 LinearBlock buffer = sPool.poll(); 3746 if (buffer == null) { 3747 buffer = new LinearBlock(); 3748 } 3749 synchronized (buffer.mLock) { 3750 buffer.native_obtain(capacity, codecNames); 3751 } 3752 return buffer; 3753 } 3754 3755 // Called from native setInternalStateLocked(long context, boolean isMappable)3756 private void setInternalStateLocked(long context, boolean isMappable) { 3757 mNativeContext = context; 3758 mMappable = isMappable; 3759 mValid = (context != 0); 3760 mInternal = true; 3761 } 3762 3763 private static final BlockingQueue<LinearBlock> sPool = 3764 new LinkedBlockingQueue<>(); 3765 3766 private final Object mLock = new Object(); 3767 private boolean mValid = false; 3768 private boolean mMappable = false; 3769 private ByteBuffer mMapped = null; 3770 private long mNativeContext = 0; 3771 private boolean mInternal = false; 3772 } 3773 3774 /** 3775 * Map a {@link HardwareBuffer} object into {@link Image}, so that the content of the buffer is 3776 * accessible. Depending on the usage and pixel format of the hardware buffer, it may not be 3777 * mappable; this method returns null in that case. 3778 * 3779 * @param hardwareBuffer {@link HardwareBuffer} to map. 3780 * @return Mapped {@link Image} object, or null if the buffer is not mappable. 3781 */ mapHardwareBuffer(@onNull HardwareBuffer hardwareBuffer)3782 public static @Nullable Image mapHardwareBuffer(@NonNull HardwareBuffer hardwareBuffer) { 3783 return native_mapHardwareBuffer(hardwareBuffer); 3784 } 3785 native_mapHardwareBuffer( @onNull HardwareBuffer hardwareBuffer)3786 private static native @Nullable Image native_mapHardwareBuffer( 3787 @NonNull HardwareBuffer hardwareBuffer); 3788 native_closeMediaImage(long context)3789 private static native void native_closeMediaImage(long context); 3790 3791 /** 3792 * Builder-like class for queue requests. Use this class to prepare a 3793 * queue request and send it. 3794 */ 3795 public final class QueueRequest { 3796 // No public constructor QueueRequest(@onNull MediaCodec codec, int index)3797 private QueueRequest(@NonNull MediaCodec codec, int index) { 3798 mCodec = codec; 3799 mIndex = index; 3800 } 3801 3802 /** 3803 * Set a linear block to this queue request. Exactly one buffer must be 3804 * set for a queue request before calling {@link #queue}. It is possible 3805 * to use the same {@link LinearBlock} object for multiple queue 3806 * requests. The behavior is undefined if the range of the buffer 3807 * overlaps for multiple requests, or the application writes into the 3808 * region being processed by the codec. 3809 * 3810 * @param block The linear block object 3811 * @param offset The byte offset into the input buffer at which the data starts. 3812 * @param size The number of bytes of valid input data. 3813 * @return this object 3814 * @throws IllegalStateException if a buffer is already set 3815 */ setLinearBlock( @onNull LinearBlock block, int offset, int size)3816 public @NonNull QueueRequest setLinearBlock( 3817 @NonNull LinearBlock block, 3818 int offset, 3819 int size) { 3820 if (!isAccessible()) { 3821 throw new IllegalStateException("The request is stale"); 3822 } 3823 if (mLinearBlock != null || mHardwareBuffer != null) { 3824 throw new IllegalStateException("Cannot set block twice"); 3825 } 3826 mLinearBlock = block; 3827 mOffset = offset; 3828 mSize = size; 3829 mCryptoInfos.clear(); 3830 return this; 3831 } 3832 3833 /** 3834 * Set a linear block that contain multiple non-encrypted access unit to this 3835 * queue request. Exactly one buffer must be set for a queue request before 3836 * calling {@link #queue}. Multiple access units if present must be laid out contiguously 3837 * and without gaps and in order. An IllegalArgumentException will be thrown 3838 * during {@link #queue} if access units are not laid out contiguously. 3839 * 3840 * @param block The linear block object 3841 * @param infos Represents {@link MediaCodec.BufferInfo} objects to mark 3842 * individual access-unit boundaries and the timestamps associated with it. 3843 * @return this object 3844 * @throws IllegalStateException if a buffer is already set 3845 */ 3846 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) setMultiFrameLinearBlock( @onNull LinearBlock block, @NonNull ArrayDeque<BufferInfo> infos)3847 public @NonNull QueueRequest setMultiFrameLinearBlock( 3848 @NonNull LinearBlock block, 3849 @NonNull ArrayDeque<BufferInfo> infos) { 3850 if (!isAccessible()) { 3851 throw new IllegalStateException("The request is stale"); 3852 } 3853 if (mLinearBlock != null || mHardwareBuffer != null) { 3854 throw new IllegalStateException("Cannot set block twice"); 3855 } 3856 mLinearBlock = block; 3857 mBufferInfos.clear(); 3858 mBufferInfos.addAll(infos); 3859 mCryptoInfos.clear(); 3860 return this; 3861 } 3862 3863 /** 3864 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3865 * set for a queue request before calling {@link #queue}. It is possible 3866 * to use the same {@link LinearBlock} object for multiple queue 3867 * requests. The behavior is undefined if the range of the buffer 3868 * overlaps for multiple requests, or the application writes into the 3869 * region being processed by the codec. 3870 * 3871 * @param block The linear block object 3872 * @param offset The byte offset into the input buffer at which the data starts. 3873 * @param size The number of bytes of valid input data. 3874 * @param cryptoInfo Metadata describing the structure of the encrypted input sample. 3875 * @return this object 3876 * @throws IllegalStateException if a buffer is already set 3877 */ setEncryptedLinearBlock( @onNull LinearBlock block, int offset, int size, @NonNull MediaCodec.CryptoInfo cryptoInfo)3878 public @NonNull QueueRequest setEncryptedLinearBlock( 3879 @NonNull LinearBlock block, 3880 int offset, 3881 int size, 3882 @NonNull MediaCodec.CryptoInfo cryptoInfo) { 3883 Objects.requireNonNull(cryptoInfo); 3884 if (!isAccessible()) { 3885 throw new IllegalStateException("The request is stale"); 3886 } 3887 if (mLinearBlock != null || mHardwareBuffer != null) { 3888 throw new IllegalStateException("Cannot set block twice"); 3889 } 3890 mLinearBlock = block; 3891 mOffset = offset; 3892 mSize = size; 3893 mCryptoInfos.clear(); 3894 mCryptoInfos.add(cryptoInfo); 3895 return this; 3896 } 3897 3898 /** 3899 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3900 * set for a queue request before calling {@link #queue}. The block can contain multiple 3901 * access units and if present should be laid out contiguously and without gaps. 3902 * 3903 * @param block The linear block object 3904 * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the 3905 * contents in the buffer. The ArrayDeque and the BufferInfo objects 3906 * provided can be recycled by the caller for re-use. 3907 * @param cryptoInfos ArrayDeque of {@link MediaCodec.CryptoInfo} that describes the 3908 * structure of the encrypted input samples. The ArrayDeque and the 3909 * BufferInfo objects provided can be recycled by the caller for re-use. 3910 * @return this object 3911 * @throws IllegalStateException if a buffer is already set 3912 * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the 3913 * access units are not contiguous. 3914 */ 3915 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) setMultiFrameEncryptedLinearBlock( @onNull LinearBlock block, @NonNull ArrayDeque<MediaCodec.BufferInfo> bufferInfos, @NonNull ArrayDeque<MediaCodec.CryptoInfo> cryptoInfos)3916 public @NonNull QueueRequest setMultiFrameEncryptedLinearBlock( 3917 @NonNull LinearBlock block, 3918 @NonNull ArrayDeque<MediaCodec.BufferInfo> bufferInfos, 3919 @NonNull ArrayDeque<MediaCodec.CryptoInfo> cryptoInfos) { 3920 if (!isAccessible()) { 3921 throw new IllegalStateException("The request is stale"); 3922 } 3923 if (mLinearBlock != null || mHardwareBuffer != null) { 3924 throw new IllegalStateException("Cannot set block twice"); 3925 } 3926 mLinearBlock = block; 3927 mBufferInfos.clear(); 3928 mBufferInfos.addAll(bufferInfos); 3929 mCryptoInfos.clear(); 3930 mCryptoInfos.addAll(cryptoInfos); 3931 return this; 3932 } 3933 3934 /** 3935 * Set a hardware graphic buffer to this queue request. Exactly one buffer must 3936 * be set for a queue request before calling {@link #queue}. Ownership of the 3937 * hardware buffer is not transferred to this queue request, nor will it be transferred 3938 * to the codec once {@link #queue} is called. 3939 * <p> 3940 * Note: buffers should have format {@link HardwareBuffer#YCBCR_420_888}, 3941 * a single layer, and an appropriate usage ({@link HardwareBuffer#USAGE_CPU_READ_OFTEN} 3942 * for software codecs and {@link HardwareBuffer#USAGE_VIDEO_ENCODE} for hardware) 3943 * for codecs to recognize. Format {@link ImageFormat#PRIVATE} together with 3944 * usage {@link HardwareBuffer#USAGE_VIDEO_ENCODE} will also work for hardware codecs. 3945 * Codecs may throw exception if the buffer is not recognizable. 3946 * 3947 * @param buffer The hardware graphic buffer object 3948 * @return this object 3949 * @throws IllegalStateException if a buffer is already set 3950 */ setHardwareBuffer( @onNull HardwareBuffer buffer)3951 public @NonNull QueueRequest setHardwareBuffer( 3952 @NonNull HardwareBuffer buffer) { 3953 if (!isAccessible()) { 3954 throw new IllegalStateException("The request is stale"); 3955 } 3956 if (mLinearBlock != null || mHardwareBuffer != null) { 3957 throw new IllegalStateException("Cannot set block twice"); 3958 } 3959 mHardwareBuffer = buffer; 3960 return this; 3961 } 3962 3963 /** 3964 * Set timestamp to this queue request. 3965 * 3966 * @param presentationTimeUs The presentation timestamp in microseconds for this 3967 * buffer. This is normally the media time at which this 3968 * buffer should be presented (rendered). When using an output 3969 * surface, this will be propagated as the {@link 3970 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3971 * conversion to nanoseconds). 3972 * @return this object 3973 */ setPresentationTimeUs(long presentationTimeUs)3974 public @NonNull QueueRequest setPresentationTimeUs(long presentationTimeUs) { 3975 if (!isAccessible()) { 3976 throw new IllegalStateException("The request is stale"); 3977 } 3978 mPresentationTimeUs = presentationTimeUs; 3979 return this; 3980 } 3981 3982 /** 3983 * Set flags to this queue request. 3984 * 3985 * @param flags A bitmask of flags 3986 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3987 * While not prohibited, most codecs do not use the 3988 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3989 * @return this object 3990 */ setFlags(@ufferFlag int flags)3991 public @NonNull QueueRequest setFlags(@BufferFlag int flags) { 3992 if (!isAccessible()) { 3993 throw new IllegalStateException("The request is stale"); 3994 } 3995 mFlags = flags; 3996 return this; 3997 } 3998 3999 /** 4000 * Add an integer parameter. 4001 * See {@link MediaFormat} for an exhaustive list of supported keys with 4002 * values of type int, that can also be set with {@link MediaFormat#setInteger}. 4003 * 4004 * If there was {@link MediaCodec#setParameters} 4005 * call with the same key which is not processed by the codec yet, the 4006 * value set from this method will override the unprocessed value. 4007 * 4008 * @return this object 4009 */ setIntegerParameter( @onNull String key, int value)4010 public @NonNull QueueRequest setIntegerParameter( 4011 @NonNull String key, int value) { 4012 if (!isAccessible()) { 4013 throw new IllegalStateException("The request is stale"); 4014 } 4015 mTuningKeys.add(key); 4016 mTuningValues.add(Integer.valueOf(value)); 4017 return this; 4018 } 4019 4020 /** 4021 * Add a long parameter. 4022 * See {@link MediaFormat} for an exhaustive list of supported keys with 4023 * values of type long, that can also be set with {@link MediaFormat#setLong}. 4024 * 4025 * If there was {@link MediaCodec#setParameters} 4026 * call with the same key which is not processed by the codec yet, the 4027 * value set from this method will override the unprocessed value. 4028 * 4029 * @return this object 4030 */ setLongParameter( @onNull String key, long value)4031 public @NonNull QueueRequest setLongParameter( 4032 @NonNull String key, long value) { 4033 if (!isAccessible()) { 4034 throw new IllegalStateException("The request is stale"); 4035 } 4036 mTuningKeys.add(key); 4037 mTuningValues.add(Long.valueOf(value)); 4038 return this; 4039 } 4040 4041 /** 4042 * Add a float parameter. 4043 * See {@link MediaFormat} for an exhaustive list of supported keys with 4044 * values of type float, that can also be set with {@link MediaFormat#setFloat}. 4045 * 4046 * If there was {@link MediaCodec#setParameters} 4047 * call with the same key which is not processed by the codec yet, the 4048 * value set from this method will override the unprocessed value. 4049 * 4050 * @return this object 4051 */ setFloatParameter( @onNull String key, float value)4052 public @NonNull QueueRequest setFloatParameter( 4053 @NonNull String key, float value) { 4054 if (!isAccessible()) { 4055 throw new IllegalStateException("The request is stale"); 4056 } 4057 mTuningKeys.add(key); 4058 mTuningValues.add(Float.valueOf(value)); 4059 return this; 4060 } 4061 4062 /** 4063 * Add a {@link ByteBuffer} parameter. 4064 * See {@link MediaFormat} for an exhaustive list of supported keys with 4065 * values of byte buffer, that can also be set with {@link MediaFormat#setByteBuffer}. 4066 * 4067 * If there was {@link MediaCodec#setParameters} 4068 * call with the same key which is not processed by the codec yet, the 4069 * value set from this method will override the unprocessed value. 4070 * 4071 * @return this object 4072 */ setByteBufferParameter( @onNull String key, @NonNull ByteBuffer value)4073 public @NonNull QueueRequest setByteBufferParameter( 4074 @NonNull String key, @NonNull ByteBuffer value) { 4075 if (!isAccessible()) { 4076 throw new IllegalStateException("The request is stale"); 4077 } 4078 mTuningKeys.add(key); 4079 mTuningValues.add(value); 4080 return this; 4081 } 4082 4083 /** 4084 * Add a string parameter. 4085 * See {@link MediaFormat} for an exhaustive list of supported keys with 4086 * values of type string, that can also be set with {@link MediaFormat#setString}. 4087 * 4088 * If there was {@link MediaCodec#setParameters} 4089 * call with the same key which is not processed by the codec yet, the 4090 * value set from this method will override the unprocessed value. 4091 * 4092 * @return this object 4093 */ setStringParameter( @onNull String key, @NonNull String value)4094 public @NonNull QueueRequest setStringParameter( 4095 @NonNull String key, @NonNull String value) { 4096 if (!isAccessible()) { 4097 throw new IllegalStateException("The request is stale"); 4098 } 4099 mTuningKeys.add(key); 4100 mTuningValues.add(value); 4101 return this; 4102 } 4103 4104 /** 4105 * Finish building a queue request and queue the buffers with tunings. 4106 */ queue()4107 public void queue() { 4108 Trace.traceBegin(Trace.TRACE_TAG_VIDEO, "MediaCodec::queueRequest-queue#java"); 4109 if (!isAccessible()) { 4110 throw new IllegalStateException("The request is stale"); 4111 } 4112 if (mLinearBlock == null && mHardwareBuffer == null) { 4113 throw new IllegalStateException("No block is set"); 4114 } 4115 setAccessible(false); 4116 if (mBufferInfos.isEmpty()) { 4117 BufferInfo info = new BufferInfo(); 4118 info.size = mSize; 4119 info.offset = mOffset; 4120 info.presentationTimeUs = mPresentationTimeUs; 4121 info.flags = mFlags; 4122 mBufferInfos.add(info); 4123 } 4124 if (mLinearBlock != null) { 4125 4126 mCodec.native_queueLinearBlock( 4127 mIndex, mLinearBlock, 4128 mCryptoInfos.isEmpty() ? null : mCryptoInfos.toArray(), 4129 mBufferInfos.toArray(), 4130 mTuningKeys, mTuningValues); 4131 } else if (mHardwareBuffer != null) { 4132 mCodec.native_queueHardwareBuffer( 4133 mIndex, mHardwareBuffer, mPresentationTimeUs, mFlags, 4134 mTuningKeys, mTuningValues); 4135 } 4136 clear(); 4137 Trace.traceEnd(Trace.TRACE_TAG_VIDEO); 4138 } 4139 clear()4140 @NonNull QueueRequest clear() { 4141 mLinearBlock = null; 4142 mOffset = 0; 4143 mSize = 0; 4144 mHardwareBuffer = null; 4145 mPresentationTimeUs = 0; 4146 mFlags = 0; 4147 mBufferInfos.clear(); 4148 mCryptoInfos.clear(); 4149 mTuningKeys.clear(); 4150 mTuningValues.clear(); 4151 return this; 4152 } 4153 isAccessible()4154 boolean isAccessible() { 4155 return mAccessible; 4156 } 4157 setAccessible(boolean accessible)4158 @NonNull QueueRequest setAccessible(boolean accessible) { 4159 mAccessible = accessible; 4160 return this; 4161 } 4162 4163 private final MediaCodec mCodec; 4164 private final int mIndex; 4165 private LinearBlock mLinearBlock = null; 4166 private int mOffset = 0; 4167 private int mSize = 0; 4168 private HardwareBuffer mHardwareBuffer = null; 4169 private long mPresentationTimeUs = 0; 4170 private @BufferFlag int mFlags = 0; 4171 private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>(); 4172 private final ArrayDeque<CryptoInfo> mCryptoInfos = new ArrayDeque<>(); 4173 private final ArrayList<String> mTuningKeys = new ArrayList<>(); 4174 private final ArrayList<Object> mTuningValues = new ArrayList<>(); 4175 4176 private boolean mAccessible = false; 4177 } 4178 native_queueLinearBlock( int index, @NonNull LinearBlock block, @Nullable Object[] cryptoInfos, @NonNull Object[] bufferInfos, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)4179 private native void native_queueLinearBlock( 4180 int index, 4181 @NonNull LinearBlock block, 4182 @Nullable Object[] cryptoInfos, 4183 @NonNull Object[] bufferInfos, 4184 @NonNull ArrayList<String> keys, 4185 @NonNull ArrayList<Object> values); 4186 native_queueHardwareBuffer( int index, @NonNull HardwareBuffer buffer, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)4187 private native void native_queueHardwareBuffer( 4188 int index, 4189 @NonNull HardwareBuffer buffer, 4190 long presentationTimeUs, 4191 int flags, 4192 @NonNull ArrayList<String> keys, 4193 @NonNull ArrayList<Object> values); 4194 4195 private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); 4196 4197 /** 4198 * Return a {@link QueueRequest} object for an input slot index. 4199 * 4200 * @param index input slot index from 4201 * {@link Callback#onInputBufferAvailable} 4202 * @return queue request object 4203 * @throws IllegalStateException if not using block model 4204 * @throws IllegalArgumentException if the input slot is not available or 4205 * the index is out of range 4206 */ getQueueRequest(int index)4207 public @NonNull QueueRequest getQueueRequest(int index) { 4208 synchronized (mBufferLock) { 4209 if (mBufferMode != BUFFER_MODE_BLOCK) { 4210 throw new IllegalStateException("The codec is not configured for block model"); 4211 } 4212 if (index < 0 || index >= mQueueRequests.size()) { 4213 throw new IndexOutOfBoundsException("Expected range of index: [0," 4214 + (mQueueRequests.size() - 1) + "]; actual: " + index); 4215 } 4216 QueueRequest request = mQueueRequests.get(index); 4217 if (request == null) { 4218 throw new IllegalArgumentException("Unavailable index: " + index); 4219 } 4220 if (!request.isAccessible()) { 4221 throw new IllegalArgumentException( 4222 "The request is stale at index " + index); 4223 } 4224 return request.clear(); 4225 } 4226 } 4227 4228 /** 4229 * If a non-negative timeout had been specified in the call 4230 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 4231 */ 4232 public static final int INFO_TRY_AGAIN_LATER = -1; 4233 4234 /** 4235 * The output format has changed, subsequent data will follow the new 4236 * format. {@link #getOutputFormat()} returns the new format. Note, that 4237 * you can also use the new {@link #getOutputFormat(int)} method to 4238 * get the format for a specific output buffer. This frees you from 4239 * having to track output format changes. 4240 */ 4241 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 4242 4243 /** 4244 * The output buffers have changed, the client must refer to the new 4245 * set of output buffers returned by {@link #getOutputBuffers} from 4246 * this point on. 4247 * 4248 * <p>Additionally, this event signals that the video scaling mode 4249 * may have been reset to the default.</p> 4250 * 4251 * @deprecated This return value can be ignored as {@link 4252 * #getOutputBuffers} has been deprecated. Client should 4253 * request a current buffer using on of the get-buffer or 4254 * get-image methods each time one has been dequeued. 4255 */ 4256 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 4257 4258 /** @hide */ 4259 @IntDef({ 4260 INFO_TRY_AGAIN_LATER, 4261 INFO_OUTPUT_FORMAT_CHANGED, 4262 INFO_OUTPUT_BUFFERS_CHANGED, 4263 }) 4264 @Retention(RetentionPolicy.SOURCE) 4265 public @interface OutputBufferInfo {} 4266 4267 /** 4268 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 4269 * Returns the index of an output buffer that has been successfully 4270 * decoded or one of the INFO_* constants. 4271 * @param info Will be filled with buffer meta data. 4272 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 4273 * @throws IllegalStateException if not in the Executing state, 4274 * or codec is configured in asynchronous mode. 4275 * @throws MediaCodec.CodecException upon codec error. 4276 */ 4277 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)4278 public final int dequeueOutputBuffer( 4279 @NonNull BufferInfo info, long timeoutUs) { 4280 synchronized (mBufferLock) { 4281 if (mBufferMode == BUFFER_MODE_BLOCK) { 4282 throw new IncompatibleWithBlockModelException("dequeueOutputBuffer() " 4283 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4284 + "Please use MediaCodec.Callback objects to get output buffer slots."); 4285 } 4286 } 4287 int res = native_dequeueOutputBuffer(info, timeoutUs); 4288 synchronized (mBufferLock) { 4289 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 4290 cacheBuffersLocked(false /* input */); 4291 } else if (res >= 0) { 4292 validateOutputByteBufferLocked(mCachedOutputBuffers, res, info); 4293 if (mHasSurface || mCachedOutputBuffers == null) { 4294 mDequeuedOutputInfos.put(res, info.dup()); 4295 } 4296 } 4297 } 4298 return res; 4299 } 4300 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)4301 private native final int native_dequeueOutputBuffer( 4302 @NonNull BufferInfo info, long timeoutUs); 4303 4304 /** 4305 * If you are done with a buffer, use this call to return the buffer to the codec 4306 * or to render it on the output surface. If you configured the codec with an 4307 * output surface, setting {@code render} to {@code true} will first send the buffer 4308 * to that output surface. The surface will release the buffer back to the codec once 4309 * it is no longer used/displayed. 4310 * 4311 * Once an output buffer is released to the codec, it MUST NOT 4312 * be used until it is later retrieved by {@link #getOutputBuffer} in response 4313 * to a {@link #dequeueOutputBuffer} return value or a 4314 * {@link Callback#onOutputBufferAvailable} callback. 4315 * 4316 * @param index The index of a client-owned output buffer previously returned 4317 * from a call to {@link #dequeueOutputBuffer}. 4318 * @param render If a valid surface was specified when configuring the codec, 4319 * passing true renders this output buffer to the surface. 4320 * @throws IllegalStateException if not in the Executing state. 4321 * @throws MediaCodec.CodecException upon codec error. 4322 */ releaseOutputBuffer(int index, boolean render)4323 public final void releaseOutputBuffer(int index, boolean render) { 4324 releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); 4325 } 4326 4327 /** 4328 * If you are done with a buffer, use this call to update its surface timestamp 4329 * and return it to the codec to render it on the output surface. If you 4330 * have not specified an output surface when configuring this video codec, 4331 * this call will simply return the buffer to the codec.<p> 4332 * 4333 * The timestamp may have special meaning depending on the destination surface. 4334 * 4335 * <table> 4336 * <tr><th>SurfaceView specifics</th></tr> 4337 * <tr><td> 4338 * If you render your buffer on a {@link android.view.SurfaceView}, 4339 * you can use the timestamp to render the buffer at a specific time (at the 4340 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 4341 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 4342 * Currently, this is set as within one (1) second. A few notes: 4343 * 4344 * <ul> 4345 * <li>the buffer will not be returned to the codec until the timestamp 4346 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 4347 * <li>buffers are processed sequentially, so you may block subsequent buffers to 4348 * be displayed on the {@link android.view.Surface}. This is important if you 4349 * want to react to user action, e.g. stop the video or seek. 4350 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 4351 * rendered at the same VSYNC, the last one will be shown, and the other ones 4352 * will be dropped. 4353 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 4354 * time, the {@link android.view.Surface} will ignore the timestamp, and 4355 * display the buffer at the earliest feasible time. In this mode it will not 4356 * drop frames. 4357 * <li>for best performance and quality, call this method when you are about 4358 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 4359 * about 33 msec. 4360 * </ul> 4361 * </td></tr> 4362 * </table> 4363 * 4364 * Once an output buffer is released to the codec, it MUST NOT 4365 * be used until it is later retrieved by {@link #getOutputBuffer} in response 4366 * to a {@link #dequeueOutputBuffer} return value or a 4367 * {@link Callback#onOutputBufferAvailable} callback. 4368 * 4369 * @param index The index of a client-owned output buffer previously returned 4370 * from a call to {@link #dequeueOutputBuffer}. 4371 * @param renderTimestampNs The timestamp to associate with this buffer when 4372 * it is sent to the Surface. 4373 * @throws IllegalStateException if not in the Executing state. 4374 * @throws MediaCodec.CodecException upon codec error. 4375 */ releaseOutputBuffer(int index, long renderTimestampNs)4376 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 4377 releaseOutputBufferInternal( 4378 index, true /* render */, true /* updatePTS */, renderTimestampNs); 4379 } 4380 releaseOutputBufferInternal( int index, boolean render, boolean updatePts, long renderTimestampNs)4381 private void releaseOutputBufferInternal( 4382 int index, boolean render, boolean updatePts, long renderTimestampNs) { 4383 BufferInfo info = null; 4384 synchronized(mBufferLock) { 4385 switch (mBufferMode) { 4386 case BUFFER_MODE_LEGACY: 4387 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4388 mDequeuedOutputBuffers.remove(index); 4389 if (mHasSurface || mCachedOutputBuffers == null) { 4390 info = mDequeuedOutputInfos.remove(index); 4391 } 4392 break; 4393 case BUFFER_MODE_BLOCK: 4394 OutputFrame frame = mOutputFrames.get(index); 4395 frame.setAccessible(false); 4396 frame.clear(); 4397 break; 4398 default: 4399 throw new IllegalStateException( 4400 "Unrecognized buffer mode: " + mBufferMode); 4401 } 4402 } 4403 releaseOutputBuffer( 4404 index, render, updatePts, renderTimestampNs); 4405 } 4406 4407 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)4408 private native final void releaseOutputBuffer( 4409 int index, boolean render, boolean updatePTS, long timeNs); 4410 4411 /** 4412 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 4413 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 4414 * encoders receiving input from a Surface created by {@link #createInputSurface}. 4415 * @throws IllegalStateException if not in the Executing state. 4416 * @throws MediaCodec.CodecException upon codec error. 4417 */ signalEndOfInputStream()4418 public native final void signalEndOfInputStream(); 4419 4420 /** 4421 * Call this after dequeueOutputBuffer signals a format change by returning 4422 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 4423 * You can also call this after {@link #configure} returns 4424 * successfully to get the output format initially configured 4425 * for the codec. Do this to determine what optional 4426 * configuration parameters were supported by the codec. 4427 * 4428 * @throws IllegalStateException if not in the Executing or 4429 * Configured state. 4430 * @throws MediaCodec.CodecException upon codec error. 4431 */ 4432 @NonNull getOutputFormat()4433 public final MediaFormat getOutputFormat() { 4434 return new MediaFormat(getFormatNative(false /* input */)); 4435 } 4436 4437 /** 4438 * Call this after {@link #configure} returns successfully to 4439 * get the input format accepted by the codec. Do this to 4440 * determine what optional configuration parameters were 4441 * supported by the codec. 4442 * 4443 * @throws IllegalStateException if not in the Executing or 4444 * Configured state. 4445 * @throws MediaCodec.CodecException upon codec error. 4446 */ 4447 @NonNull getInputFormat()4448 public final MediaFormat getInputFormat() { 4449 return new MediaFormat(getFormatNative(true /* input */)); 4450 } 4451 4452 /** 4453 * Returns the output format for a specific output buffer. 4454 * 4455 * @param index The index of a client-owned input buffer previously 4456 * returned from a call to {@link #dequeueInputBuffer}. 4457 * 4458 * @return the format for the output buffer, or null if the index 4459 * is not a dequeued output buffer. 4460 */ 4461 @NonNull getOutputFormat(int index)4462 public final MediaFormat getOutputFormat(int index) { 4463 return new MediaFormat(getOutputFormatNative(index)); 4464 } 4465 4466 @NonNull getFormatNative(boolean input)4467 private native final Map<String, Object> getFormatNative(boolean input); 4468 4469 @NonNull getOutputFormatNative(int index)4470 private native final Map<String, Object> getOutputFormatNative(int index); 4471 4472 // used to track dequeued buffers 4473 private static class BufferMap { 4474 // various returned representations of the codec buffer 4475 private static class CodecBuffer { 4476 private Image mImage; 4477 private ByteBuffer mByteBuffer; 4478 free()4479 public void free() { 4480 if (mByteBuffer != null) { 4481 // all of our ByteBuffers are direct 4482 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 4483 mByteBuffer = null; 4484 } 4485 if (mImage != null) { 4486 mImage.close(); 4487 mImage = null; 4488 } 4489 } 4490 setImage(@ullable Image image)4491 public void setImage(@Nullable Image image) { 4492 free(); 4493 mImage = image; 4494 } 4495 setByteBuffer(@ullable ByteBuffer buffer)4496 public void setByteBuffer(@Nullable ByteBuffer buffer) { 4497 free(); 4498 mByteBuffer = buffer; 4499 } 4500 } 4501 4502 private final Map<Integer, CodecBuffer> mMap = 4503 new HashMap<Integer, CodecBuffer>(); 4504 remove(int index)4505 public void remove(int index) { 4506 CodecBuffer buffer = mMap.get(index); 4507 if (buffer != null) { 4508 buffer.free(); 4509 mMap.remove(index); 4510 } 4511 } 4512 put(int index, @Nullable ByteBuffer newBuffer)4513 public void put(int index, @Nullable ByteBuffer newBuffer) { 4514 CodecBuffer buffer = mMap.get(index); 4515 if (buffer == null) { // likely 4516 buffer = new CodecBuffer(); 4517 mMap.put(index, buffer); 4518 } 4519 buffer.setByteBuffer(newBuffer); 4520 } 4521 put(int index, @Nullable Image newImage)4522 public void put(int index, @Nullable Image newImage) { 4523 CodecBuffer buffer = mMap.get(index); 4524 if (buffer == null) { // likely 4525 buffer = new CodecBuffer(); 4526 mMap.put(index, buffer); 4527 } 4528 buffer.setImage(newImage); 4529 } 4530 clear()4531 public void clear() { 4532 for (CodecBuffer buffer: mMap.values()) { 4533 buffer.free(); 4534 } 4535 mMap.clear(); 4536 } 4537 } 4538 4539 private ByteBuffer[] mCachedInputBuffers; 4540 private ByteBuffer[] mCachedOutputBuffers; 4541 private BitSet mValidInputIndices = new BitSet(); 4542 private BitSet mValidOutputIndices = new BitSet(); 4543 4544 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 4545 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 4546 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 4547 new HashMap<Integer, BufferInfo>(); 4548 final private Object mBufferLock; 4549 invalidateByteBufferLocked( @ullable ByteBuffer[] buffers, int index, boolean input)4550 private void invalidateByteBufferLocked( 4551 @Nullable ByteBuffer[] buffers, int index, boolean input) { 4552 if (buffers == null) { 4553 if (index >= 0) { 4554 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4555 indices.clear(index); 4556 } 4557 } else if (index >= 0 && index < buffers.length) { 4558 ByteBuffer buffer = buffers[index]; 4559 if (buffer != null) { 4560 buffer.setAccessible(false); 4561 } 4562 } 4563 } 4564 validateInputByteBufferLocked( @ullable ByteBuffer[] buffers, int index)4565 private void validateInputByteBufferLocked( 4566 @Nullable ByteBuffer[] buffers, int index) { 4567 if (buffers == null) { 4568 if (index >= 0) { 4569 mValidInputIndices.set(index); 4570 } 4571 } else if (index >= 0 && index < buffers.length) { 4572 ByteBuffer buffer = buffers[index]; 4573 if (buffer != null) { 4574 buffer.setAccessible(true); 4575 buffer.clear(); 4576 } 4577 } 4578 } 4579 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index, boolean input)4580 private void revalidateByteBuffer( 4581 @Nullable ByteBuffer[] buffers, int index, boolean input) { 4582 synchronized(mBufferLock) { 4583 if (buffers == null) { 4584 if (index >= 0) { 4585 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4586 indices.set(index); 4587 } 4588 } else if (index >= 0 && index < buffers.length) { 4589 ByteBuffer buffer = buffers[index]; 4590 if (buffer != null) { 4591 buffer.setAccessible(true); 4592 } 4593 } 4594 } 4595 } 4596 validateOutputByteBuffersLocked( @ullable ByteBuffer[] buffers, int index, @NonNull ArrayDeque<BufferInfo> infoDeque)4597 private void validateOutputByteBuffersLocked( 4598 @Nullable ByteBuffer[] buffers, int index, @NonNull ArrayDeque<BufferInfo> infoDeque) { 4599 Optional<BufferInfo> minInfo = infoDeque.stream().min( 4600 (info1, info2) -> Integer.compare(info1.offset, info2.offset)); 4601 Optional<BufferInfo> maxInfo = infoDeque.stream().max( 4602 (info1, info2) -> Integer.compare(info1.offset, info2.offset)); 4603 if (buffers == null) { 4604 if (index >= 0) { 4605 mValidOutputIndices.set(index); 4606 } 4607 } else if (index >= 0 && index < buffers.length) { 4608 ByteBuffer buffer = buffers[index]; 4609 if (buffer != null && minInfo.isPresent() && maxInfo.isPresent()) { 4610 buffer.setAccessible(true); 4611 buffer.limit(maxInfo.get().offset + maxInfo.get().size); 4612 buffer.position(minInfo.get().offset); 4613 } 4614 } 4615 4616 } 4617 validateOutputByteBufferLocked( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)4618 private void validateOutputByteBufferLocked( 4619 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 4620 if (buffers == null) { 4621 if (index >= 0) { 4622 mValidOutputIndices.set(index); 4623 } 4624 } else if (index >= 0 && index < buffers.length) { 4625 ByteBuffer buffer = buffers[index]; 4626 if (buffer != null) { 4627 buffer.setAccessible(true); 4628 buffer.limit(info.offset + info.size).position(info.offset); 4629 } 4630 } 4631 } 4632 invalidateByteBuffersLocked(@ullable ByteBuffer[] buffers)4633 private void invalidateByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 4634 if (buffers != null) { 4635 for (ByteBuffer buffer: buffers) { 4636 if (buffer != null) { 4637 buffer.setAccessible(false); 4638 } 4639 } 4640 } 4641 } 4642 freeByteBufferLocked(@ullable ByteBuffer buffer)4643 private void freeByteBufferLocked(@Nullable ByteBuffer buffer) { 4644 if (buffer != null /* && buffer.isDirect() */) { 4645 // all of our ByteBuffers are direct 4646 java.nio.NioUtils.freeDirectBuffer(buffer); 4647 } 4648 } 4649 freeByteBuffersLocked(@ullable ByteBuffer[] buffers)4650 private void freeByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 4651 if (buffers != null) { 4652 for (ByteBuffer buffer: buffers) { 4653 freeByteBufferLocked(buffer); 4654 } 4655 } 4656 } 4657 freeAllTrackedBuffers()4658 private void freeAllTrackedBuffers() { 4659 synchronized(mBufferLock) { 4660 freeByteBuffersLocked(mCachedInputBuffers); 4661 freeByteBuffersLocked(mCachedOutputBuffers); 4662 mCachedInputBuffers = null; 4663 mCachedOutputBuffers = null; 4664 mValidInputIndices.clear(); 4665 mValidOutputIndices.clear(); 4666 mDequeuedInputBuffers.clear(); 4667 mDequeuedOutputBuffers.clear(); 4668 mQueueRequests.clear(); 4669 mOutputFrames.clear(); 4670 } 4671 } 4672 cacheBuffersLocked(boolean input)4673 private void cacheBuffersLocked(boolean input) { 4674 ByteBuffer[] buffers = null; 4675 try { 4676 buffers = getBuffers(input); 4677 invalidateByteBuffersLocked(buffers); 4678 } catch (IllegalStateException e) { 4679 // we don't get buffers in async mode 4680 } 4681 if (buffers != null) { 4682 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4683 for (int i = 0; i < buffers.length; ++i) { 4684 ByteBuffer buffer = buffers[i]; 4685 if (buffer == null || !indices.get(i)) { 4686 continue; 4687 } 4688 buffer.setAccessible(true); 4689 if (!input) { 4690 BufferInfo info = mDequeuedOutputInfos.get(i); 4691 if (info != null) { 4692 buffer.limit(info.offset + info.size).position(info.offset); 4693 } 4694 } 4695 } 4696 indices.clear(); 4697 } 4698 if (input) { 4699 mCachedInputBuffers = buffers; 4700 } else { 4701 mCachedOutputBuffers = buffers; 4702 } 4703 } 4704 4705 /** 4706 * Retrieve the set of input buffers. Call this after start() 4707 * returns. After calling this method, any ByteBuffers 4708 * previously returned by an earlier call to this method MUST no 4709 * longer be used. 4710 * 4711 * @deprecated Use the new {@link #getInputBuffer} method instead 4712 * each time an input buffer is dequeued. 4713 * 4714 * <b>Note:</b> As of API 21, dequeued input buffers are 4715 * automatically {@link java.nio.Buffer#clear cleared}. 4716 * 4717 * <em>Do not use this method if using an input surface.</em> 4718 * 4719 * @throws IllegalStateException if not in the Executing state, 4720 * or codec is configured in asynchronous mode. 4721 * @throws MediaCodec.CodecException upon codec error. 4722 */ 4723 @NonNull getInputBuffers()4724 public ByteBuffer[] getInputBuffers() { 4725 synchronized (mBufferLock) { 4726 if (mBufferMode == BUFFER_MODE_BLOCK) { 4727 throw new IncompatibleWithBlockModelException("getInputBuffers() " 4728 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4729 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4730 + "objects and attach to QueueRequest objects."); 4731 } 4732 if (mCachedInputBuffers == null) { 4733 cacheBuffersLocked(true /* input */); 4734 } 4735 if (mCachedInputBuffers == null) { 4736 throw new IllegalStateException(); 4737 } 4738 // FIXME: check codec status 4739 return mCachedInputBuffers; 4740 } 4741 } 4742 4743 /** 4744 * Retrieve the set of output buffers. Call this after start() 4745 * returns and whenever dequeueOutputBuffer signals an output 4746 * buffer change by returning {@link 4747 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 4748 * ByteBuffers previously returned by an earlier call to this 4749 * method MUST no longer be used. 4750 * 4751 * @deprecated Use the new {@link #getOutputBuffer} method instead 4752 * each time an output buffer is dequeued. This method is not 4753 * supported if codec is configured in asynchronous mode. 4754 * 4755 * <b>Note:</b> As of API 21, the position and limit of output 4756 * buffers that are dequeued will be set to the valid data 4757 * range. 4758 * 4759 * <em>Do not use this method if using an output surface.</em> 4760 * 4761 * @throws IllegalStateException if not in the Executing state, 4762 * or codec is configured in asynchronous mode. 4763 * @throws MediaCodec.CodecException upon codec error. 4764 */ 4765 @NonNull getOutputBuffers()4766 public ByteBuffer[] getOutputBuffers() { 4767 synchronized (mBufferLock) { 4768 if (mBufferMode == BUFFER_MODE_BLOCK) { 4769 throw new IncompatibleWithBlockModelException("getOutputBuffers() " 4770 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4771 + "Please use getOutputFrame to get output frames."); 4772 } 4773 if (mCachedOutputBuffers == null) { 4774 cacheBuffersLocked(false /* input */); 4775 } 4776 if (mCachedOutputBuffers == null) { 4777 throw new IllegalStateException(); 4778 } 4779 // FIXME: check codec status 4780 return mCachedOutputBuffers; 4781 } 4782 } 4783 4784 /** 4785 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 4786 * object for a dequeued input buffer index to contain the input data. 4787 * 4788 * After calling this method any ByteBuffer or Image object 4789 * previously returned for the same input index MUST no longer 4790 * be used. 4791 * 4792 * @param index The index of a client-owned input buffer previously 4793 * returned from a call to {@link #dequeueInputBuffer}, 4794 * or received via an onInputBufferAvailable callback. 4795 * 4796 * @return the input buffer, or null if the index is not a dequeued 4797 * input buffer, or if the codec is configured for surface input. 4798 * 4799 * @throws IllegalStateException if not in the Executing state. 4800 * @throws MediaCodec.CodecException upon codec error. 4801 */ 4802 @Nullable getInputBuffer(int index)4803 public ByteBuffer getInputBuffer(int index) { 4804 synchronized (mBufferLock) { 4805 if (mBufferMode == BUFFER_MODE_BLOCK) { 4806 throw new IncompatibleWithBlockModelException("getInputBuffer() " 4807 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4808 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4809 + "objects and attach to QueueRequest objects."); 4810 } 4811 } 4812 ByteBuffer newBuffer = getBuffer(true /* input */, index); 4813 synchronized (mBufferLock) { 4814 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4815 mDequeuedInputBuffers.put(index, newBuffer); 4816 } 4817 return newBuffer; 4818 } 4819 4820 /** 4821 * Returns a writable Image object for a dequeued input buffer 4822 * index to contain the raw input video frame. 4823 * 4824 * After calling this method any ByteBuffer or Image object 4825 * previously returned for the same input index MUST no longer 4826 * be used. 4827 * 4828 * @param index The index of a client-owned input buffer previously 4829 * returned from a call to {@link #dequeueInputBuffer}, 4830 * or received via an onInputBufferAvailable callback. 4831 * 4832 * @return the input image, or null if the index is not a 4833 * dequeued input buffer, or not a ByteBuffer that contains a 4834 * raw image. 4835 * 4836 * @throws IllegalStateException if not in the Executing state. 4837 * @throws MediaCodec.CodecException upon codec error. 4838 */ 4839 @Nullable getInputImage(int index)4840 public Image getInputImage(int index) { 4841 synchronized (mBufferLock) { 4842 if (mBufferMode == BUFFER_MODE_BLOCK) { 4843 throw new IncompatibleWithBlockModelException("getInputImage() " 4844 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4845 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4846 + "objects and attach to QueueRequest objects."); 4847 } 4848 } 4849 Image newImage = getImage(true /* input */, index); 4850 synchronized (mBufferLock) { 4851 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4852 mDequeuedInputBuffers.put(index, newImage); 4853 } 4854 return newImage; 4855 } 4856 4857 /** 4858 * Returns a read-only ByteBuffer for a dequeued output buffer 4859 * index. The position and limit of the returned buffer are set 4860 * to the valid output data. 4861 * 4862 * After calling this method, any ByteBuffer or Image object 4863 * previously returned for the same output index MUST no longer 4864 * be used. 4865 * 4866 * @param index The index of a client-owned output buffer previously 4867 * returned from a call to {@link #dequeueOutputBuffer}, 4868 * or received via an onOutputBufferAvailable callback. 4869 * 4870 * @return the output buffer, or null if the index is not a dequeued 4871 * output buffer, or the codec is configured with an output surface. 4872 * 4873 * @throws IllegalStateException if not in the Executing state. 4874 * @throws MediaCodec.CodecException upon codec error. 4875 */ 4876 @Nullable getOutputBuffer(int index)4877 public ByteBuffer getOutputBuffer(int index) { 4878 synchronized (mBufferLock) { 4879 if (mBufferMode == BUFFER_MODE_BLOCK) { 4880 throw new IncompatibleWithBlockModelException("getOutputBuffer() " 4881 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4882 + "Please use getOutputFrame() to get output frames."); 4883 } 4884 } 4885 ByteBuffer newBuffer = getBuffer(false /* input */, index); 4886 synchronized (mBufferLock) { 4887 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4888 mDequeuedOutputBuffers.put(index, newBuffer); 4889 } 4890 return newBuffer; 4891 } 4892 4893 /** 4894 * Returns a read-only Image object for a dequeued output buffer 4895 * index that contains the raw video frame. 4896 * 4897 * After calling this method, any ByteBuffer or Image object previously 4898 * returned for the same output index MUST no longer be used. 4899 * 4900 * @param index The index of a client-owned output buffer previously 4901 * returned from a call to {@link #dequeueOutputBuffer}, 4902 * or received via an onOutputBufferAvailable callback. 4903 * 4904 * @return the output image, or null if the index is not a 4905 * dequeued output buffer, not a raw video frame, or if the codec 4906 * was configured with an output surface. 4907 * 4908 * @throws IllegalStateException if not in the Executing state. 4909 * @throws MediaCodec.CodecException upon codec error. 4910 */ 4911 @Nullable getOutputImage(int index)4912 public Image getOutputImage(int index) { 4913 synchronized (mBufferLock) { 4914 if (mBufferMode == BUFFER_MODE_BLOCK) { 4915 throw new IncompatibleWithBlockModelException("getOutputImage() " 4916 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4917 + "Please use getOutputFrame() to get output frames."); 4918 } 4919 } 4920 Image newImage = getImage(false /* input */, index); 4921 synchronized (mBufferLock) { 4922 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4923 mDequeuedOutputBuffers.put(index, newImage); 4924 } 4925 return newImage; 4926 } 4927 4928 /** 4929 * A single output frame and its associated metadata. 4930 */ 4931 public static final class OutputFrame { 4932 // No public constructor OutputFrame(int index)4933 OutputFrame(int index) { 4934 mIndex = index; 4935 } 4936 4937 /** 4938 * Returns the output linear block, or null if this frame is empty. 4939 * 4940 * @throws IllegalStateException if this output frame is not linear. 4941 */ getLinearBlock()4942 public @Nullable LinearBlock getLinearBlock() { 4943 if (mHardwareBuffer != null) { 4944 throw new IllegalStateException("This output frame is not linear"); 4945 } 4946 return mLinearBlock; 4947 } 4948 4949 /** 4950 * Returns the output hardware graphic buffer, or null if this frame is empty. 4951 * 4952 * @throws IllegalStateException if this output frame is not graphic. 4953 */ getHardwareBuffer()4954 public @Nullable HardwareBuffer getHardwareBuffer() { 4955 if (mLinearBlock != null) { 4956 throw new IllegalStateException("This output frame is not graphic"); 4957 } 4958 return mHardwareBuffer; 4959 } 4960 4961 /** 4962 * Returns the presentation timestamp in microseconds. 4963 */ getPresentationTimeUs()4964 public long getPresentationTimeUs() { 4965 return mPresentationTimeUs; 4966 } 4967 4968 /** 4969 * Returns the buffer flags. 4970 */ getFlags()4971 public @BufferFlag int getFlags() { 4972 return mFlags; 4973 } 4974 4975 /* 4976 * Returns the BufferInfos associated with this OutputFrame. These BufferInfos 4977 * describes the access units present in the OutputFrame. Access units are laid 4978 * out contiguously without gaps and in order. 4979 */ 4980 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) getBufferInfos()4981 public @NonNull ArrayDeque<BufferInfo> getBufferInfos() { 4982 if (mBufferInfos.isEmpty()) { 4983 // single BufferInfo could be present. 4984 BufferInfo bufferInfo = new BufferInfo(); 4985 bufferInfo.set(0, 0, mPresentationTimeUs, mFlags); 4986 mBufferInfos.add(bufferInfo); 4987 } 4988 return mBufferInfos; 4989 } 4990 4991 /** 4992 * Returns a read-only {@link MediaFormat} for this frame. The returned 4993 * object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}. 4994 */ getFormat()4995 public @NonNull MediaFormat getFormat() { 4996 return mFormat; 4997 } 4998 4999 /** 5000 * Returns an unmodifiable set of the names of entries that has changed from 5001 * the previous frame. The entries may have been removed/changed/added. 5002 * Client can find out what the change is by querying {@link MediaFormat} 5003 * object returned from {@link #getFormat}. 5004 */ getChangedKeys()5005 public @NonNull Set<String> getChangedKeys() { 5006 if (mKeySet.isEmpty() && !mChangedKeys.isEmpty()) { 5007 mKeySet.addAll(mChangedKeys); 5008 } 5009 return Collections.unmodifiableSet(mKeySet); 5010 } 5011 clear()5012 void clear() { 5013 mLinearBlock = null; 5014 mHardwareBuffer = null; 5015 mFormat = null; 5016 mBufferInfos.clear(); 5017 mChangedKeys.clear(); 5018 mKeySet.clear(); 5019 mLoaded = false; 5020 } 5021 isAccessible()5022 boolean isAccessible() { 5023 return mAccessible; 5024 } 5025 setAccessible(boolean accessible)5026 void setAccessible(boolean accessible) { 5027 mAccessible = accessible; 5028 } 5029 setBufferInfo(MediaCodec.BufferInfo info)5030 void setBufferInfo(MediaCodec.BufferInfo info) { 5031 // since any of setBufferInfo(s) should translate to getBufferInfos, 5032 // mBufferInfos needs to be reset for every setBufferInfo(s) 5033 mBufferInfos.clear(); 5034 mPresentationTimeUs = info.presentationTimeUs; 5035 mFlags = info.flags; 5036 } 5037 setBufferInfos(ArrayDeque<BufferInfo> infos)5038 void setBufferInfos(ArrayDeque<BufferInfo> infos) { 5039 mBufferInfos.clear(); 5040 mBufferInfos.addAll(infos); 5041 } 5042 isLoaded()5043 boolean isLoaded() { 5044 return mLoaded; 5045 } 5046 setLoaded(boolean loaded)5047 void setLoaded(boolean loaded) { 5048 mLoaded = loaded; 5049 } 5050 5051 private final int mIndex; 5052 private LinearBlock mLinearBlock = null; 5053 private HardwareBuffer mHardwareBuffer = null; 5054 private long mPresentationTimeUs = 0; 5055 private @BufferFlag int mFlags = 0; 5056 private MediaFormat mFormat = null; 5057 private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>(); 5058 private final ArrayList<String> mChangedKeys = new ArrayList<>(); 5059 private final Set<String> mKeySet = new HashSet<>(); 5060 private boolean mAccessible = false; 5061 private boolean mLoaded = false; 5062 } 5063 5064 private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); 5065 5066 /** 5067 * Returns an {@link OutputFrame} object. 5068 * 5069 * @param index output buffer index from 5070 * {@link Callback#onOutputBufferAvailable} 5071 * @return {@link OutputFrame} object describing the output buffer 5072 * @throws IllegalStateException if not using block model 5073 * @throws IllegalArgumentException if the output buffer is not available or 5074 * the index is out of range 5075 */ getOutputFrame(int index)5076 public @NonNull OutputFrame getOutputFrame(int index) { 5077 synchronized (mBufferLock) { 5078 if (mBufferMode != BUFFER_MODE_BLOCK) { 5079 throw new IllegalStateException("The codec is not configured for block model"); 5080 } 5081 if (index < 0 || index >= mOutputFrames.size()) { 5082 throw new IndexOutOfBoundsException("Expected range of index: [0," 5083 + (mQueueRequests.size() - 1) + "]; actual: " + index); 5084 } 5085 OutputFrame frame = mOutputFrames.get(index); 5086 if (frame == null) { 5087 throw new IllegalArgumentException("Unavailable index: " + index); 5088 } 5089 if (!frame.isAccessible()) { 5090 throw new IllegalArgumentException( 5091 "The output frame is stale at index " + index); 5092 } 5093 if (!frame.isLoaded()) { 5094 native_getOutputFrame(frame, index); 5095 frame.setLoaded(true); 5096 } 5097 return frame; 5098 } 5099 } 5100 native_getOutputFrame(OutputFrame frame, int index)5101 private native void native_getOutputFrame(OutputFrame frame, int index); 5102 5103 /** 5104 * The content is scaled to the surface dimensions 5105 */ 5106 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 5107 5108 /** 5109 * The content is scaled, maintaining its aspect ratio, the whole 5110 * surface area is used, content may be cropped. 5111 * <p class=note> 5112 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 5113 * configure the pixel aspect ratio for a {@link Surface}. 5114 * <p class=note> 5115 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 5116 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 5117 */ 5118 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 5119 5120 /** @hide */ 5121 @IntDef({ 5122 VIDEO_SCALING_MODE_SCALE_TO_FIT, 5123 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 5124 }) 5125 @Retention(RetentionPolicy.SOURCE) 5126 public @interface VideoScalingMode {} 5127 5128 /** 5129 * If a surface has been specified in a previous call to {@link #configure} 5130 * specifies the scaling mode to use. The default is "scale to fit". 5131 * <p class=note> 5132 * The scaling mode may be reset to the <strong>default</strong> each time an 5133 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 5134 * must call this method after every buffer change event (and before the first output buffer is 5135 * released for rendering) to ensure consistent scaling mode. 5136 * <p class=note> 5137 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 5138 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 5139 * 5140 * @throws IllegalArgumentException if mode is not recognized. 5141 * @throws IllegalStateException if in the Released state. 5142 */ setVideoScalingMode(@ideoScalingMode int mode)5143 public native final void setVideoScalingMode(@VideoScalingMode int mode); 5144 5145 /** 5146 * Sets the audio presentation. 5147 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 5148 */ setAudioPresentation(@onNull AudioPresentation presentation)5149 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 5150 if (presentation == null) { 5151 throw new NullPointerException("audio presentation is null"); 5152 } 5153 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 5154 } 5155 native_setAudioPresentation(int presentationId, int programId)5156 private native void native_setAudioPresentation(int presentationId, int programId); 5157 5158 /** 5159 * Retrieve the codec name. 5160 * 5161 * If the codec was created by createDecoderByType or createEncoderByType, what component is 5162 * chosen is not known beforehand. This method returns the name of the codec that was 5163 * selected by the platform. 5164 * 5165 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 5166 * names) for the same underlying codec, any of which can be used to instantiate the same 5167 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 5168 * name used to create the codec in this case. 5169 * 5170 * @throws IllegalStateException if in the Released state. 5171 */ 5172 @NonNull getName()5173 public final String getName() { 5174 // get canonical name to handle exception 5175 String canonicalName = getCanonicalName(); 5176 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 5177 } 5178 5179 /** 5180 * Retrieve the underlying codec name. 5181 * 5182 * This method is similar to {@link #getName}, except that it returns the underlying component 5183 * name even if an alias was used to create this MediaCodec object by name, 5184 * 5185 * @throws IllegalStateException if in the Released state. 5186 */ 5187 @NonNull getCanonicalName()5188 public native final String getCanonicalName(); 5189 5190 /** 5191 * Return Metrics data about the current codec instance. 5192 * <p> 5193 * Call this method after configuration, during execution, or after 5194 * the codec has been already stopped. 5195 * <p> 5196 * Beginning with {@link android.os.Build.VERSION_CODES#B} 5197 * this method can be used to get the Metrics data prior to an error. 5198 * (e.g. in {@link Callback#onError} or after a method throws 5199 * {@link MediaCodec.CodecException}.) Before that, the Metrics data was 5200 * cleared on error, resulting in a null return value. 5201 * 5202 * @return a {@link PersistableBundle} containing the set of attributes and values 5203 * available for the media being handled by this instance of MediaCodec 5204 * The attributes are descibed in {@link MetricsConstants}. 5205 * 5206 * Additional vendor-specific fields may also be present in 5207 * the return value. Returns null if there is no Metrics data. 5208 * 5209 */ getMetrics()5210 public PersistableBundle getMetrics() { 5211 PersistableBundle bundle = native_getMetrics(); 5212 return bundle; 5213 } 5214 native_getMetrics()5215 private native PersistableBundle native_getMetrics(); 5216 5217 /** 5218 * Change a video encoder's target bitrate on the fly. The value is an 5219 * Integer object containing the new bitrate in bps. 5220 * 5221 * @see #setParameters(Bundle) 5222 */ 5223 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 5224 5225 /** 5226 * Temporarily suspend/resume encoding of input data. While suspended 5227 * input data is effectively discarded instead of being fed into the 5228 * encoder. This parameter really only makes sense to use with an encoder 5229 * in "surface-input" mode, as the client code has no control over the 5230 * input-side of the encoder in that case. 5231 * The value is an Integer object containing the value 1 to suspend 5232 * or the value 0 to resume. 5233 * 5234 * @see #setParameters(Bundle) 5235 */ 5236 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 5237 5238 /** 5239 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 5240 * optionally use this key to specify the timestamp (in micro-second) 5241 * at which the suspend/resume operation takes effect. 5242 * 5243 * Note that the specified timestamp must be greater than or equal to the 5244 * timestamp of any previously queued suspend/resume operations. 5245 * 5246 * The value is a long int, indicating the timestamp to suspend/resume. 5247 * 5248 * @see #setParameters(Bundle) 5249 */ 5250 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 5251 5252 /** 5253 * Specify an offset (in micro-second) to be added on top of the timestamps 5254 * onward. A typical use case is to apply an adjust to the timestamps after 5255 * a period of pause by the user. 5256 * 5257 * This parameter can only be used on an encoder in "surface-input" mode. 5258 * 5259 * The value is a long int, indicating the timestamp offset to be applied. 5260 * 5261 * @see #setParameters(Bundle) 5262 */ 5263 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 5264 5265 /** 5266 * Request that the encoder produce a sync frame "soon". 5267 * Provide an Integer with the value 0. 5268 * 5269 * @see #setParameters(Bundle) 5270 */ 5271 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 5272 5273 /** 5274 * Set the HDR10+ metadata on the next queued input frame. 5275 * 5276 * Provide a byte array of data that's conforming to the 5277 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 5278 *<p> 5279 * For decoders: 5280 *<p> 5281 * When a decoder is configured for one of the HDR10+ profiles that uses 5282 * out-of-band metadata (such as {@link 5283 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 5284 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 5285 * parameter sets the HDR10+ metadata on the next input buffer queued 5286 * to the decoder. A decoder supporting these profiles must propagate 5287 * the metadata to the format of the output buffer corresponding to this 5288 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 5289 * The metadata should be applied to that output buffer and the buffers 5290 * following it (in display order), until the next output buffer (in 5291 * display order) upon which an HDR10+ metadata is set. 5292 *<p> 5293 * This parameter shouldn't be set if the decoder is not configured for 5294 * an HDR10+ profile that uses out-of-band metadata. In particular, 5295 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 5296 * where the metadata is embedded in the input buffers, for example 5297 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 5298 *<p> 5299 * For encoders: 5300 *<p> 5301 * When an encoder is configured for one of the HDR10+ profiles and the 5302 * operates in byte buffer input mode (instead of surface input mode), 5303 * this parameter sets the HDR10+ metadata on the next input buffer queued 5304 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 5305 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 5306 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 5307 * the metadata must be propagated to the format of the output buffer 5308 * corresponding to this particular input buffer (under key {@link 5309 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 5310 * in-band metadata (such as {@link 5311 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 5312 * metadata info must be embedded in the corresponding output buffer itself. 5313 *<p> 5314 * This parameter shouldn't be set if the encoder is not configured for 5315 * an HDR10+ profile, or if it's operating in surface input mode. 5316 *<p> 5317 * 5318 * @see MediaFormat#KEY_HDR10_PLUS_INFO 5319 */ 5320 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 5321 5322 /** 5323 * Enable/disable low latency decoding mode. 5324 * When enabled, the decoder doesn't hold input and output data more than 5325 * required by the codec standards. 5326 * The value is an Integer object containing the value 1 to enable 5327 * or the value 0 to disable. 5328 * 5329 * @see #setParameters(Bundle) 5330 * @see MediaFormat#KEY_LOW_LATENCY 5331 */ 5332 public static final String PARAMETER_KEY_LOW_LATENCY = 5333 MediaFormat.KEY_LOW_LATENCY; 5334 5335 /** 5336 * Control video peek of the first frame when a codec is configured for tunnel mode with 5337 * {@link MediaFormat#KEY_AUDIO_SESSION_ID} while the {@link AudioTrack} is paused. 5338 *<p> 5339 * When disabled (1) after a {@link #flush} or {@link #start}, (2) while the corresponding 5340 * {@link AudioTrack} is paused and (3) before any buffers are queued, the first frame is not to 5341 * be rendered until either this parameter is enabled or the corresponding {@link AudioTrack} 5342 * has begun playback. Once the frame is decoded and ready to be rendered, 5343 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called but the frame is 5344 * not rendered. The surface continues to show the previously-rendered content, or black if the 5345 * surface is new. A subsequent call to {@link AudioTrack#play} renders this frame and triggers 5346 * a callback to {@link OnFrameRenderedListener#onFrameRendered}, and video playback begins. 5347 *<p> 5348 * <b>Note</b>: To clear any previously rendered content and show black, configure the 5349 * MediaCodec with {@code KEY_PUSH_BLANK_BUFFERS_ON_STOP(1)}, and call {@link #stop} before 5350 * pushing new video frames to the codec. 5351 *<p> 5352 * When enabled (1) after a {@link #flush} or {@link #start} and (2) while the corresponding 5353 * {@link AudioTrack} is paused, the first frame is rendered as soon as it is decoded, or 5354 * immediately, if it has already been decoded. If not already decoded, when the frame is 5355 * decoded and ready to be rendered, 5356 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called. The frame is then 5357 * immediately rendered and {@link OnFrameRenderedListener#onFrameRendered} is subsequently 5358 * called. 5359 *<p> 5360 * The value is an Integer object containing the value 1 to enable or the value 0 to disable. 5361 *<p> 5362 * The default for this parameter is <b>enabled</b>. Once a frame has been rendered, changing 5363 * this parameter has no effect until a subsequent {@link #flush} or 5364 * {@link #stop}/{@link #start}. 5365 * 5366 * @see #setParameters(Bundle) 5367 */ 5368 public static final String PARAMETER_KEY_TUNNEL_PEEK = "tunnel-peek"; 5369 5370 /** 5371 * Set the region of interest as QpOffset-Map on the next queued input frame. 5372 * <p> 5373 * The associated value is a byte array containing quantization parameter (QP) offsets in 5374 * raster scan order for the entire frame at 16x16 granularity. The size of the byte array 5375 * shall be ((frame_width + 15) / 16) * ((frame_height + 15) / 16), where frame_width and 5376 * frame_height correspond to width and height configured using {@link MediaFormat#KEY_WIDTH} 5377 * and {@link MediaFormat#KEY_HEIGHT} keys respectively. During encoding, if the coding unit 5378 * size is larger than 16x16, then the qpOffset information of all 16x16 blocks that 5379 * encompass the coding unit is combined and used. The QP of target block will be calculated 5380 * as 'frameQP + offsetQP'. If the result exceeds minQP or maxQP configured then the value 5381 * will be clamped. Negative offset results in blocks encoded at lower QP than frame QP and 5382 * positive offsets will result in encoding blocks at higher QP than frame QP. If the areas 5383 * of negative QP and positive QP are chosen wisely, the overall viewing experience can be 5384 * improved. 5385 * <p> 5386 * If byte array size is smaller than the expected size, components will ignore the 5387 * configuration and print an error message. If the byte array exceeds the expected size, 5388 * components will use the initial portion and ignore the rest. 5389 * <p> 5390 * The scope of this key is throughout the encoding session until it is reconfigured during 5391 * running state. 5392 * <p> 5393 * @see #setParameters(Bundle) 5394 */ 5395 @FlaggedApi(FLAG_REGION_OF_INTEREST) 5396 public static final String PARAMETER_KEY_QP_OFFSET_MAP = "qp-offset-map"; 5397 5398 /** 5399 * Set the region of interest as QpOffset-Rects on the next queued input frame. 5400 * <p> 5401 * The associated value is a String in the format "Top1,Left1-Bottom1,Right1=Offset1;Top2, 5402 * Left2-Bottom2,Right2=Offset2;...". If the configuration doesn't follow this pattern, 5403 * it will be ignored. Co-ordinates (Top, Left), (Top, Right), (Bottom, Left) 5404 * and (Bottom, Right) form the vertices of bounding box of region of interest in pixels. 5405 * Pixel (0, 0) points to the top-left corner of the frame. Offset is the suggested 5406 * quantization parameter (QP) offset of the blocks in the bounding box. The bounding box 5407 * will get stretched outwards to align to LCU boundaries during encoding. The Qp Offset is 5408 * integral and shall be in the range [-128, 127]. The QP of target block will be calculated 5409 * as frameQP + offsetQP. If the result exceeds minQP or maxQP configured then the value will 5410 * be clamped. Negative offset results in blocks encoded at lower QP than frame QP and 5411 * positive offsets will result in blocks encoded at higher QP than frame QP. If the areas of 5412 * negative QP and positive QP are chosen wisely, the overall viewing experience can be 5413 * improved. 5414 * <p> 5415 * If roi (region of interest) rect is outside the frame boundaries, that is, left < 0 or 5416 * top < 0 or right > width or bottom > height, then rect shall be clamped to the frame 5417 * boundaries. If roi rect is not valid, that is left > right or top > bottom, then the 5418 * parameter setting is ignored. 5419 * <p> 5420 * The scope of this key is throughout the encoding session until it is reconfigured during 5421 * running state. 5422 * <p> 5423 * The maximum number of contours (rectangles) that can be specified for a given input frame 5424 * is device specific. Implementations will drop/ignore the rectangles that are beyond their 5425 * supported limit. Hence it is preferable to place the rects in descending order of 5426 * importance. Transitively, if the bounding boxes overlap, then the most preferred 5427 * rectangle's qp offset (earlier rectangle qp offset) will be used to quantize the block. 5428 * <p> 5429 * @see #setParameters(Bundle) 5430 */ 5431 @FlaggedApi(FLAG_REGION_OF_INTEREST) 5432 public static final String PARAMETER_KEY_QP_OFFSET_RECTS = "qp-offset-rects"; 5433 5434 /** 5435 * Communicate additional parameter changes to the component instance. 5436 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 5437 * 5438 * @param params The bundle of parameters to set. 5439 * @throws IllegalStateException if in the Released state. 5440 */ 5441 5442 private static final String PARAMETER_KEY_PICTURE_PROFILE_HANDLE = "picture-profile-handle"; 5443 setParameters(@ullable Bundle params)5444 public final void setParameters(@Nullable Bundle params) { 5445 if (params == null) { 5446 return; 5447 } 5448 5449 String[] keys = new String[params.size()]; 5450 Object[] values = new Object[params.size()]; 5451 5452 int i = 0; 5453 for (final String key: params.keySet()) { 5454 if (key.equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 5455 int sessionId = 0; 5456 try { 5457 sessionId = (Integer) params.get(key); 5458 } catch (Exception e) { 5459 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 5460 } 5461 keys[i] = "audio-hw-sync"; 5462 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 5463 } else if (applyPictureProfiles() && mediaQualityFw() 5464 && key.equals(MediaFormat.KEY_PICTURE_PROFILE_INSTANCE)) { 5465 PictureProfile pictureProfile = null; 5466 try { 5467 pictureProfile = (PictureProfile) params.get(key); 5468 } catch (ClassCastException e) { 5469 throw new IllegalArgumentException( 5470 "Cannot cast the instance parameter to PictureProfile!"); 5471 } catch (Exception e) { 5472 Log.e(TAG, Log.getStackTraceString(e)); 5473 throw new IllegalArgumentException("Unexpected exception when casting the " 5474 + "instance parameter to PictureProfile!"); 5475 } 5476 if (pictureProfile == null) { 5477 throw new IllegalArgumentException( 5478 "Picture profile instance parameter is null!"); 5479 } 5480 PictureProfileHandle handle = pictureProfile.getHandle(); 5481 if (handle != PictureProfileHandle.NONE) { 5482 keys[i] = PARAMETER_KEY_PICTURE_PROFILE_HANDLE; 5483 values[i] = Long.valueOf(handle.getId()); 5484 } 5485 } else if (applyPictureProfiles() && mediaQualityFw() 5486 && key.equals(MediaFormat.KEY_PICTURE_PROFILE_ID)) { 5487 String pictureProfileId = null; 5488 try { 5489 pictureProfileId = (String) params.get(key); 5490 } catch (ClassCastException e) { 5491 throw new IllegalArgumentException( 5492 "Cannot cast the KEY_PICTURE_PROFILE_ID parameter to String!"); 5493 } catch (Exception e) { 5494 Log.e(TAG, Log.getStackTraceString(e)); 5495 throw new IllegalArgumentException("Unexpected exception when casting the " 5496 + "KEY_PICTURE_PROFILE_ID parameter!"); 5497 } 5498 if (pictureProfileId == null) { 5499 throw new IllegalArgumentException("KEY_PICTURE_PROFILE_ID parameter is null!"); 5500 } 5501 if (!pictureProfileId.isEmpty()) { 5502 keys[i] = MediaFormat.KEY_PICTURE_PROFILE_ID; 5503 values[i] = pictureProfileId; 5504 } 5505 } else { 5506 keys[i] = key; 5507 Object value = params.get(key); 5508 5509 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 5510 if (value instanceof byte[]) { 5511 values[i] = ByteBuffer.wrap((byte[]) value); 5512 } else { 5513 values[i] = value; 5514 } 5515 } 5516 ++i; 5517 } 5518 5519 setParameters(keys, values); 5520 } 5521 logAndRun(String message, Runnable r)5522 private void logAndRun(String message, Runnable r) { 5523 Log.d(TAG, "enter: " + message); 5524 r.run(); 5525 Log.d(TAG, "exit : " + message); 5526 } 5527 5528 /** 5529 * Sets an asynchronous callback for actionable MediaCodec events. 5530 * 5531 * If the client intends to use the component in asynchronous mode, 5532 * a valid callback should be provided before {@link #configure} is called. 5533 * 5534 * When asynchronous callback is enabled, the client should not call 5535 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 5536 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 5537 * <p> 5538 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 5539 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 5540 * even if an input surface was created. 5541 * 5542 * @param cb The callback that will run. Use {@code null} to clear a previously 5543 * set callback (before {@link #configure configure} is called and run 5544 * in synchronous mode). 5545 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 5546 * callbacks are done on the default thread (the caller's thread or the 5547 * main thread.) 5548 */ setCallback(@ullable Callback cb, @Nullable Handler handler)5549 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 5550 boolean setCallbackStallFlag = 5551 GetFlag(() -> android.media.codec.Flags.setCallbackStall()); 5552 if (cb != null) { 5553 synchronized (mListenerLock) { 5554 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 5555 // NOTE: there are no callbacks on the handler at this time, but check anyways 5556 // even if we were to extend this to be callable dynamically, it must 5557 // be called when codec is flushed, so no messages are pending. 5558 if (newHandler != mCallbackHandler) { 5559 if (setCallbackStallFlag) { 5560 logAndRun( 5561 "[new handler] removeMessages(SET_CALLBACK)", 5562 () -> { 5563 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5564 }); 5565 logAndRun( 5566 "[new handler] removeMessages(CALLBACK)", 5567 () -> { 5568 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5569 }); 5570 } else { 5571 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5572 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5573 } 5574 mCallbackHandler = newHandler; 5575 } 5576 } 5577 } else if (mCallbackHandler != null) { 5578 if (setCallbackStallFlag) { 5579 logAndRun( 5580 "[null handler] removeMessages(SET_CALLBACK)", 5581 () -> { 5582 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5583 }); 5584 logAndRun( 5585 "[null handler] removeMessages(CALLBACK)", 5586 () -> { 5587 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5588 }); 5589 } else { 5590 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5591 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5592 } 5593 } 5594 5595 if (mCallbackHandler != null) { 5596 // set java callback on main handler 5597 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 5598 mCallbackHandler.sendMessage(msg); 5599 5600 // set native handler here, don't post to handler because 5601 // it may cause the callback to be delayed and set in a wrong state. 5602 // Note that native codec may start sending events to the callback 5603 // handler after this returns. 5604 native_setCallback(cb); 5605 } 5606 } 5607 5608 /** 5609 * Sets an asynchronous callback for actionable MediaCodec events on the default 5610 * looper. 5611 * <p> 5612 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 5613 * @param cb The callback that will run. Use {@code null} to clear a previously 5614 * set callback (before {@link #configure configure} is called and run 5615 * in synchronous mode). 5616 * @see #setCallback(Callback, Handler) 5617 */ setCallback(@ullable Callback cb)5618 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 5619 setCallback(cb, null /* handler */); 5620 } 5621 5622 /** 5623 * Listener to be called when the first output frame has been decoded 5624 * and is ready to be rendered for a codec configured for tunnel mode with 5625 * {@code KEY_AUDIO_SESSION_ID}. 5626 * 5627 * @see MediaCodec#setOnFirstTunnelFrameReadyListener 5628 */ 5629 public interface OnFirstTunnelFrameReadyListener { 5630 5631 /** 5632 * Called when the first output frame has been decoded and is ready to be 5633 * rendered. 5634 */ onFirstTunnelFrameReady(@onNull MediaCodec codec)5635 void onFirstTunnelFrameReady(@NonNull MediaCodec codec); 5636 } 5637 5638 /** 5639 * Registers a callback to be invoked when the first output frame has been decoded 5640 * and is ready to be rendered on a codec configured for tunnel mode with {@code 5641 * KEY_AUDIO_SESSION_ID}. 5642 * 5643 * @param handler the callback will be run on the handler's thread. If {@code 5644 * null}, the callback will be run on the default thread, which is the looper from 5645 * which the codec was created, or a new thread if there was none. 5646 * 5647 * @param listener the callback that will be run. If {@code null}, clears any registered 5648 * listener. 5649 */ setOnFirstTunnelFrameReadyListener( @ullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener)5650 public void setOnFirstTunnelFrameReadyListener( 5651 @Nullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener) { 5652 synchronized (mListenerLock) { 5653 mOnFirstTunnelFrameReadyListener = listener; 5654 if (listener != null) { 5655 EventHandler newHandler = getEventHandlerOn( 5656 handler, 5657 mOnFirstTunnelFrameReadyHandler); 5658 if (newHandler != mOnFirstTunnelFrameReadyHandler) { 5659 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 5660 } 5661 mOnFirstTunnelFrameReadyHandler = newHandler; 5662 } else if (mOnFirstTunnelFrameReadyHandler != null) { 5663 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 5664 } 5665 native_enableOnFirstTunnelFrameReadyListener(listener != null); 5666 } 5667 } 5668 native_enableOnFirstTunnelFrameReadyListener(boolean enable)5669 private native void native_enableOnFirstTunnelFrameReadyListener(boolean enable); 5670 5671 /** 5672 * Listener to be called when an output frame has rendered on the output surface 5673 * 5674 * @see MediaCodec#setOnFrameRenderedListener 5675 */ 5676 public interface OnFrameRenderedListener { 5677 5678 /** 5679 * Called when an output frame has rendered on the output surface. 5680 * <p> 5681 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 5682 * render timing samples, and can be significantly delayed and batched. Starting with 5683 * Android {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE}, a callback will always 5684 * be received for each rendered frame providing the MediaCodec is still in the executing 5685 * state when the callback is dispatched. Prior to Android 5686 * {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE}, some frames may have been 5687 * rendered even if there was no callback generated. 5688 * 5689 * @param codec the MediaCodec instance 5690 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 5691 * This is usually the same as specified in {@link #queueInputBuffer}; however, 5692 * some codecs may alter the media time by applying some time-based transformation, 5693 * such as frame rate conversion. In that case, presentation time corresponds 5694 * to the actual output frame rendered. 5695 * @param nanoTime The system time when the frame was rendered. 5696 * 5697 * @see System#nanoTime 5698 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)5699 public void onFrameRendered( 5700 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 5701 } 5702 5703 /** 5704 * Registers a callback to be invoked when an output frame is rendered on the output surface. 5705 * <p> 5706 * This method can be called in any codec state, but will only have an effect in the 5707 * Executing state for codecs that render buffers to the output surface. 5708 * <p> 5709 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 5710 * render timing samples, and can be significantly delayed and batched. Some frames may have 5711 * been rendered even if there was no callback generated. 5712 * 5713 * @param listener the callback that will be run 5714 * @param handler the callback will be run on the handler's thread. If {@code null}, 5715 * the callback will be run on the default thread, which is the looper 5716 * from which the codec was created, or a new thread if there was none. 5717 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)5718 public void setOnFrameRenderedListener( 5719 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 5720 synchronized (mListenerLock) { 5721 mOnFrameRenderedListener = listener; 5722 if (listener != null) { 5723 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 5724 if (newHandler != mOnFrameRenderedHandler) { 5725 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 5726 } 5727 mOnFrameRenderedHandler = newHandler; 5728 } else if (mOnFrameRenderedHandler != null) { 5729 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 5730 } 5731 native_enableOnFrameRenderedListener(listener != null); 5732 } 5733 } 5734 native_enableOnFrameRenderedListener(boolean enable)5735 private native void native_enableOnFrameRenderedListener(boolean enable); 5736 5737 /** 5738 * Returns a list of vendor parameter names. 5739 * <p> 5740 * This method can be called in any codec state except for released state. 5741 * 5742 * @return a list containing supported vendor parameters; an empty 5743 * list if no vendor parameters are supported. The order of the 5744 * parameters is arbitrary. 5745 * @throws IllegalStateException if in the Released state. 5746 */ 5747 @NonNull getSupportedVendorParameters()5748 public List<String> getSupportedVendorParameters() { 5749 return native_getSupportedVendorParameters(); 5750 } 5751 5752 @NonNull native_getSupportedVendorParameters()5753 private native List<String> native_getSupportedVendorParameters(); 5754 5755 /** 5756 * Contains description of a parameter. 5757 */ 5758 public static class ParameterDescriptor { ParameterDescriptor()5759 private ParameterDescriptor() {} 5760 5761 /** 5762 * Returns the name of the parameter. 5763 */ 5764 @NonNull getName()5765 public String getName() { 5766 return mName; 5767 } 5768 5769 /** 5770 * Returns the type of the parameter. 5771 * {@link MediaFormat#TYPE_NULL} is never returned. 5772 */ 5773 @MediaFormat.Type getType()5774 public int getType() { 5775 return mType; 5776 } 5777 5778 @Override equals(Object o)5779 public boolean equals(Object o) { 5780 if (o == null) { 5781 return false; 5782 } 5783 if (!(o instanceof ParameterDescriptor)) { 5784 return false; 5785 } 5786 ParameterDescriptor other = (ParameterDescriptor) o; 5787 return this.mName.equals(other.mName) && this.mType == other.mType; 5788 } 5789 5790 @Override hashCode()5791 public int hashCode() { 5792 return Arrays.asList( 5793 (Object) mName, 5794 (Object) Integer.valueOf(mType)).hashCode(); 5795 } 5796 5797 private String mName; 5798 private @MediaFormat.Type int mType; 5799 } 5800 5801 /** 5802 * Describe a parameter with the name. 5803 * <p> 5804 * This method can be called in any codec state except for released state. 5805 * 5806 * @param name name of the parameter to describe, typically one from 5807 * {@link #getSupportedVendorParameters}. 5808 * @return {@link ParameterDescriptor} object that describes the parameter. 5809 * {@code null} if unrecognized / not able to describe. 5810 * @throws IllegalStateException if in the Released state. 5811 */ 5812 @Nullable getParameterDescriptor(@onNull String name)5813 public ParameterDescriptor getParameterDescriptor(@NonNull String name) { 5814 return native_getParameterDescriptor(name); 5815 } 5816 5817 @Nullable native_getParameterDescriptor(@onNull String name)5818 private native ParameterDescriptor native_getParameterDescriptor(@NonNull String name); 5819 5820 /** 5821 * Subscribe to vendor parameters, so that these parameters will be present in 5822 * {@link #getOutputFormat} and changes to these parameters generate 5823 * output format change event. 5824 * <p> 5825 * Unrecognized parameter names or standard (non-vendor) parameter names will be ignored. 5826 * {@link #reset} also resets the list of subscribed parameters. 5827 * If a parameter in {@code names} is already subscribed, it will remain subscribed. 5828 * <p> 5829 * This method can be called in any codec state except for released state. When called in 5830 * running state with newly subscribed parameters, it takes effect no later than the 5831 * processing of the subsequently queued buffer. For the new parameters, the codec will generate 5832 * output format change event. 5833 * <p> 5834 * Note that any vendor parameters set in a {@link #configure} or 5835 * {@link #setParameters} call are automatically subscribed. 5836 * <p> 5837 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 5838 * for output format change events. 5839 * 5840 * @param names names of the vendor parameters to subscribe. This may be an empty list, 5841 * and in that case this method will not change the list of subscribed parameters. 5842 * @throws IllegalStateException if in the Released state. 5843 */ subscribeToVendorParameters(@onNull List<String> names)5844 public void subscribeToVendorParameters(@NonNull List<String> names) { 5845 native_subscribeToVendorParameters(names); 5846 } 5847 native_subscribeToVendorParameters(@onNull List<String> names)5848 private native void native_subscribeToVendorParameters(@NonNull List<String> names); 5849 5850 /** 5851 * Unsubscribe from vendor parameters, so that these parameters will not be present in 5852 * {@link #getOutputFormat} and changes to these parameters no longer generate 5853 * output format change event. 5854 * <p> 5855 * Unrecognized parameter names, standard (non-vendor) parameter names will be ignored. 5856 * {@link #reset} also resets the list of subscribed parameters. 5857 * If a parameter in {@code names} is already unsubscribed, it will remain unsubscribed. 5858 * <p> 5859 * This method can be called in any codec state except for released state. When called in 5860 * running state with newly unsubscribed parameters, it takes effect no later than the 5861 * processing of the subsequently queued buffer. For the removed parameters, the codec will 5862 * generate output format change event. 5863 * <p> 5864 * Note that any vendor parameters set in a {@link #configure} or 5865 * {@link #setParameters} call are automatically subscribed, and with this method 5866 * they can be unsubscribed. 5867 * <p> 5868 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 5869 * for output format change events. 5870 * 5871 * @param names names of the vendor parameters to unsubscribe. This may be an empty list, 5872 * and in that case this method will not change the list of subscribed parameters. 5873 * @throws IllegalStateException if in the Released state. 5874 */ unsubscribeFromVendorParameters(@onNull List<String> names)5875 public void unsubscribeFromVendorParameters(@NonNull List<String> names) { 5876 native_unsubscribeFromVendorParameters(names); 5877 } 5878 native_unsubscribeFromVendorParameters(@onNull List<String> names)5879 private native void native_unsubscribeFromVendorParameters(@NonNull List<String> names); 5880 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)5881 private EventHandler getEventHandlerOn( 5882 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 5883 if (handler == null) { 5884 return mEventHandler; 5885 } else { 5886 Looper looper = handler.getLooper(); 5887 if (lastHandler.getLooper() == looper) { 5888 return lastHandler; 5889 } else { 5890 return new EventHandler(this, looper); 5891 } 5892 } 5893 } 5894 5895 /** 5896 * MediaCodec callback interface. Used to notify the user asynchronously 5897 * of various MediaCodec events. 5898 */ 5899 public static abstract class Callback { 5900 /** 5901 * Called when an input buffer becomes available. 5902 * 5903 * @param codec The MediaCodec object. 5904 * @param index The index of the available input buffer. 5905 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)5906 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 5907 5908 /** 5909 * Called when an output buffer becomes available. 5910 * 5911 * @param codec The MediaCodec object. 5912 * @param index The index of the available output buffer. 5913 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 5914 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)5915 public abstract void onOutputBufferAvailable( 5916 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 5917 5918 /** 5919 * Called when multiple access-units are available in the output. 5920 * 5921 * @param codec The MediaCodec object. 5922 * @param index The index of the available output buffer. 5923 * @param infos Infos describing the available output buffer {@link MediaCodec.BufferInfo}. 5924 * Access units present in the output buffer are laid out contiguously 5925 * without gaps and in order. 5926 */ 5927 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) onOutputBuffersAvailable( @onNull MediaCodec codec, int index, @NonNull ArrayDeque<BufferInfo> infos)5928 public void onOutputBuffersAvailable( 5929 @NonNull MediaCodec codec, int index, @NonNull ArrayDeque<BufferInfo> infos) { 5930 /* 5931 * This callback returns multiple BufferInfos when codecs are configured to operate on 5932 * large audio frame. Since at this point, we have a single large buffer, returning 5933 * each BufferInfo using 5934 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} may cause the 5935 * index to be released to the codec using {@link MediaCodec#releaseOutputBuffer} 5936 * before all BuffersInfos can be returned to the client. 5937 * Hence this callback is required to be implemented or else an exception is thrown. 5938 */ 5939 throw new IllegalStateException( 5940 "Client must override onOutputBuffersAvailable when codec is " + 5941 "configured to operate with multiple access units"); 5942 } 5943 5944 /** 5945 * Called when the MediaCodec encountered an error 5946 * 5947 * @param codec The MediaCodec object. 5948 * @param e The {@link MediaCodec.CodecException} object describing the error. 5949 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)5950 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 5951 5952 /** 5953 * Called only when MediaCodec encountered a crypto(decryption) error when using 5954 * a decoder configured with CONFIGURE_FLAG_USE_CRYPTO_ASYNC flag along with crypto 5955 * or descrambler object. 5956 * 5957 * @param codec The MediaCodec object 5958 * @param e The {@link MediaCodec.CryptoException} object with error details. 5959 */ onCryptoError(@onNull MediaCodec codec, @NonNull CryptoException e)5960 public void onCryptoError(@NonNull MediaCodec codec, @NonNull CryptoException e) { 5961 /* 5962 * A default implementation for backward compatibility. 5963 * Use of CONFIGURE_FLAG_USE_CRYPTO_ASYNC requires override of this callback 5964 * to receive CrytoInfo. Without an orverride an exception is thrown. 5965 */ 5966 throw new IllegalStateException( 5967 "Client must override onCryptoError when the codec is " + 5968 "configured with CONFIGURE_FLAG_USE_CRYPTO_ASYNC.", e); 5969 } 5970 5971 /** 5972 * Called when the output format has changed 5973 * 5974 * @param codec The MediaCodec object. 5975 * @param format The new output format. 5976 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)5977 public abstract void onOutputFormatChanged( 5978 @NonNull MediaCodec codec, @NonNull MediaFormat format); 5979 5980 /** 5981 * Called when the metrics for this codec have been flushed "mid-stream" 5982 * due to the start of a new subsession during execution. 5983 * <p> 5984 * A new codec subsession normally starts when the codec is reconfigured 5985 * after stop(), but it can also happen mid-stream e.g. if the video size 5986 * changes. When this happens, the metrics for the previous subsession 5987 * are flushed, and {@link MediaCodec#getMetrics} will return the metrics 5988 * for the new subsession. 5989 * <p> 5990 * For subsessions that begin due to a reconfiguration, the metrics for 5991 * the prior subsession can be retrieved via {@link MediaCodec#getMetrics} 5992 * prior to calling {@link #configure}. 5993 * <p> 5994 * When a new subsession begins "mid-stream", the metrics for the prior 5995 * subsession are flushed just before the {@link Callback#onOutputFormatChanged} 5996 * event, so this <b>optional</b> callback is provided to be able to 5997 * capture the final metrics for the previous subsession. 5998 * 5999 * @param codec The MediaCodec object. 6000 * @param metrics The flushed metrics for this codec. This is a 6001 * {@link PersistableBundle} containing the set of 6002 * attributes and values available for the media being 6003 * handled by this instance of MediaCodec. The attributes 6004 * are described in {@link MetricsConstants}. Additional 6005 * vendor-specific fields may also be present. 6006 */ 6007 @FlaggedApi(FLAG_SUBSESSION_METRICS) onMetricsFlushed( @onNull MediaCodec codec, @NonNull PersistableBundle metrics)6008 public void onMetricsFlushed( 6009 @NonNull MediaCodec codec, @NonNull PersistableBundle metrics) { 6010 // default implementation ignores this callback. 6011 } 6012 6013 /** 6014 * @hide 6015 * Called when there is a change in the required resources for the codec. 6016 * <p> 6017 * Upon receiving this notification, the updated resource requirement 6018 * can be queried through {@link #getRequiredResources}. 6019 * 6020 * @param codec The MediaCodec object. 6021 */ 6022 @FlaggedApi(FLAG_CODEC_AVAILABILITY) 6023 @TestApi onRequiredResourcesChanged(@onNull MediaCodec codec)6024 public void onRequiredResourcesChanged(@NonNull MediaCodec codec) { 6025 /* 6026 * A default implementation for backward compatibility. 6027 * Since this is a TestApi, we are not enforcing the callback to be 6028 * overridden. 6029 */ 6030 } 6031 } 6032 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)6033 private void postEventFromNative( 6034 int what, int arg1, int arg2, @Nullable Object obj) { 6035 synchronized (mListenerLock) { 6036 EventHandler handler = mEventHandler; 6037 if (what == EVENT_CALLBACK) { 6038 handler = mCallbackHandler; 6039 } else if (what == EVENT_FIRST_TUNNEL_FRAME_READY) { 6040 handler = mOnFirstTunnelFrameReadyHandler; 6041 } else if (what == EVENT_FRAME_RENDERED) { 6042 handler = mOnFrameRenderedHandler; 6043 } 6044 if (handler != null) { 6045 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 6046 handler.sendMessage(msg); 6047 } 6048 } 6049 } 6050 6051 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)6052 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 6053 6054 /** 6055 * Get the codec info. If the codec was created by createDecoderByType 6056 * or createEncoderByType, what component is chosen is not known beforehand, 6057 * and thus the caller does not have the MediaCodecInfo. 6058 * @throws IllegalStateException if in the Released state. 6059 */ 6060 @NonNull getCodecInfo()6061 public MediaCodecInfo getCodecInfo() { 6062 // Get the codec name first. If the codec is already released, 6063 // IllegalStateException will be thrown here. 6064 String name = getName(); 6065 synchronized (mCodecInfoLock) { 6066 if (mCodecInfo == null) { 6067 // Get the codec info for this codec itself first. Only initialize 6068 // the full codec list if this somehow fails because it can be slow. 6069 mCodecInfo = getOwnCodecInfo(); 6070 if (mCodecInfo == null) { 6071 mCodecInfo = MediaCodecList.getInfoFor(name); 6072 } 6073 } 6074 return mCodecInfo; 6075 } 6076 } 6077 6078 @NonNull getOwnCodecInfo()6079 private native final MediaCodecInfo getOwnCodecInfo(); 6080 6081 @NonNull 6082 @UnsupportedAppUsage getBuffers(boolean input)6083 private native final ByteBuffer[] getBuffers(boolean input); 6084 6085 @Nullable getBuffer(boolean input, int index)6086 private native final ByteBuffer getBuffer(boolean input, int index); 6087 6088 @Nullable getImage(boolean input, int index)6089 private native final Image getImage(boolean input, int index); 6090 native_init()6091 private static native final void native_init(); 6092 native_setup( @onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)6093 private native final void native_setup( 6094 @NonNull String name, boolean nameIsType, boolean encoder, int pid, int uid); 6095 native_finalize()6096 private native final void native_finalize(); 6097 6098 static { 6099 System.loadLibrary("media_jni"); native_init()6100 native_init(); 6101 } 6102 6103 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 6104 private long mNativeContext = 0; 6105 private final Lock mNativeContextLock = new ReentrantLock(); 6106 lockAndGetContext()6107 private final long lockAndGetContext() { 6108 mNativeContextLock.lock(); 6109 return mNativeContext; 6110 } 6111 setAndUnlockContext(long context)6112 private final void setAndUnlockContext(long context) { 6113 mNativeContext = context; 6114 mNativeContextLock.unlock(); 6115 } 6116 6117 /** @hide */ 6118 public static class MediaImage extends Image { 6119 private final boolean mIsReadOnly; 6120 private final int mWidth; 6121 private final int mHeight; 6122 private final int mFormat; 6123 private long mTimestamp; 6124 private final Plane[] mPlanes; 6125 private final ByteBuffer mBuffer; 6126 private final ByteBuffer mInfo; 6127 private final int mXOffset; 6128 private final int mYOffset; 6129 private final long mBufferContext; 6130 6131 private final static int TYPE_YUV = 1; 6132 6133 private final int mTransform = 0; //Default no transform 6134 private final int mScalingMode = 0; //Default frozen scaling mode 6135 6136 @Override getFormat()6137 public int getFormat() { 6138 throwISEIfImageIsInvalid(); 6139 return mFormat; 6140 } 6141 6142 @Override getHeight()6143 public int getHeight() { 6144 throwISEIfImageIsInvalid(); 6145 return mHeight; 6146 } 6147 6148 @Override getWidth()6149 public int getWidth() { 6150 throwISEIfImageIsInvalid(); 6151 return mWidth; 6152 } 6153 6154 @Override getTransform()6155 public int getTransform() { 6156 throwISEIfImageIsInvalid(); 6157 return mTransform; 6158 } 6159 6160 @Override getScalingMode()6161 public int getScalingMode() { 6162 throwISEIfImageIsInvalid(); 6163 return mScalingMode; 6164 } 6165 6166 @Override getTimestamp()6167 public long getTimestamp() { 6168 throwISEIfImageIsInvalid(); 6169 return mTimestamp; 6170 } 6171 6172 @Override 6173 @NonNull getPlanes()6174 public Plane[] getPlanes() { 6175 throwISEIfImageIsInvalid(); 6176 return Arrays.copyOf(mPlanes, mPlanes.length); 6177 } 6178 6179 @Override close()6180 public void close() { 6181 if (mIsImageValid) { 6182 if (mBuffer != null) { 6183 java.nio.NioUtils.freeDirectBuffer(mBuffer); 6184 } 6185 if (mBufferContext != 0) { 6186 native_closeMediaImage(mBufferContext); 6187 } 6188 mIsImageValid = false; 6189 } 6190 } 6191 6192 /** 6193 * Set the crop rectangle associated with this frame. 6194 * <p> 6195 * The crop rectangle specifies the region of valid pixels in the image, 6196 * using coordinates in the largest-resolution plane. 6197 */ 6198 @Override setCropRect(@ullable Rect cropRect)6199 public void setCropRect(@Nullable Rect cropRect) { 6200 if (mIsReadOnly) { 6201 throw new ReadOnlyBufferException(); 6202 } 6203 super.setCropRect(cropRect); 6204 } 6205 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)6206 public MediaImage( 6207 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 6208 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 6209 mTimestamp = timestamp; 6210 mIsImageValid = true; 6211 mIsReadOnly = buffer.isReadOnly(); 6212 mBuffer = buffer.duplicate(); 6213 6214 // save offsets and info 6215 mXOffset = xOffset; 6216 mYOffset = yOffset; 6217 mInfo = info; 6218 6219 mBufferContext = 0; 6220 6221 int cbPlaneOffset = -1; 6222 int crPlaneOffset = -1; 6223 int planeOffsetInc = -1; 6224 int pixelStride = -1; 6225 6226 // read media-info. See MediaImage2 6227 if (info.remaining() == 104) { 6228 int type = info.getInt(); 6229 if (type != TYPE_YUV) { 6230 throw new UnsupportedOperationException("unsupported type: " + type); 6231 } 6232 int numPlanes = info.getInt(); 6233 if (numPlanes != 3) { 6234 throw new RuntimeException("unexpected number of planes: " + numPlanes); 6235 } 6236 mWidth = info.getInt(); 6237 mHeight = info.getInt(); 6238 if (mWidth < 1 || mHeight < 1) { 6239 throw new UnsupportedOperationException( 6240 "unsupported size: " + mWidth + "x" + mHeight); 6241 } 6242 int bitDepth = info.getInt(); 6243 if (bitDepth != 8 && bitDepth != 10) { 6244 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 6245 } 6246 int bitDepthAllocated = info.getInt(); 6247 if (bitDepthAllocated != 8 && bitDepthAllocated != 16) { 6248 throw new UnsupportedOperationException( 6249 "unsupported allocated bit depth: " + bitDepthAllocated); 6250 } 6251 if (bitDepth == 8 && bitDepthAllocated == 8) { 6252 mFormat = ImageFormat.YUV_420_888; 6253 planeOffsetInc = 1; 6254 pixelStride = 2; 6255 } else if (bitDepth == 10 && bitDepthAllocated == 16) { 6256 mFormat = ImageFormat.YCBCR_P010; 6257 planeOffsetInc = 2; 6258 pixelStride = 4; 6259 } else { 6260 throw new UnsupportedOperationException("couldn't infer ImageFormat" 6261 + " bitDepth: " + bitDepth + " bitDepthAllocated: " + bitDepthAllocated); 6262 } 6263 6264 mPlanes = new MediaPlane[numPlanes]; 6265 for (int ix = 0; ix < numPlanes; ix++) { 6266 int planeOffset = info.getInt(); 6267 int colInc = info.getInt(); 6268 int rowInc = info.getInt(); 6269 int horiz = info.getInt(); 6270 int vert = info.getInt(); 6271 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 6272 throw new UnsupportedOperationException("unexpected subsampling: " 6273 + horiz + "x" + vert + " on plane " + ix); 6274 } 6275 if (colInc < 1 || rowInc < 1) { 6276 throw new UnsupportedOperationException("unexpected strides: " 6277 + colInc + " pixel, " + rowInc + " row on plane " + ix); 6278 } 6279 buffer.clear(); 6280 buffer.position(mBuffer.position() + planeOffset 6281 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 6282 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 6283 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 6284 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 6285 if ((mFormat == ImageFormat.YUV_420_888 || mFormat == ImageFormat.YCBCR_P010 6286 || mFormat == ImageFormat.YCBCR_P210) 6287 && ix == 1) { 6288 cbPlaneOffset = planeOffset; 6289 } else if ((mFormat == ImageFormat.YUV_420_888 6290 || mFormat == ImageFormat.YCBCR_P010 6291 || mFormat == ImageFormat.YCBCR_P210) 6292 && ix == 2) { 6293 crPlaneOffset = planeOffset; 6294 } 6295 } 6296 } else { 6297 throw new UnsupportedOperationException( 6298 "unsupported info length: " + info.remaining()); 6299 } 6300 6301 // Validate chroma semiplanerness. 6302 if (mFormat == ImageFormat.YCBCR_P010 || mFormat == ImageFormat.YCBCR_P210) { 6303 if (crPlaneOffset != cbPlaneOffset + planeOffsetInc) { 6304 throw new UnsupportedOperationException("Invalid plane offsets" 6305 + " cbPlaneOffset: " + cbPlaneOffset + " crPlaneOffset: " + crPlaneOffset); 6306 } 6307 if (mPlanes[1].getPixelStride() != pixelStride 6308 || mPlanes[2].getPixelStride() != pixelStride) { 6309 throw new UnsupportedOperationException("Invalid pixelStride"); 6310 } 6311 } 6312 6313 if (cropRect == null) { 6314 cropRect = new Rect(0, 0, mWidth, mHeight); 6315 } 6316 cropRect.offset(-xOffset, -yOffset); 6317 super.setCropRect(cropRect); 6318 } 6319 MediaImage( @onNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, int width, int height, int format, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context)6320 public MediaImage( 6321 @NonNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, 6322 int width, int height, int format, boolean readOnly, 6323 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context) { 6324 if (buffers.length != rowStrides.length || buffers.length != pixelStrides.length) { 6325 throw new IllegalArgumentException( 6326 "buffers, rowStrides and pixelStrides should have the same length"); 6327 } 6328 mWidth = width; 6329 mHeight = height; 6330 mFormat = format; 6331 mTimestamp = timestamp; 6332 mIsImageValid = true; 6333 mIsReadOnly = readOnly; 6334 mBuffer = null; 6335 mInfo = null; 6336 mPlanes = new MediaPlane[buffers.length]; 6337 for (int i = 0; i < buffers.length; ++i) { 6338 mPlanes[i] = new MediaPlane(buffers[i], rowStrides[i], pixelStrides[i]); 6339 } 6340 6341 // save offsets and info 6342 mXOffset = xOffset; 6343 mYOffset = yOffset; 6344 6345 if (cropRect == null) { 6346 cropRect = new Rect(0, 0, mWidth, mHeight); 6347 } 6348 cropRect.offset(-xOffset, -yOffset); 6349 super.setCropRect(cropRect); 6350 6351 mBufferContext = context; 6352 } 6353 6354 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)6355 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 6356 mData = buffer; 6357 mRowInc = rowInc; 6358 mColInc = colInc; 6359 } 6360 6361 @Override getRowStride()6362 public int getRowStride() { 6363 throwISEIfImageIsInvalid(); 6364 return mRowInc; 6365 } 6366 6367 @Override getPixelStride()6368 public int getPixelStride() { 6369 throwISEIfImageIsInvalid(); 6370 return mColInc; 6371 } 6372 6373 @Override 6374 @NonNull getBuffer()6375 public ByteBuffer getBuffer() { 6376 throwISEIfImageIsInvalid(); 6377 return mData; 6378 } 6379 6380 private final int mRowInc; 6381 private final int mColInc; 6382 private final ByteBuffer mData; 6383 } 6384 } 6385 6386 public final static class MetricsConstants 6387 { MetricsConstants()6388 private MetricsConstants() {} 6389 6390 /** 6391 * Key to extract the codec being used 6392 * from the {@link MediaCodec#getMetrics} return value. 6393 * The value is a String. 6394 */ 6395 public static final String CODEC = "android.media.mediacodec.codec"; 6396 6397 /** 6398 * Key to extract the MIME type 6399 * from the {@link MediaCodec#getMetrics} return value. 6400 * The value is a String. 6401 */ 6402 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 6403 6404 /** 6405 * Key to extract what the codec mode 6406 * from the {@link MediaCodec#getMetrics} return value. 6407 * The value is a String. Values will be one of the constants 6408 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 6409 */ 6410 public static final String MODE = "android.media.mediacodec.mode"; 6411 6412 /** 6413 * The value returned for the key {@link #MODE} when the 6414 * codec is a audio codec. 6415 */ 6416 public static final String MODE_AUDIO = "audio"; 6417 6418 /** 6419 * The value returned for the key {@link #MODE} when the 6420 * codec is a video codec. 6421 */ 6422 public static final String MODE_VIDEO = "video"; 6423 6424 /** 6425 * Key to extract the flag indicating whether the codec is running 6426 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 6427 * The value is an integer. 6428 * A 0 indicates decoder; 1 indicates encoder. 6429 */ 6430 public static final String ENCODER = "android.media.mediacodec.encoder"; 6431 6432 /** 6433 * Key to extract the flag indicating whether the codec is running 6434 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 6435 * The value is an integer. 6436 */ 6437 public static final String SECURE = "android.media.mediacodec.secure"; 6438 6439 /** 6440 * Key to extract the width (in pixels) of the video track 6441 * from the {@link MediaCodec#getMetrics} return value. 6442 * The value is an integer. 6443 */ 6444 public static final String WIDTH = "android.media.mediacodec.width"; 6445 6446 /** 6447 * Key to extract the height (in pixels) of the video track 6448 * from the {@link MediaCodec#getMetrics} return value. 6449 * The value is an integer. 6450 */ 6451 public static final String HEIGHT = "android.media.mediacodec.height"; 6452 6453 /** 6454 * Key to extract the rotation (in degrees) to properly orient the video 6455 * from the {@link MediaCodec#getMetrics} return. 6456 * The value is a integer. 6457 */ 6458 public static final String ROTATION = "android.media.mediacodec.rotation"; 6459 6460 } 6461 } 6462
