1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.exoplayer2.util; 17 18 import java.nio.ByteBuffer; 19 import java.util.Arrays; 20 21 /** 22 * Utility methods for handling H.264/AVC and H.265/HEVC NAL units. 23 */ 24 public final class NalUnitUtil { 25 26 private static final String TAG = "NalUnitUtil"; 27 28 /** 29 * Holds data parsed from a sequence parameter set NAL unit. 30 */ 31 public static final class SpsData { 32 33 public final int profileIdc; 34 public final int constraintsFlagsAndReservedZero2Bits; 35 public final int levelIdc; 36 public final int seqParameterSetId; 37 public final int width; 38 public final int height; 39 public final float pixelWidthAspectRatio; 40 public final boolean separateColorPlaneFlag; 41 public final boolean frameMbsOnlyFlag; 42 public final int frameNumLength; 43 public final int picOrderCountType; 44 public final int picOrderCntLsbLength; 45 public final boolean deltaPicOrderAlwaysZeroFlag; 46 SpsData( int profileIdc, int constraintsFlagsAndReservedZero2Bits, int levelIdc, int seqParameterSetId, int width, int height, float pixelWidthAspectRatio, boolean separateColorPlaneFlag, boolean frameMbsOnlyFlag, int frameNumLength, int picOrderCountType, int picOrderCntLsbLength, boolean deltaPicOrderAlwaysZeroFlag)47 public SpsData( 48 int profileIdc, 49 int constraintsFlagsAndReservedZero2Bits, 50 int levelIdc, 51 int seqParameterSetId, 52 int width, 53 int height, 54 float pixelWidthAspectRatio, 55 boolean separateColorPlaneFlag, 56 boolean frameMbsOnlyFlag, 57 int frameNumLength, 58 int picOrderCountType, 59 int picOrderCntLsbLength, 60 boolean deltaPicOrderAlwaysZeroFlag) { 61 this.profileIdc = profileIdc; 62 this.constraintsFlagsAndReservedZero2Bits = constraintsFlagsAndReservedZero2Bits; 63 this.levelIdc = levelIdc; 64 this.seqParameterSetId = seqParameterSetId; 65 this.width = width; 66 this.height = height; 67 this.pixelWidthAspectRatio = pixelWidthAspectRatio; 68 this.separateColorPlaneFlag = separateColorPlaneFlag; 69 this.frameMbsOnlyFlag = frameMbsOnlyFlag; 70 this.frameNumLength = frameNumLength; 71 this.picOrderCountType = picOrderCountType; 72 this.picOrderCntLsbLength = picOrderCntLsbLength; 73 this.deltaPicOrderAlwaysZeroFlag = deltaPicOrderAlwaysZeroFlag; 74 } 75 76 } 77 78 /** 79 * Holds data parsed from a picture parameter set NAL unit. 80 */ 81 public static final class PpsData { 82 83 public final int picParameterSetId; 84 public final int seqParameterSetId; 85 public final boolean bottomFieldPicOrderInFramePresentFlag; 86 PpsData(int picParameterSetId, int seqParameterSetId, boolean bottomFieldPicOrderInFramePresentFlag)87 public PpsData(int picParameterSetId, int seqParameterSetId, 88 boolean bottomFieldPicOrderInFramePresentFlag) { 89 this.picParameterSetId = picParameterSetId; 90 this.seqParameterSetId = seqParameterSetId; 91 this.bottomFieldPicOrderInFramePresentFlag = bottomFieldPicOrderInFramePresentFlag; 92 } 93 94 } 95 96 /** Four initial bytes that must prefix NAL units for decoding. */ 97 public static final byte[] NAL_START_CODE = new byte[] {0, 0, 0, 1}; 98 99 /** Value for aspect_ratio_idc indicating an extended aspect ratio, in H.264 and H.265 SPSs. */ 100 public static final int EXTENDED_SAR = 0xFF; 101 /** Aspect ratios indexed by aspect_ratio_idc, in H.264 and H.265 SPSs. */ 102 public static final float[] ASPECT_RATIO_IDC_VALUES = new float[] { 103 1f /* Unspecified. Assume square */, 104 1f, 105 12f / 11f, 106 10f / 11f, 107 16f / 11f, 108 40f / 33f, 109 24f / 11f, 110 20f / 11f, 111 32f / 11f, 112 80f / 33f, 113 18f / 11f, 114 15f / 11f, 115 64f / 33f, 116 160f / 99f, 117 4f / 3f, 118 3f / 2f, 119 2f 120 }; 121 122 private static final int H264_NAL_UNIT_TYPE_SEI = 6; // Supplemental enhancement information 123 private static final int H264_NAL_UNIT_TYPE_SPS = 7; // Sequence parameter set 124 private static final int H265_NAL_UNIT_TYPE_PREFIX_SEI = 39; 125 126 private static final Object scratchEscapePositionsLock = new Object(); 127 128 /** 129 * Temporary store for positions of escape codes in {@link #unescapeStream(byte[], int)}. Guarded 130 * by {@link #scratchEscapePositionsLock}. 131 */ 132 private static int[] scratchEscapePositions = new int[10]; 133 134 /** 135 * Unescapes {@code data} up to the specified limit, replacing occurrences of [0, 0, 3] with 136 * [0, 0]. The unescaped data is returned in-place, with the return value indicating its length. 137 * <p> 138 * Executions of this method are mutually exclusive, so it should not be called with very large 139 * buffers. 140 * 141 * @param data The data to unescape. 142 * @param limit The limit (exclusive) of the data to unescape. 143 * @return The length of the unescaped data. 144 */ unescapeStream(byte[] data, int limit)145 public static int unescapeStream(byte[] data, int limit) { 146 synchronized (scratchEscapePositionsLock) { 147 int position = 0; 148 int scratchEscapeCount = 0; 149 while (position < limit) { 150 position = findNextUnescapeIndex(data, position, limit); 151 if (position < limit) { 152 if (scratchEscapePositions.length <= scratchEscapeCount) { 153 // Grow scratchEscapePositions to hold a larger number of positions. 154 scratchEscapePositions = Arrays.copyOf(scratchEscapePositions, 155 scratchEscapePositions.length * 2); 156 } 157 scratchEscapePositions[scratchEscapeCount++] = position; 158 position += 3; 159 } 160 } 161 162 int unescapedLength = limit - scratchEscapeCount; 163 int escapedPosition = 0; // The position being read from. 164 int unescapedPosition = 0; // The position being written to. 165 for (int i = 0; i < scratchEscapeCount; i++) { 166 int nextEscapePosition = scratchEscapePositions[i]; 167 int copyLength = nextEscapePosition - escapedPosition; 168 System.arraycopy(data, escapedPosition, data, unescapedPosition, copyLength); 169 unescapedPosition += copyLength; 170 data[unescapedPosition++] = 0; 171 data[unescapedPosition++] = 0; 172 escapedPosition += copyLength + 3; 173 } 174 175 int remainingLength = unescapedLength - unescapedPosition; 176 System.arraycopy(data, escapedPosition, data, unescapedPosition, remainingLength); 177 return unescapedLength; 178 } 179 } 180 181 /** 182 * Discards data from the buffer up to the first SPS, where {@code data.position()} is interpreted 183 * as the length of the buffer. 184 * <p> 185 * When the method returns, {@code data.position()} will contain the new length of the buffer. If 186 * the buffer is not empty it is guaranteed to start with an SPS. 187 * 188 * @param data Buffer containing start code delimited NAL units. 189 */ discardToSps(ByteBuffer data)190 public static void discardToSps(ByteBuffer data) { 191 int length = data.position(); 192 int consecutiveZeros = 0; 193 int offset = 0; 194 while (offset + 1 < length) { 195 int value = data.get(offset) & 0xFF; 196 if (consecutiveZeros == 3) { 197 if (value == 1 && (data.get(offset + 1) & 0x1F) == H264_NAL_UNIT_TYPE_SPS) { 198 // Copy from this NAL unit onwards to the start of the buffer. 199 ByteBuffer offsetData = data.duplicate(); 200 offsetData.position(offset - 3); 201 offsetData.limit(length); 202 data.position(0); 203 data.put(offsetData); 204 return; 205 } 206 } else if (value == 0) { 207 consecutiveZeros++; 208 } 209 if (value != 0) { 210 consecutiveZeros = 0; 211 } 212 offset++; 213 } 214 // Empty the buffer if the SPS NAL unit was not found. 215 data.clear(); 216 } 217 218 /** 219 * Returns whether the NAL unit with the specified header contains supplemental enhancement 220 * information. 221 * 222 * @param mimeType The sample MIME type. 223 * @param nalUnitHeaderFirstByte The first byte of nal_unit(). 224 * @return Whether the NAL unit with the specified header is an SEI NAL unit. 225 */ isNalUnitSei(String mimeType, byte nalUnitHeaderFirstByte)226 public static boolean isNalUnitSei(String mimeType, byte nalUnitHeaderFirstByte) { 227 return (MimeTypes.VIDEO_H264.equals(mimeType) 228 && (nalUnitHeaderFirstByte & 0x1F) == H264_NAL_UNIT_TYPE_SEI) 229 || (MimeTypes.VIDEO_H265.equals(mimeType) 230 && ((nalUnitHeaderFirstByte & 0x7E) >> 1) == H265_NAL_UNIT_TYPE_PREFIX_SEI); 231 } 232 233 /** 234 * Returns the type of the NAL unit in {@code data} that starts at {@code offset}. 235 * 236 * @param data The data to search. 237 * @param offset The start offset of a NAL unit. Must lie between {@code -3} (inclusive) and 238 * {@code data.length - 3} (exclusive). 239 * @return The type of the unit. 240 */ getNalUnitType(byte[] data, int offset)241 public static int getNalUnitType(byte[] data, int offset) { 242 return data[offset + 3] & 0x1F; 243 } 244 245 /** 246 * Returns the type of the H.265 NAL unit in {@code data} that starts at {@code offset}. 247 * 248 * @param data The data to search. 249 * @param offset The start offset of a NAL unit. Must lie between {@code -3} (inclusive) and 250 * {@code data.length - 3} (exclusive). 251 * @return The type of the unit. 252 */ getH265NalUnitType(byte[] data, int offset)253 public static int getH265NalUnitType(byte[] data, int offset) { 254 return (data[offset + 3] & 0x7E) >> 1; 255 } 256 257 /** 258 * Parses an SPS NAL unit using the syntax defined in ITU-T Recommendation H.264 (2013) subsection 259 * 7.3.2.1.1. 260 * 261 * @param nalData A buffer containing escaped SPS data. 262 * @param nalOffset The offset of the NAL unit header in {@code nalData}. 263 * @param nalLimit The limit of the NAL unit in {@code nalData}. 264 * @return A parsed representation of the SPS data. 265 */ parseSpsNalUnit(byte[] nalData, int nalOffset, int nalLimit)266 public static SpsData parseSpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { 267 ParsableNalUnitBitArray data = new ParsableNalUnitBitArray(nalData, nalOffset, nalLimit); 268 data.skipBits(8); // nal_unit 269 int profileIdc = data.readBits(8); 270 int constraintsFlagsAndReservedZero2Bits = data.readBits(8); 271 int levelIdc = data.readBits(8); 272 int seqParameterSetId = data.readUnsignedExpGolombCodedInt(); 273 274 int chromaFormatIdc = 1; // Default is 4:2:0 275 boolean separateColorPlaneFlag = false; 276 if (profileIdc == 100 || profileIdc == 110 || profileIdc == 122 || profileIdc == 244 277 || profileIdc == 44 || profileIdc == 83 || profileIdc == 86 || profileIdc == 118 278 || profileIdc == 128 || profileIdc == 138) { 279 chromaFormatIdc = data.readUnsignedExpGolombCodedInt(); 280 if (chromaFormatIdc == 3) { 281 separateColorPlaneFlag = data.readBit(); 282 } 283 data.readUnsignedExpGolombCodedInt(); // bit_depth_luma_minus8 284 data.readUnsignedExpGolombCodedInt(); // bit_depth_chroma_minus8 285 data.skipBit(); // qpprime_y_zero_transform_bypass_flag 286 boolean seqScalingMatrixPresentFlag = data.readBit(); 287 if (seqScalingMatrixPresentFlag) { 288 int limit = (chromaFormatIdc != 3) ? 8 : 12; 289 for (int i = 0; i < limit; i++) { 290 boolean seqScalingListPresentFlag = data.readBit(); 291 if (seqScalingListPresentFlag) { 292 skipScalingList(data, i < 6 ? 16 : 64); 293 } 294 } 295 } 296 } 297 298 int frameNumLength = data.readUnsignedExpGolombCodedInt() + 4; // log2_max_frame_num_minus4 + 4 299 int picOrderCntType = data.readUnsignedExpGolombCodedInt(); 300 int picOrderCntLsbLength = 0; 301 boolean deltaPicOrderAlwaysZeroFlag = false; 302 if (picOrderCntType == 0) { 303 // log2_max_pic_order_cnt_lsb_minus4 + 4 304 picOrderCntLsbLength = data.readUnsignedExpGolombCodedInt() + 4; 305 } else if (picOrderCntType == 1) { 306 deltaPicOrderAlwaysZeroFlag = data.readBit(); // delta_pic_order_always_zero_flag 307 data.readSignedExpGolombCodedInt(); // offset_for_non_ref_pic 308 data.readSignedExpGolombCodedInt(); // offset_for_top_to_bottom_field 309 long numRefFramesInPicOrderCntCycle = data.readUnsignedExpGolombCodedInt(); 310 for (int i = 0; i < numRefFramesInPicOrderCntCycle; i++) { 311 data.readUnsignedExpGolombCodedInt(); // offset_for_ref_frame[i] 312 } 313 } 314 data.readUnsignedExpGolombCodedInt(); // max_num_ref_frames 315 data.skipBit(); // gaps_in_frame_num_value_allowed_flag 316 317 int picWidthInMbs = data.readUnsignedExpGolombCodedInt() + 1; 318 int picHeightInMapUnits = data.readUnsignedExpGolombCodedInt() + 1; 319 boolean frameMbsOnlyFlag = data.readBit(); 320 int frameHeightInMbs = (2 - (frameMbsOnlyFlag ? 1 : 0)) * picHeightInMapUnits; 321 if (!frameMbsOnlyFlag) { 322 data.skipBit(); // mb_adaptive_frame_field_flag 323 } 324 325 data.skipBit(); // direct_8x8_inference_flag 326 int frameWidth = picWidthInMbs * 16; 327 int frameHeight = frameHeightInMbs * 16; 328 boolean frameCroppingFlag = data.readBit(); 329 if (frameCroppingFlag) { 330 int frameCropLeftOffset = data.readUnsignedExpGolombCodedInt(); 331 int frameCropRightOffset = data.readUnsignedExpGolombCodedInt(); 332 int frameCropTopOffset = data.readUnsignedExpGolombCodedInt(); 333 int frameCropBottomOffset = data.readUnsignedExpGolombCodedInt(); 334 int cropUnitX; 335 int cropUnitY; 336 if (chromaFormatIdc == 0) { 337 cropUnitX = 1; 338 cropUnitY = 2 - (frameMbsOnlyFlag ? 1 : 0); 339 } else { 340 int subWidthC = (chromaFormatIdc == 3) ? 1 : 2; 341 int subHeightC = (chromaFormatIdc == 1) ? 2 : 1; 342 cropUnitX = subWidthC; 343 cropUnitY = subHeightC * (2 - (frameMbsOnlyFlag ? 1 : 0)); 344 } 345 frameWidth -= (frameCropLeftOffset + frameCropRightOffset) * cropUnitX; 346 frameHeight -= (frameCropTopOffset + frameCropBottomOffset) * cropUnitY; 347 } 348 349 float pixelWidthHeightRatio = 1; 350 boolean vuiParametersPresentFlag = data.readBit(); 351 if (vuiParametersPresentFlag) { 352 boolean aspectRatioInfoPresentFlag = data.readBit(); 353 if (aspectRatioInfoPresentFlag) { 354 int aspectRatioIdc = data.readBits(8); 355 if (aspectRatioIdc == NalUnitUtil.EXTENDED_SAR) { 356 int sarWidth = data.readBits(16); 357 int sarHeight = data.readBits(16); 358 if (sarWidth != 0 && sarHeight != 0) { 359 pixelWidthHeightRatio = (float) sarWidth / sarHeight; 360 } 361 } else if (aspectRatioIdc < NalUnitUtil.ASPECT_RATIO_IDC_VALUES.length) { 362 pixelWidthHeightRatio = NalUnitUtil.ASPECT_RATIO_IDC_VALUES[aspectRatioIdc]; 363 } else { 364 Log.w(TAG, "Unexpected aspect_ratio_idc value: " + aspectRatioIdc); 365 } 366 } 367 } 368 369 return new SpsData( 370 profileIdc, 371 constraintsFlagsAndReservedZero2Bits, 372 levelIdc, 373 seqParameterSetId, 374 frameWidth, 375 frameHeight, 376 pixelWidthHeightRatio, 377 separateColorPlaneFlag, 378 frameMbsOnlyFlag, 379 frameNumLength, 380 picOrderCntType, 381 picOrderCntLsbLength, 382 deltaPicOrderAlwaysZeroFlag); 383 } 384 385 /** 386 * Parses a PPS NAL unit using the syntax defined in ITU-T Recommendation H.264 (2013) subsection 387 * 7.3.2.2. 388 * 389 * @param nalData A buffer containing escaped PPS data. 390 * @param nalOffset The offset of the NAL unit header in {@code nalData}. 391 * @param nalLimit The limit of the NAL unit in {@code nalData}. 392 * @return A parsed representation of the PPS data. 393 */ parsePpsNalUnit(byte[] nalData, int nalOffset, int nalLimit)394 public static PpsData parsePpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { 395 ParsableNalUnitBitArray data = new ParsableNalUnitBitArray(nalData, nalOffset, nalLimit); 396 data.skipBits(8); // nal_unit 397 int picParameterSetId = data.readUnsignedExpGolombCodedInt(); 398 int seqParameterSetId = data.readUnsignedExpGolombCodedInt(); 399 data.skipBit(); // entropy_coding_mode_flag 400 boolean bottomFieldPicOrderInFramePresentFlag = data.readBit(); 401 return new PpsData(picParameterSetId, seqParameterSetId, bottomFieldPicOrderInFramePresentFlag); 402 } 403 404 /** 405 * Finds the first NAL unit in {@code data}. 406 * <p> 407 * If {@code prefixFlags} is null then the first three bytes of a NAL unit must be entirely 408 * contained within the part of the array being searched in order for it to be found. 409 * <p> 410 * When {@code prefixFlags} is non-null, this method supports finding NAL units whose first four 411 * bytes span {@code data} arrays passed to successive calls. To use this feature, pass the same 412 * {@code prefixFlags} parameter to successive calls. State maintained in this parameter enables 413 * the detection of such NAL units. Note that when using this feature, the return value may be 3, 414 * 2 or 1 less than {@code startOffset}, to indicate a NAL unit starting 3, 2 or 1 bytes before 415 * the first byte in the current array. 416 * 417 * @param data The data to search. 418 * @param startOffset The offset (inclusive) in the data to start the search. 419 * @param endOffset The offset (exclusive) in the data to end the search. 420 * @param prefixFlags A boolean array whose first three elements are used to store the state 421 * required to detect NAL units where the NAL unit prefix spans array boundaries. The array 422 * must be at least 3 elements long. 423 * @return The offset of the NAL unit, or {@code endOffset} if a NAL unit was not found. 424 */ findNalUnit(byte[] data, int startOffset, int endOffset, boolean[] prefixFlags)425 public static int findNalUnit(byte[] data, int startOffset, int endOffset, 426 boolean[] prefixFlags) { 427 int length = endOffset - startOffset; 428 429 Assertions.checkState(length >= 0); 430 if (length == 0) { 431 return endOffset; 432 } 433 434 if (prefixFlags != null) { 435 if (prefixFlags[0]) { 436 clearPrefixFlags(prefixFlags); 437 return startOffset - 3; 438 } else if (length > 1 && prefixFlags[1] && data[startOffset] == 1) { 439 clearPrefixFlags(prefixFlags); 440 return startOffset - 2; 441 } else if (length > 2 && prefixFlags[2] && data[startOffset] == 0 442 && data[startOffset + 1] == 1) { 443 clearPrefixFlags(prefixFlags); 444 return startOffset - 1; 445 } 446 } 447 448 int limit = endOffset - 1; 449 // We're looking for the NAL unit start code prefix 0x000001. The value of i tracks the index of 450 // the third byte. 451 for (int i = startOffset + 2; i < limit; i += 3) { 452 if ((data[i] & 0xFE) != 0) { 453 // There isn't a NAL prefix here, or at the next two positions. Do nothing and let the 454 // loop advance the index by three. 455 } else if (data[i - 2] == 0 && data[i - 1] == 0 && data[i] == 1) { 456 if (prefixFlags != null) { 457 clearPrefixFlags(prefixFlags); 458 } 459 return i - 2; 460 } else { 461 // There isn't a NAL prefix here, but there might be at the next position. We should 462 // only skip forward by one. The loop will skip forward by three, so subtract two here. 463 i -= 2; 464 } 465 } 466 467 if (prefixFlags != null) { 468 // True if the last three bytes in the data seen so far are {0,0,1}. 469 prefixFlags[0] = length > 2 470 ? (data[endOffset - 3] == 0 && data[endOffset - 2] == 0 && data[endOffset - 1] == 1) 471 : length == 2 ? (prefixFlags[2] && data[endOffset - 2] == 0 && data[endOffset - 1] == 1) 472 : (prefixFlags[1] && data[endOffset - 1] == 1); 473 // True if the last two bytes in the data seen so far are {0,0}. 474 prefixFlags[1] = length > 1 ? data[endOffset - 2] == 0 && data[endOffset - 1] == 0 475 : prefixFlags[2] && data[endOffset - 1] == 0; 476 // True if the last byte in the data seen so far is {0}. 477 prefixFlags[2] = data[endOffset - 1] == 0; 478 } 479 480 return endOffset; 481 } 482 483 /** 484 * Clears prefix flags, as used by {@link #findNalUnit(byte[], int, int, boolean[])}. 485 * 486 * @param prefixFlags The flags to clear. 487 */ clearPrefixFlags(boolean[] prefixFlags)488 public static void clearPrefixFlags(boolean[] prefixFlags) { 489 prefixFlags[0] = false; 490 prefixFlags[1] = false; 491 prefixFlags[2] = false; 492 } 493 findNextUnescapeIndex(byte[] bytes, int offset, int limit)494 private static int findNextUnescapeIndex(byte[] bytes, int offset, int limit) { 495 for (int i = offset; i < limit - 2; i++) { 496 if (bytes[i] == 0x00 && bytes[i + 1] == 0x00 && bytes[i + 2] == 0x03) { 497 return i; 498 } 499 } 500 return limit; 501 } 502 skipScalingList(ParsableNalUnitBitArray bitArray, int size)503 private static void skipScalingList(ParsableNalUnitBitArray bitArray, int size) { 504 int lastScale = 8; 505 int nextScale = 8; 506 for (int i = 0; i < size; i++) { 507 if (nextScale != 0) { 508 int deltaScale = bitArray.readSignedExpGolombCodedInt(); 509 nextScale = (lastScale + deltaScale + 256) % 256; 510 } 511 lastScale = (nextScale == 0) ? lastScale : nextScale; 512 } 513 } 514 NalUnitUtil()515 private NalUnitUtil() { 516 // Prevent instantiation. 517 } 518 519 } 520