1 #ifndef _EXTNVIDIAVIDEOPARSERIF_HPP 2 #define _EXTNVIDIAVIDEOPARSERIF_HPP 3 4 /* 5 * Copyright 2021 NVIDIA Corporation. 6 * Copyright (c) 2021 The Khronos Group Inc. 7 * 8 * Licensed under the Apache License, Version 2.0 (the "License"); 9 * you may not use this file except in compliance with the License. 10 * You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, software 15 * distributed under the License is distributed on an "AS IS" BASIS, 16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 * See the License for the specific language governing permissions and 18 * limitations under the License. 19 */ 20 /* 21 * Copyright 2020 NVIDIA Corporation. 22 * 23 * Licensed under the Apache License, Version 2.0 (the "License"); 24 * you may not use this file except in compliance with the License. 25 * You may obtain a copy of the License at 26 * 27 * http://www.apache.org/licenses/LICENSE-2.0 28 * 29 * Unless required by applicable law or agreed to in writing, software 30 * distributed under the License is distributed on an "AS IS" BASIS, 31 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 32 * See the License for the specific language governing permissions and 33 * limitations under the License. 34 */ 35 36 #include "vkDefs.hpp" 37 38 #include <atomic> 39 #include <stdint.h> 40 #include <string.h> 41 42 namespace vkt 43 { 44 namespace video 45 { 46 #define DEBUGLOG(X) 47 48 using namespace vk; 49 using namespace std; 50 51 class NvidiaParserVideoRefCountBase 52 { 53 public: 54 //! Increment the reference count by 1. 55 virtual int32_t AddRef() = 0; 56 57 //! Decrement the reference count by 1. When the reference count 58 //! goes to 0 the object is automatically destroyed. 59 virtual int32_t Release() = 0; 60 61 protected: ~NvidiaParserVideoRefCountBase()62 virtual ~NvidiaParserVideoRefCountBase() { } 63 }; 64 65 template<class NvidiaBaseObjType> 66 class NvidiaSharedBaseObj 67 { 68 public: Reset(NvidiaBaseObjType * const newObjectPtr)69 NvidiaSharedBaseObj<NvidiaBaseObjType>& Reset (NvidiaBaseObjType* const newObjectPtr) 70 { 71 if (newObjectPtr != m_sharedObject) 72 { 73 int refCount = -1; 74 75 if (m_sharedObject != nullptr) 76 { 77 refCount = m_sharedObject->Release(); 78 79 if (refCount < 0) 80 { 81 DE_ASSERT(0 && "RefCount less than zero"); 82 } 83 } 84 85 m_sharedObject = newObjectPtr; 86 87 if (newObjectPtr != nullptr) 88 { 89 refCount = newObjectPtr->AddRef(); 90 91 if (!(refCount > 0)) 92 { 93 DE_ASSERT(0 && "RefCount is smaller not greater than zero"); 94 } 95 } 96 } 97 return *this; 98 } 99 100 // Constructors increment the refcount of the provided object if non-nullptr NvidiaSharedBaseObj(NvidiaBaseObjType * const newObjectPtr=nullptr)101 explicit NvidiaSharedBaseObj (NvidiaBaseObjType* const newObjectPtr = nullptr) 102 : m_sharedObject(nullptr) 103 { 104 Reset(newObjectPtr); 105 } 106 NvidiaSharedBaseObj(const NvidiaSharedBaseObj<NvidiaBaseObjType> & newObject)107 NvidiaSharedBaseObj (const NvidiaSharedBaseObj<NvidiaBaseObjType>& newObject) 108 : m_sharedObject(nullptr) 109 { 110 Reset(newObject.Get()); 111 } 112 ~NvidiaSharedBaseObj()113 ~NvidiaSharedBaseObj () 114 { 115 Reset(nullptr); 116 } 117 118 // Assignment from another smart pointer maps to raw pointer assignment operator =(const NvidiaSharedBaseObj<NvidiaBaseObjType> & sharedObject)119 NvidiaSharedBaseObj<NvidiaBaseObjType>& operator= (const NvidiaSharedBaseObj<NvidiaBaseObjType>& sharedObject) 120 { 121 return Reset(sharedObject.Get()); 122 } 123 operator =(NvidiaBaseObjType * const newObjectPtr)124 NvidiaSharedBaseObj<NvidiaBaseObjType>& operator= (NvidiaBaseObjType* const newObjectPtr) 125 { 126 return Reset(newObjectPtr); 127 } 128 129 template <class VkBaseObjType2> operator =(const NvidiaSharedBaseObj<VkBaseObjType2> & otherSharedObject)130 const NvidiaSharedBaseObj<NvidiaBaseObjType>& operator= (const NvidiaSharedBaseObj<VkBaseObjType2>& otherSharedObject) 131 { 132 return Reset(otherSharedObject.Get()); 133 } 134 135 // Comparison operators can be used with any compatible types operator ==(const NvidiaSharedBaseObj<NvidiaBaseObjType> & otherObject)136 inline bool operator== (const NvidiaSharedBaseObj<NvidiaBaseObjType>& otherObject) 137 { 138 return (this->Get() == otherObject.Get()); 139 } 140 operator !=(const NvidiaSharedBaseObj<NvidiaBaseObjType> & otherObject)141 inline bool operator!= (const NvidiaSharedBaseObj<NvidiaBaseObjType>& otherObject) 142 { 143 return !(*this == otherObject); 144 } 145 operator !() const146 bool operator! () const 147 { 148 return m_sharedObject == nullptr; 149 } 150 151 // Exchange Swap(NvidiaSharedBaseObj<NvidiaBaseObjType> & sharedObject)152 void Swap (NvidiaSharedBaseObj<NvidiaBaseObjType>& sharedObject) 153 { 154 NvidiaSharedBaseObj<NvidiaBaseObjType> tmp(m_sharedObject); 155 156 m_sharedObject = sharedObject.m_sharedObject; 157 158 sharedObject.m_sharedObject = tmp; 159 } 160 161 // Non ref-counted access to the underlying object Get(void) const162 NvidiaBaseObjType* Get (void) const 163 { 164 return m_sharedObject; 165 } 166 167 // Cast to a raw object pointer operator NvidiaBaseObjType*() const168 operator NvidiaBaseObjType* () const 169 { 170 return m_sharedObject; 171 } 172 operator ->() const173 NvidiaBaseObjType* operator-> () const 174 { 175 return m_sharedObject; 176 } 177 operator *() const178 NvidiaBaseObjType& operator* () const 179 { 180 return *m_sharedObject; 181 } 182 183 private: 184 NvidiaBaseObjType* m_sharedObject; 185 }; 186 187 class INvidiaVulkanPicture 188 { 189 public: 190 virtual void AddRef() = 0; 191 virtual void Release() = 0; 192 193 int32_t decodeWidth; 194 int32_t decodeHeight; 195 int32_t decodeSuperResWidth; 196 int32_t reserved[16 - 3]; 197 198 protected: ~INvidiaVulkanPicture()199 virtual ~INvidiaVulkanPicture() { } 200 }; 201 202 class NvidiaVulkanPictureBase : public INvidiaVulkanPicture 203 { 204 private: 205 std::atomic<int32_t> m_refCount; 206 207 public: 208 int32_t m_picIdx; 209 int32_t m_displayOrder; 210 int32_t m_decodeOrder; 211 uint64_t m_timestamp; 212 uint64_t m_presentTime; 213 214 public: AddRef()215 virtual void AddRef () 216 { 217 DE_ASSERT(m_refCount >= 0); 218 219 ++m_refCount; 220 } 221 Release()222 virtual void Release () 223 { 224 DE_ASSERT(m_refCount > 0); 225 226 int32_t ref = --m_refCount; 227 228 if (ref == 0) 229 { 230 Reset(); 231 } 232 } 233 234 public: NvidiaVulkanPictureBase()235 NvidiaVulkanPictureBase () 236 : m_refCount (0) 237 , m_picIdx (-1) 238 , m_displayOrder (-1) 239 , m_decodeOrder (-1) 240 , m_timestamp (0) 241 , m_presentTime (0) 242 { 243 } 244 IsAvailable() const245 bool IsAvailable () const 246 { 247 DE_ASSERT(m_refCount >= 0); 248 249 return (m_refCount == 0); 250 } 251 Reset()252 int32_t Reset () 253 { 254 int32_t ref = m_refCount; 255 256 m_picIdx = -1; 257 m_displayOrder = -1; 258 m_decodeOrder = -1; 259 m_timestamp = 0; 260 m_presentTime = 0; 261 m_refCount = 0; 262 263 return ref; 264 } 265 ~NvidiaVulkanPictureBase()266 virtual ~NvidiaVulkanPictureBase () 267 { 268 Reset(); 269 } 270 }; 271 272 #define NV_VULKAN_VIDEO_PARSER_API_VERSION_0_9_7 VK_MAKE_VIDEO_STD_VERSION(0, 9, 7) 273 274 #define NV_VULKAN_VIDEO_PARSER_API_VERSION NV_VULKAN_VIDEO_PARSER_API_VERSION_0_9_7 275 276 typedef uint32_t FrameRate; // Packed 18-bit numerator & 14-bit denominator 277 278 // Definitions for video_format 279 enum 280 { 281 VideoFormatComponent = 0, 282 VideoFormatPAL, 283 VideoFormatNTSC, 284 VideoFormatSECAM, 285 VideoFormatMAC, 286 VideoFormatUnspecified, 287 VideoFormatReserved6, 288 VideoFormatReserved7 289 }; 290 291 // Definitions for color_primaries 292 enum 293 { 294 ColorPrimariesForbidden = 0, 295 ColorPrimariesBT709 = 1, 296 ColorPrimariesUnspecified = 2, 297 ColorPrimariesReserved = 3, 298 ColorPrimariesBT470M = 4, 299 ColorPrimariesBT470BG = 5, 300 ColorPrimariesSMPTE170M = 6, // Also, ITU-R BT.601 301 ColorPrimariesSMPTE240M = 7, 302 ColorPrimariesGenericFilm = 8, 303 ColorPrimariesBT2020 = 9, 304 // below are defined in AOM standard 305 ColorPrimariesXYZ = 10, // SMPTE 428 (CIE 1921 XYZ) 306 ColorPrimariesSMPTE431 = 11, // SMPTE RP 431-2 307 ColorPrimariesSMPTE432 = 12, // SMPTE EG 432-1 308 ColorPrimariesRESERVED13 = 13, // For future use (values 13 - 21) 309 ColorPrimariesEBU3213 = 22, // EBU Tech. 3213-E 310 ColorPrimariesRESERVED23 = 23 // For future use (values 23 - 255) 311 }; 312 313 // Definitions for transfer_characteristics 314 enum 315 { 316 TransferCharacteristicsForbidden = 0, 317 TransferCharacteristicsBT709 = 1, 318 TransferCharacteristicsUnspecified = 2, 319 TransferCharacteristicsReserved = 3, 320 TransferCharacteristicsBT470M = 4, 321 TransferCharacteristicsBT470BG = 5, 322 TransferCharacteristicsSMPTE170M = 6, 323 TransferCharacteristicsSMPTE240M = 7, 324 TransferCharacteristicsLinear = 8, 325 TransferCharacteristicsLog100 = 9, 326 TransferCharacteristicsLog316 = 10, 327 TransferCharacteristicsIEC61966_2_4 = 11, 328 TransferCharacteristicsBT1361 = 12, 329 TransferCharacteristicsIEC61966_2_1 = 13, 330 TransferCharacteristicsBT2020 = 14, 331 TransferCharacteristicsBT2020_2 = 15, 332 TransferCharacteristicsST2084 = 16, 333 TransferCharacteristicsST428_1 = 17, 334 // below are defined in AOM standard 335 TransferCharacteristicsHLG = 18, // BT.2100 HLG, ARIB STD-B67 336 TransferCharacteristicsRESERVED19 = 19 // For future use (values 19-255) 337 }; 338 339 // Definitions for matrix_coefficients 340 enum 341 { 342 MatrixCoefficientsForbidden = 0, 343 MatrixCoefficientsBT709 = 1, 344 MatrixCoefficientsUnspecified = 2, 345 MatrixCoefficientsReserved = 3, 346 MatrixCoefficientsFCC = 4, 347 MatrixCoefficientsBT470BG = 5, 348 MatrixCoefficientsSMPTE170M = 6, 349 MatrixCoefficientsSMPTE240M = 7, 350 MatrixCoefficientsYCgCo = 8, 351 MatrixCoefficientsBT2020_NCL = 9, // Non-constant luminance 352 MatrixCoefficientsBT2020_CL = 10, // Constant luminance 353 // below are defined in AOM standard 354 MatrixCoefficientsSMPTE2085 = 11, // SMPTE ST 2085 YDzDx 355 MatrixCoefficientsCHROMAT_NCL = 12, // Chromaticity-derived non-constant luminance 356 MatrixCoefficientsCHROMAT_CL = 13, // Chromaticity-derived constant luminance 357 MatrixCoefficientsICTCP = 14, // BT.2100 ICtCp 358 MatrixCoefficientsRESERVED15 = 15 359 }; 360 361 // Maximum raw sequence header length (all codecs) i.e. 1024 bytes 362 #define VK_MAX_SEQ_HDR_LEN (1024) 363 364 typedef struct NvidiaVulkanParserH264DpbEntry 365 { 366 INvidiaVulkanPicture* pNvidiaVulkanPicture; // ptr to reference frame 367 int32_t FrameIdx; // frame_num(short-term) or LongTermFrameIdx(long-term) 368 int32_t is_long_term; // 0=short term reference, 1=long term reference 369 int32_t not_existing; // non-existing reference frame (corresponding PicIdx should be set to -1) 370 int32_t used_for_reference; // 0=unused, 1=top_field, 2=bottom_field, 3=both_fields 371 int32_t FieldOrderCnt[2]; // field order count of top and bottom fields 372 } NvidiaVulkanParserH264DpbEntry; 373 374 typedef struct NvidiaVulkanParserH264PictureData 375 { 376 // SPS 377 const struct vk::StdVideoH264SequenceParameterSet* pStdSps; 378 NvidiaParserVideoRefCountBase* pSpsClientObject; 379 380 // PPS 381 const struct vk::StdVideoH264PictureParameterSet* pStdPps; 382 NvidiaParserVideoRefCountBase* pPpsClientObject; 383 384 uint8_t pic_parameter_set_id; // PPS ID 385 uint8_t seq_parameter_set_id; // SPS ID 386 uint8_t vps_video_parameter_set_id; // VPS ID 387 int32_t num_ref_idx_l0_active_minus1; 388 int32_t num_ref_idx_l1_active_minus1; 389 int32_t weighted_pred_flag; 390 int32_t weighted_bipred_idc; 391 int32_t pic_init_qp_minus26; 392 int32_t redundant_pic_cnt_present_flag; 393 uint8_t deblocking_filter_control_present_flag; 394 uint8_t transform_8x8_mode_flag; 395 uint8_t MbaffFrameFlag; 396 uint8_t constrained_intra_pred_flag; 397 uint8_t entropy_coding_mode_flag; 398 uint8_t pic_order_present_flag; 399 int8_t chroma_qp_index_offset; 400 int8_t second_chroma_qp_index_offset; 401 int32_t frame_num; 402 int32_t CurrFieldOrderCnt[2]; 403 uint8_t fmo_aso_enable; 404 uint8_t num_slice_groups_minus1; 405 uint8_t slice_group_map_type; 406 int8_t pic_init_qs_minus26; 407 uint32_t slice_group_change_rate_minus1; 408 const uint8_t* pMb2SliceGroupMap; 409 // DPB 410 NvidiaVulkanParserH264DpbEntry dpb[16 + 1]; // List of reference frames within the DPB 411 412 // Quantization Matrices (raster-order) 413 union 414 { 415 // MVC extension 416 struct 417 { 418 int32_t num_views_minus1; 419 int32_t view_id; 420 uint8_t inter_view_flag; 421 uint8_t num_inter_view_refs_l0; 422 uint8_t num_inter_view_refs_l1; 423 uint8_t MVCReserved8Bits; 424 int32_t InterViewRefsL0[16]; 425 int32_t InterViewRefsL1[16]; 426 } mvcext; 427 // SVC extension 428 struct 429 { 430 uint8_t profile_idc; 431 uint8_t level_idc; 432 uint8_t DQId; 433 uint8_t DQIdMax; 434 uint8_t disable_inter_layer_deblocking_filter_idc; 435 uint8_t ref_layer_chroma_phase_y_plus1; 436 int8_t inter_layer_slice_alpha_c0_offset_div2; 437 int8_t inter_layer_slice_beta_offset_div2; 438 uint16_t DPBEntryValidFlag; 439 440 union 441 { 442 struct 443 { 444 uint8_t inter_layer_deblocking_filter_control_present_flag : 1; 445 uint8_t extended_spatial_scalability_idc : 2; 446 uint8_t adaptive_tcoeff_level_prediction_flag : 1; 447 uint8_t slice_header_restriction_flag : 1; 448 uint8_t chroma_phase_x_plus1_flag : 1; 449 uint8_t chroma_phase_y_plus1 : 2; 450 uint8_t tcoeff_level_prediction_flag : 1; 451 uint8_t constrained_intra_resampling_flag : 1; 452 uint8_t ref_layer_chroma_phase_x_plus1_flag : 1; 453 uint8_t store_ref_base_pic_flag : 1; 454 uint8_t Reserved : 4; 455 } f; 456 uint8_t ucBitFields[2]; 457 }; 458 459 union { 460 int16_t seq_scaled_ref_layer_left_offset; 461 int16_t scaled_ref_layer_left_offset; 462 }; 463 union { 464 int16_t seq_scaled_ref_layer_top_offset; 465 int16_t scaled_ref_layer_top_offset; 466 }; 467 union { 468 int16_t seq_scaled_ref_layer_right_offset; 469 int16_t scaled_ref_layer_right_offset; 470 }; 471 union { 472 int16_t seq_scaled_ref_layer_bottom_offset; 473 int16_t scaled_ref_layer_bottom_offset; 474 }; 475 } svcext; 476 }; 477 } NvidiaVulkanParserH264PictureData; 478 479 typedef struct NvidiaVulkanParserH265PictureData 480 { 481 // VPS 482 const StdVideoH265VideoParameterSet* pStdVps; 483 NvidiaParserVideoRefCountBase* pVpsClientObject; 484 485 // SPS 486 const struct vk::StdVideoH265SequenceParameterSet* pStdSps; 487 NvidiaParserVideoRefCountBase* pSpsClientObject; 488 489 // PPS 490 const struct vk::StdVideoH265PictureParameterSet* pStdPps; 491 NvidiaParserVideoRefCountBase* pPpsClientObject; 492 493 uint8_t pic_parameter_set_id; // PPS ID 494 uint8_t seq_parameter_set_id; // SPS ID 495 uint8_t vps_video_parameter_set_id; // VPS ID 496 497 uint8_t IrapPicFlag; 498 uint8_t IdrPicFlag; 499 500 // RefPicSets 501 int32_t NumBitsForShortTermRPSInSlice; 502 int32_t NumDeltaPocsOfRefRpsIdx; 503 int32_t NumPocTotalCurr; 504 int32_t NumPocStCurrBefore; 505 int32_t NumPocStCurrAfter; 506 int32_t NumPocLtCurr; 507 int32_t CurrPicOrderCntVal; 508 INvidiaVulkanPicture* RefPics[16]; 509 int32_t PicOrderCntVal[16]; 510 uint8_t IsLongTerm[16]; // 1=long-term reference 511 int8_t RefPicSetStCurrBefore[8]; 512 int8_t RefPicSetStCurrAfter[8]; 513 int8_t RefPicSetLtCurr[8]; 514 515 // various profile related 516 // 0 = invalid, 1 = Main, 2 = Main10, 3 = still picture, 4 = Main 12, 5 = MV-HEVC Main8 517 uint8_t ProfileLevel; 518 uint8_t ColorPrimaries; // ColorPrimariesBTXXXX enum 519 uint8_t bit_depth_luma_minus8; 520 uint8_t bit_depth_chroma_minus8; 521 522 // MV-HEVC related fields 523 uint8_t mv_hevc_enable; 524 uint8_t nuh_layer_id; 525 uint8_t default_ref_layers_active_flag; 526 uint8_t NumDirectRefLayers; 527 uint8_t max_one_active_ref_layer_flag; 528 uint8_t poc_lsb_not_present_flag; 529 uint8_t pad0[2]; 530 531 int32_t NumActiveRefLayerPics0; 532 int32_t NumActiveRefLayerPics1; 533 int8_t RefPicSetInterLayer0[8]; 534 int8_t RefPicSetInterLayer1[8]; 535 } NvidiaVulkanParserH265PictureData; 536 537 typedef struct NvidiaVulkanParserPictureData 538 { 539 int32_t PicWidthInMbs; // Coded Frame Size 540 int32_t FrameHeightInMbs; // Coded Frame Height 541 INvidiaVulkanPicture* pCurrPic; // Current picture (output) 542 int32_t field_pic_flag; // 0=frame picture, 1=field picture 543 int32_t bottom_field_flag; // 0=top field, 1=bottom field (ignored if field_pic_flag=0) 544 int32_t second_field; // Second field of a complementary field pair 545 int32_t progressive_frame; // Frame is progressive 546 int32_t top_field_first; // Frame pictures only 547 int32_t repeat_first_field; // For 3:2 pulldown (number of additional fields, 2=frame doubling, 4=frame tripling) 548 int32_t ref_pic_flag; // Frame is a reference frame 549 int32_t intra_pic_flag; // Frame is entirely intra coded (no temporal dependencies) 550 int32_t chroma_format; // Chroma Format (should match sequence info) 551 int32_t picture_order_count; // picture order count (if known) 552 uint8_t* pbSideData; // Encryption Info 553 uint32_t nSideDataLen; // Encryption Info length 554 555 // Bitstream data 556 uint32_t nBitstreamDataLen; // Number of bytes in bitstream data buffer 557 uint8_t* pBitstreamData; // Ptr to bitstream data for this picture (slice-layer) 558 uint32_t nNumSlices; // Number of slices(tiles in case of AV1) in this picture 559 const uint32_t* pSliceDataOffsets; // nNumSlices entries, contains offset of each slice within the bitstream data buffer 560 561 // Codec-specific data 562 union 563 { 564 NvidiaVulkanParserH264PictureData h264; 565 NvidiaVulkanParserH265PictureData h265; 566 } CodecSpecific; 567 } NvidiaVulkanParserPictureData; 568 569 // Packet input for parsing 570 typedef struct NvidiaVulkanParserBitstreamPacket 571 { 572 const uint8_t* pByteStream; // Ptr to byte stream data 573 int32_t nDataLength; // Data length for this packet 574 int32_t bEOS; // true if this is an End-Of-Stream packet (flush everything) 575 int32_t bPTSValid; // true if llPTS is valid (also used to detect frame boundaries for VC1 SP/MP) 576 int32_t bDiscontinuity; // true if DecMFT is signalling a discontinuity 577 int32_t bPartialParsing; // 0: parse entire packet, 1: parse until next decode/display event 578 int64_t llPTS; // Presentation Time Stamp for this packet (clock rate specified at initialization) 579 bool bDisablePP; // optional flag for VC1 580 bool bEOP; // true if the packet in pByteStream is exactly one frame 581 uint8_t* pbSideData; // Auxiliary encryption information 582 int32_t nSideDataLength; // Auxiliary encrypton information length 583 } NvidiaVulkanParserBitstreamPacket; 584 585 // Sequence information 586 typedef struct NvidiaVulkanParserSequenceInfo 587 { 588 vk::VkVideoCodecOperationFlagBitsKHR eCodec; // Compression Standard 589 bool isSVC; // h.264 SVC 590 FrameRate frameRate; // Frame Rate stored in the bitstream 591 int32_t bProgSeq; // Progressive Sequence 592 int32_t nDisplayWidth; // Displayed Horizontal Size 593 int32_t nDisplayHeight; // Displayed Vertical Size 594 int32_t nCodedWidth; // Coded Picture Width 595 int32_t nCodedHeight; // Coded Picture Height 596 int32_t nMaxWidth; // Max width within sequence 597 int32_t nMaxHeight; // Max height within sequence 598 uint8_t nChromaFormat; // Chroma Format (0=4:0:0, 1=4:2:0, 2=4:2:2, 3=4:4:4) 599 uint8_t uBitDepthLumaMinus8; // Luma bit depth (0=8bit) 600 uint8_t uBitDepthChromaMinus8; // Chroma bit depth (0=8bit) 601 uint8_t uVideoFullRange; // 0=16-235, 1=0-255 602 int32_t lBitrate; // Video bitrate (bps) 603 int32_t lDARWidth; // Display Aspect Ratio = lDARWidth : lDARHeight 604 int32_t lDARHeight; // Display Aspect Ratio = lDARWidth : lDARHeight 605 int32_t lVideoFormat; // Video Format (VideoFormatXXX) 606 int32_t lColorPrimaries; // Colour Primaries (ColorPrimariesXXX) 607 int32_t lTransferCharacteristics; // Transfer Characteristics 608 int32_t lMatrixCoefficients; // Matrix Coefficients 609 int32_t cbSequenceHeader; // Number of bytes in SequenceHeaderData 610 int32_t nMinNumDpbSlots; // Minimum number of DPB slots for correct decoding 611 int32_t nMinNumDecodeSurfaces; // Minimum number of decode surfaces for correct decoding 612 uint8_t SequenceHeaderData[VK_MAX_SEQ_HDR_LEN]; // Raw sequence header data (codec-specific) 613 uint8_t* pbSideData; // Auxiliary encryption information 614 uint32_t cbSideData; // Auxiliary encryption information length 615 uint32_t codecProfile; // Codec Profile IDC 616 } NvidiaVulkanParserSequenceInfo; 617 618 enum { 619 VK_PARSER_CAPS_MVC = 0x01, 620 VK_PARSER_CAPS_SVC = 0x02, 621 }; 622 623 enum NvidiaParserPictureParametersUpdateType 624 { 625 VK_PICTURE_PARAMETERS_UPDATE_H264_SPS = 0, 626 VK_PICTURE_PARAMETERS_UPDATE_H264_PPS, 627 VK_PICTURE_PARAMETERS_UPDATE_H265_VPS, 628 VK_PICTURE_PARAMETERS_UPDATE_H265_SPS, 629 VK_PICTURE_PARAMETERS_UPDATE_H265_PPS, 630 }; 631 632 typedef struct NvidiaVulkanPictureParameters 633 { 634 NvidiaParserPictureParametersUpdateType updateType; 635 union 636 { 637 const struct vk::StdVideoH264SequenceParameterSet* pH264Sps; 638 const struct vk::StdVideoH264PictureParameterSet* pH264Pps; 639 const struct vk::StdVideoH265VideoParameterSet* pH265Vps; 640 const struct vk::StdVideoH265SequenceParameterSet* pH265Sps; 641 const struct vk::StdVideoH265PictureParameterSet* pH265Pps; 642 }; 643 uint32_t updateSequenceCount; 644 } NvidiaVulkanPictureParameters; 645 646 // Interface to allow decoder to communicate with the client 647 class NvidiaVulkanParserVideoDecodeClient 648 { 649 public: 650 // callback 651 virtual int32_t BeginSequence (const NvidiaVulkanParserSequenceInfo* pnvsi) = 0; // Returns max number of reference frames (always at least 2 for MPEG-2) 652 virtual bool AllocPictureBuffer (INvidiaVulkanPicture** ppNvidiaVulkanPicture) = 0; // Returns a new INvidiaVulkanPicture interface 653 virtual bool DecodePicture (NvidiaVulkanParserPictureData* pNvidiaVulkanParserPictureData) = 0; // Called when a picture is ready to be decoded 654 virtual bool UpdatePictureParameters (NvidiaVulkanPictureParameters* pNvidiaVulkanPictureParameters, 655 NvidiaSharedBaseObj<NvidiaParserVideoRefCountBase>& pictureParametersObject, 656 uint64_t updateSequenceCount) = 0; // Called when a picture is ready to be decoded 657 virtual bool DisplayPicture (INvidiaVulkanPicture* pNvidiaVulkanPicture, 658 int64_t llPTS) = 0; 659 virtual void UnhandledNALU (const uint8_t* pbData, 660 int32_t cbData) = 0; // Called for custom NAL parsing (not required) GetDecodeCaps()661 virtual uint32_t GetDecodeCaps () { return 0; } // NVD_CAPS_XXX GetOperatingPoint(void *)662 virtual int32_t GetOperatingPoint (void* /* pOPInfo */) { return 0; } // called from sequence header of av1 scalable video streams 663 664 protected: ~NvidiaVulkanParserVideoDecodeClient()665 virtual ~NvidiaVulkanParserVideoDecodeClient () {} 666 }; 667 668 // Initialization parameters for decoder class 669 typedef struct NvidiaVulkanParserInitDecodeParameters 670 { 671 uint32_t interfaceVersion; 672 NvidiaVulkanParserVideoDecodeClient* pClient; // should always be present if using parsing functionality 673 uint64_t lReferenceClockRate; // ticks per second of PTS clock (0=default=10000000=10Mhz) 674 int32_t lErrorThreshold; // threshold for deciding to bypass of picture (0=do not decode, 100=always decode) 675 NvidiaVulkanParserSequenceInfo* pExternalSeqInfo; // optional external sequence header data from system layer 676 bool bOutOfBandPictureParameters; // If set, Picture Parameters are going to be provided via UpdatePictureParameters callback 677 } NvidiaVulkanParserInitDecodeParameters; 678 679 // High-level interface to video decoder (Note that parsing and decoding functionality are decoupled from each other) 680 // Some unused parameters has been replaced with void* 681 class NvidiaVulkanVideoDecodeParser : public virtual NvidiaParserVideoRefCountBase 682 { 683 public: 684 virtual VkResult Initialize (NvidiaVulkanParserInitDecodeParameters* pVulkanParserInitDecodeParameters) = 0; 685 virtual bool Deinitialize () = 0; 686 virtual bool DecodePicture (NvidiaVulkanParserPictureData* pVulkanParserPictureData) = 0; 687 virtual bool ParseByteStream (const NvidiaVulkanParserBitstreamPacket* pVulkanParserBitstreamPacket, 688 int32_t* pParsedBytes = NULL) = 0; 689 virtual bool DecodeSliceInfo (void* pVulkanParserSliceInfo, 690 const void* pVulkanParserPictureData, 691 int32_t iSlice) = 0; 692 virtual bool GetDisplayMasteringInfo (void* pVulkanParserDisplayMasteringInfo) = 0; 693 }; 694 695 struct SpsVideoH264PictureParametersSet 696 { 697 vk::StdVideoH264SequenceParameterSet stdSps; 698 vk::StdVideoH264SequenceParameterSetVui stdVui; 699 vk::StdVideoH264ScalingLists spsStdScalingLists; 700 }; 701 702 struct PpsVideoH264PictureParametersSet 703 { 704 vk::StdVideoH264PictureParameterSet stdPps; 705 vk::StdVideoH264ScalingLists ppsStdScalingLists; 706 }; 707 708 struct SpsVideoH265PictureParametersSet 709 { 710 vk::StdVideoH265SequenceParameterSet stdSps; 711 vk::StdVideoH265SequenceParameterSetVui stdVui; 712 vk::StdVideoH265ScalingLists spsStdScalingLists; 713 }; 714 715 struct PpsVideoH265PictureParametersSet 716 { 717 vk::StdVideoH265PictureParameterSet stdPps; 718 vk::StdVideoH265ScalingLists ppsStdScalingLists; 719 }; 720 721 class StdVideoPictureParametersSet : public NvidiaParserVideoRefCountBase 722 { 723 public: 724 Update(NvidiaVulkanPictureParameters * pPictureParameters,uint32_t updateSequenceCount)725 void Update(NvidiaVulkanPictureParameters* pPictureParameters, uint32_t updateSequenceCount) 726 { 727 switch (pPictureParameters->updateType) 728 { 729 case VK_PICTURE_PARAMETERS_UPDATE_H264_SPS: 730 case VK_PICTURE_PARAMETERS_UPDATE_H264_PPS: 731 { 732 733 if (pPictureParameters->updateType == VK_PICTURE_PARAMETERS_UPDATE_H264_SPS) 734 { 735 m_data.h264Sps.stdSps = *pPictureParameters->pH264Sps; 736 if (pPictureParameters->pH264Sps->pScalingLists) 737 { 738 m_data.h264Sps.spsStdScalingLists = *pPictureParameters->pH264Sps->pScalingLists; 739 m_data.h264Sps.stdSps.pScalingLists = &m_data.h264Sps.spsStdScalingLists; 740 } 741 742 if (pPictureParameters->pH264Sps->pSequenceParameterSetVui) 743 { 744 m_data.h264Sps.stdVui = *pPictureParameters->pH264Sps->pSequenceParameterSetVui; 745 m_data.h264Sps.stdSps.pSequenceParameterSetVui = &m_data.h264Sps.stdVui; 746 } 747 } 748 else if (pPictureParameters->updateType == VK_PICTURE_PARAMETERS_UPDATE_H264_PPS) 749 { 750 m_data.h264Pps.stdPps = *pPictureParameters->pH264Pps; 751 752 if (pPictureParameters->pH264Pps->pScalingLists) 753 { 754 m_data.h264Pps.ppsStdScalingLists = *pPictureParameters->pH264Pps->pScalingLists; 755 m_data.h264Pps.stdPps.pScalingLists = &m_data.h264Pps.ppsStdScalingLists; 756 } 757 } 758 759 break; 760 } 761 case VK_PICTURE_PARAMETERS_UPDATE_H265_VPS: 762 { 763 // Vulkan Video Decode APIs do not support VPS parameters 764 break; 765 } 766 767 case VK_PICTURE_PARAMETERS_UPDATE_H265_SPS: 768 case VK_PICTURE_PARAMETERS_UPDATE_H265_PPS: 769 { 770 if (pPictureParameters->updateType == VK_PICTURE_PARAMETERS_UPDATE_H265_SPS) 771 { 772 m_data.h265Sps.stdSps = *pPictureParameters->pH265Sps; 773 if (pPictureParameters->pH265Sps->pScalingLists) 774 { 775 m_data.h265Sps.spsStdScalingLists = *pPictureParameters->pH265Sps->pScalingLists; 776 m_data.h265Sps.stdSps.pScalingLists = &m_data.h265Sps.spsStdScalingLists; 777 } 778 if (pPictureParameters->pH265Sps->pSequenceParameterSetVui) 779 { 780 m_data.h265Sps.stdVui = *pPictureParameters->pH265Sps->pSequenceParameterSetVui; 781 m_data.h265Sps.stdSps.pSequenceParameterSetVui = &m_data.h265Sps.stdVui; 782 } 783 784 } 785 else if (pPictureParameters->updateType == VK_PICTURE_PARAMETERS_UPDATE_H265_PPS) 786 { 787 m_data.h265Pps.stdPps = *pPictureParameters->pH265Pps; 788 789 if (pPictureParameters->pH265Pps->pScalingLists) 790 { 791 m_data.h265Pps.ppsStdScalingLists = *pPictureParameters->pH265Pps->pScalingLists; 792 m_data.h265Pps.stdPps.pScalingLists = &m_data.h265Pps.ppsStdScalingLists; 793 } 794 } 795 796 break; 797 } 798 default: 799 DE_ASSERT(0 && "Invalid Parser format"); 800 } 801 802 m_updateSequenceCount = updateSequenceCount; 803 } 804 GetSpsId(bool & isSps) const805 int32_t GetSpsId(bool& isSps) const 806 { 807 isSps = false; 808 809 switch (m_updateType) 810 { 811 case VK_PICTURE_PARAMETERS_UPDATE_H264_SPS: 812 isSps = true; 813 return m_data.h264Sps.stdSps.seq_parameter_set_id; 814 815 case VK_PICTURE_PARAMETERS_UPDATE_H264_PPS: 816 return m_data.h264Pps.stdPps.seq_parameter_set_id; 817 818 case VK_PICTURE_PARAMETERS_UPDATE_H265_SPS: 819 isSps = true; 820 return m_data.h265Sps.stdSps.sps_seq_parameter_set_id; 821 822 case VK_PICTURE_PARAMETERS_UPDATE_H265_PPS: 823 return m_data.h265Pps.stdPps.pps_seq_parameter_set_id; 824 825 case VK_PICTURE_PARAMETERS_UPDATE_H265_VPS: 826 return m_data.h265Pps.stdPps.pps_seq_parameter_set_id; 827 828 default: 829 DE_ASSERT(0 && "Invalid STD type"); 830 } 831 832 return -1; 833 } 834 GetPpsId(bool & isPps) const835 int32_t GetPpsId(bool& isPps) const 836 { 837 isPps = false; 838 switch (m_updateType) 839 { 840 case VK_PICTURE_PARAMETERS_UPDATE_H264_SPS: 841 break; 842 case VK_PICTURE_PARAMETERS_UPDATE_H264_PPS: 843 isPps = true; 844 return m_data.h264Pps.stdPps.pic_parameter_set_id; 845 case VK_PICTURE_PARAMETERS_UPDATE_H265_SPS: 846 break; 847 case VK_PICTURE_PARAMETERS_UPDATE_H265_PPS: 848 isPps = true; 849 return m_data.h265Pps.stdPps.pps_pic_parameter_set_id; 850 default: 851 DE_ASSERT(0 && "Invalid STD type"); 852 } 853 return -1; 854 } 855 Create(NvidiaVulkanPictureParameters * pPictureParameters,uint64_t updateSequenceCount)856 static StdVideoPictureParametersSet* Create(NvidiaVulkanPictureParameters* pPictureParameters, uint64_t updateSequenceCount) 857 { 858 StdVideoPictureParametersSet* pNewSet = new StdVideoPictureParametersSet(pPictureParameters->updateType); 859 860 pNewSet->Update(pPictureParameters, (uint32_t)updateSequenceCount); 861 862 return pNewSet; 863 } 864 StdVideoPictureParametersSetFromBase(NvidiaParserVideoRefCountBase * pBase)865 static StdVideoPictureParametersSet* StdVideoPictureParametersSetFromBase(NvidiaParserVideoRefCountBase* pBase) 866 { 867 if (!pBase) 868 return DE_NULL; 869 870 StdVideoPictureParametersSet* result = dynamic_cast<StdVideoPictureParametersSet*>(pBase); 871 872 if (result) 873 return result; 874 875 TCU_THROW(InternalError, "Invalid StdVideoPictureParametersSet from pBase"); 876 } 877 AddRef()878 virtual int32_t AddRef() 879 { 880 return ++m_refCount; 881 } 882 Release()883 virtual int32_t Release() 884 { 885 uint32_t ret = --m_refCount; 886 887 // Destroy the device if refcount reaches zero 888 if (ret == 0) 889 { 890 delete this; 891 } 892 893 return ret; 894 } 895 896 private: 897 std::atomic<int32_t> m_refCount; 898 899 public: 900 NvidiaParserPictureParametersUpdateType m_updateType; 901 union 902 { 903 SpsVideoH264PictureParametersSet h264Sps; 904 PpsVideoH264PictureParametersSet h264Pps; 905 SpsVideoH265PictureParametersSet h265Sps; 906 PpsVideoH265PictureParametersSet h265Pps; 907 } m_data; 908 909 uint32_t m_updateSequenceCount; 910 NvidiaSharedBaseObj<NvidiaParserVideoRefCountBase> m_vkObjectOwner; // NvidiaParserVideoPictureParameters 911 vk::VkVideoSessionKHR m_vkVideoDecodeSession; 912 private: 913 StdVideoPictureParametersSet(NvidiaParserPictureParametersUpdateType updateType)914 StdVideoPictureParametersSet (NvidiaParserPictureParametersUpdateType updateType) 915 : m_refCount (0) 916 , m_updateType (updateType) 917 , m_data () 918 , m_updateSequenceCount (0) 919 , m_vkVideoDecodeSession (DE_NULL) 920 { 921 } 922 ~StdVideoPictureParametersSet()923 ~StdVideoPictureParametersSet() 924 { 925 m_vkObjectOwner = nullptr; 926 m_vkVideoDecodeSession = vk::VkVideoSessionKHR(); 927 } 928 929 }; 930 931 struct VulkanParserDetectedVideoFormat 932 { 933 vk::VkVideoCodecOperationFlagBitsKHR codec; // Compression format 934 uint32_t codecProfile; 935 VkVideoComponentBitDepthFlagsKHR lumaBitDepth; 936 VkVideoComponentBitDepthFlagsKHR chromaBitDepth; 937 VkVideoChromaSubsamplingFlagBitsKHR chromaSubsampling; 938 uint32_t frame_rate_numerator; // Frame rate denominator (0 = unspecified or variable frame rate) 939 uint32_t frame_rate_denominator; 940 uint8_t sequenceUpdate : 1; // if true, this is a sequence update and not the first time StartVideoSequence is being called. 941 uint8_t sequenceReconfigireFormat : 1; // if true, this is a sequence update for the video format. 942 uint8_t sequenceReconfigireCodedExtent : 1; // if true, this is a sequence update for the video coded extent. 943 uint8_t progressive_sequence : 1; // false = interlaced, true = progressive 944 uint8_t bit_depth_luma_minus8; // high bit depth luma. E.g, 2 for 10-bitdepth, 4 for 12-bitdepth 945 uint8_t bit_depth_chroma_minus8; // high bit depth chroma. E.g, 2 for 10-bitdepth, 4 for 12-bitdepth 946 uint8_t reserved1; // Reserved for future use 947 uint32_t coded_width; // Coded frame width in pixels 948 uint32_t coded_height; // Coded frame height in pixels 949 950 struct 951 { 952 int32_t left; // left position of display rect 953 int32_t top; // top position of display rect 954 int32_t right; // right position of display rect 955 int32_t bottom; // bottom position of display rect 956 } display_area; 957 958 uint32_t bitrate; // Video bitrate (bps, 0=unknown) 959 int32_t display_aspect_ratio_x; 960 int32_t display_aspect_ratio_y; 961 uint32_t minNumDecodeSurfaces; // Minimum number of decode surfaces for correct decoding (NumRefFrames + 2;) 962 uint32_t maxNumDpbSlots; // Can't be more than 16 + 1 963 964 struct 965 { 966 uint8_t video_format : 3; // 0-Component, 1-PAL, 2-NTSC, 3-SECAM, 4-MAC, 5-Unspecified 967 uint8_t video_full_range_flag : 1; // indicates the black level and luma and chroma range 968 uint8_t reserved_zero_bits : 4; // Reserved bits 969 uint8_t color_primaries; // Chromaticity coordinates of source primaries 970 uint8_t transfer_characteristics; // Opto-electronic transfer characteristic of the source picture 971 uint8_t matrix_coefficients; // used in deriving luma and chroma signals from RGB primaries 972 } video_signal_description; // Video Signal Description. Refer section E.2.1 (VUI parameters semantics) of H264 spec file 973 uint32_t seqhdr_data_length; // Additional bytes following (NVVIDEOFORMATEX) 974 }; 975 976 union VulkanParserFieldFlags 977 { 978 struct 979 { 980 uint32_t progressiveFrame : 1; // Frame is progressive 981 uint32_t fieldPic : 1; // 0 = frame picture, 1 = field picture 982 uint32_t bottomField : 1; // 0 = top field, 1 = bottom field (ignored if field_pic_flag=0) 983 uint32_t secondField : 1; // Second field of a complementary field pair 984 uint32_t topFieldFirst : 1; // Frame pictures only 985 uint32_t unpairedField : 1; // Incomplete (half) frame. 986 uint32_t syncFirstReady : 1; // Synchronize the second field to the first one. 987 uint32_t syncToFirstField : 1; // Synchronize the second field to the first one. 988 uint32_t repeatFirstField : 3; // For 3:2 pulldown (number of additional fields, 2 = frame doubling, 4 = frame tripling) 989 uint32_t refPic : 1; // Frame is a reference frame 990 }; 991 uint32_t fieldFlags; 992 }; 993 994 struct VulkanParserFrameSyncinfo 995 { 996 uint32_t unpairedField : 1; // Generate a semaphore reference, do not return the semaphore. 997 uint32_t syncToFirstField : 1; // Use the semaphore from the unpared field to wait on. 998 void* pDebugInterface; 999 }; 1000 1001 struct VulkanParserDecodePictureInfo 1002 { 1003 int32_t displayWidth; 1004 int32_t displayHeight; 1005 int32_t pictureIndex; // Index of the current picture 1006 VulkanParserFieldFlags flags; 1007 VulkanParserFrameSyncinfo frameSyncinfo; 1008 uint16_t videoFrameType; // VideoFrameType - use Vulkan codec specific type pd->CodecSpecific.h264.slice_type. 1009 uint16_t viewId; // from pictureInfoH264->ext.mvcext.view_id 1010 }; 1011 1012 struct PerFrameDecodeParameters 1013 { 1014 enum 1015 { 1016 MAX_DPB_REF_SLOTS = 16 + 1 1017 }; 1018 1019 int currPicIdx; // Output index of the current picture 1020 StdVideoPictureParametersSet* pCurrentPictureParameters; 1021 unsigned int bitstreamDataLen; // Number of bytes in bitstream data buffer 1022 const unsigned char* pBitstreamData; // ptr to bitstream data for this picture (slice-layer) 1023 vk::VkVideoDecodeInfoKHR decodeFrameInfo; 1024 int32_t numGopReferenceSlots; 1025 int8_t pGopReferenceImagesIndexes[MAX_DPB_REF_SLOTS]; 1026 vk::VkVideoPictureResourceInfoKHR pictureResources[MAX_DPB_REF_SLOTS]; 1027 }; 1028 1029 } // video 1030 } // vkt 1031 1032 #endif // _EXTNVIDIAVIDEOPARSERIF_HPP 1033