1 // Copyright 2023 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #[cfg(test)] 6 mod dummy; 7 #[cfg(feature = "v4l2")] 8 mod v4l2; 9 #[cfg(feature = "vaapi")] 10 mod vaapi; 11 12 use std::cell::RefCell; 13 use std::io::Cursor; 14 use std::os::fd::AsFd; 15 use std::os::fd::BorrowedFd; 16 use std::rc::Rc; 17 18 use anyhow::anyhow; 19 use anyhow::Context; 20 21 use crate::codec::h265::dpb::Dpb; 22 use crate::codec::h265::dpb::DpbEntry; 23 use crate::codec::h265::parser::Nalu; 24 use crate::codec::h265::parser::NaluType; 25 use crate::codec::h265::parser::Parser; 26 use crate::codec::h265::parser::Pps; 27 use crate::codec::h265::parser::ShortTermRefPicSet; 28 use crate::codec::h265::parser::Slice; 29 use crate::codec::h265::parser::SliceHeader; 30 use crate::codec::h265::parser::Sps; 31 use crate::codec::h265::picture::PictureData; 32 use crate::codec::h265::picture::Reference; 33 use crate::decoder::stateless::DecodeError; 34 use crate::decoder::stateless::DecodingState; 35 use crate::decoder::stateless::NewPictureResult; 36 use crate::decoder::stateless::StatelessBackendResult; 37 use crate::decoder::stateless::StatelessCodec; 38 use crate::decoder::stateless::StatelessDecoder; 39 use crate::decoder::stateless::StatelessDecoderBackend; 40 use crate::decoder::stateless::StatelessDecoderBackendPicture; 41 use crate::decoder::stateless::StatelessVideoDecoder; 42 use crate::decoder::BlockingMode; 43 use crate::decoder::DecodedHandle; 44 use crate::decoder::DecoderEvent; 45 use crate::decoder::StreamInfo; 46 use crate::Resolution; 47 48 const MAX_DPB_SIZE: usize = 16; 49 /// Stateless backend methods specific to H.265. 50 pub trait StatelessH265DecoderBackend: 51 StatelessDecoderBackend + StatelessDecoderBackendPicture<H265> 52 { 53 /// Called when a new SPS is parsed. new_sequence(&mut self, sps: &Sps) -> StatelessBackendResult<()>54 fn new_sequence(&mut self, sps: &Sps) -> StatelessBackendResult<()>; 55 56 /// Called when the decoder determines that a frame or field was found. new_picture( &mut self, timestamp: u64, alloc_cb: &mut dyn FnMut() -> Option< <<Self as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, >, ) -> NewPictureResult<Self::Picture>57 fn new_picture( 58 &mut self, 59 timestamp: u64, 60 alloc_cb: &mut dyn FnMut() -> Option< 61 <<Self as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, 62 >, 63 ) -> NewPictureResult<Self::Picture>; 64 65 /// Called by the decoder for every frame or field found. 66 #[allow(clippy::too_many_arguments)] begin_picture( &mut self, picture: &mut Self::Picture, picture_data: &PictureData, sps: &Sps, pps: &Pps, dpb: &Dpb<Self::Handle>, rps: &RefPicSet<Self::Handle>, slice: &Slice, ) -> StatelessBackendResult<()>67 fn begin_picture( 68 &mut self, 69 picture: &mut Self::Picture, 70 picture_data: &PictureData, 71 sps: &Sps, 72 pps: &Pps, 73 dpb: &Dpb<Self::Handle>, 74 rps: &RefPicSet<Self::Handle>, 75 slice: &Slice, 76 ) -> StatelessBackendResult<()>; 77 78 /// Called to dispatch a decode operation to the backend. 79 #[allow(clippy::too_many_arguments)] decode_slice( &mut self, picture: &mut Self::Picture, slice: &Slice, sps: &Sps, pps: &Pps, ref_pic_list0: &[Option<RefPicListEntry<Self::Handle>>; 16], ref_pic_list1: &[Option<RefPicListEntry<Self::Handle>>; 16], ) -> StatelessBackendResult<()>80 fn decode_slice( 81 &mut self, 82 picture: &mut Self::Picture, 83 slice: &Slice, 84 sps: &Sps, 85 pps: &Pps, 86 ref_pic_list0: &[Option<RefPicListEntry<Self::Handle>>; 16], 87 ref_pic_list1: &[Option<RefPicListEntry<Self::Handle>>; 16], 88 ) -> StatelessBackendResult<()>; 89 90 /// Called when the decoder wants the backend to finish the decoding 91 /// operations for `picture`. At this point, `decode_slice` has been called 92 /// for all slices. submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle>93 fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle>; 94 } 95 96 /// An entry in the Reference Picture Lists. Unlike H.264, H.265 can use the 97 /// current picture itself as a reference. 98 #[derive(Clone)] 99 pub enum RefPicListEntry<T> { 100 CurrentPicture(PictureData), 101 DpbEntry(DpbEntry<T>), 102 } 103 104 enum BumpingType { 105 BeforeDecoding, 106 AfterDecoding, 107 } 108 109 enum RenegotiationType<'a> { 110 CurrentSps, 111 NewSps(&'a Sps), 112 } 113 114 /// Keeps track of the last values seen for negotiation purposes. 115 #[derive(Clone, Debug, Default, PartialEq, Eq)] 116 struct NegotiationInfo { 117 /// The current coded resolution 118 coded_resolution: Resolution, 119 general_profile_idc: u8, 120 bit_depth_luma_minus8: u8, 121 bit_depth_chroma_minus8: u8, 122 chroma_format_idc: u8, 123 } 124 125 impl From<&Sps> for NegotiationInfo { from(sps: &Sps) -> Self126 fn from(sps: &Sps) -> Self { 127 NegotiationInfo { 128 coded_resolution: Resolution { width: sps.width().into(), height: sps.height().into() }, 129 general_profile_idc: sps.profile_tier_level.general_profile_idc, 130 bit_depth_luma_minus8: sps.bit_depth_luma_minus8, 131 bit_depth_chroma_minus8: sps.bit_depth_chroma_minus8, 132 chroma_format_idc: sps.chroma_format_idc, 133 } 134 } 135 } 136 137 /// The RefPicSet data, derived once per picture. 138 #[derive(Clone, Debug)] 139 pub struct RefPicSet<T> { 140 curr_delta_poc_msb_present_flag: [bool; MAX_DPB_SIZE], 141 foll_delta_poc_msb_present_flag: [bool; MAX_DPB_SIZE], 142 143 num_poc_st_curr_before: usize, 144 num_poc_st_curr_after: usize, 145 num_poc_st_foll: usize, 146 num_poc_lt_curr: usize, 147 num_poc_lt_foll: usize, 148 149 poc_st_curr_before: [i32; MAX_DPB_SIZE], 150 poc_st_curr_after: [i32; MAX_DPB_SIZE], 151 poc_st_foll: [i32; MAX_DPB_SIZE], 152 poc_lt_curr: [i32; MAX_DPB_SIZE], 153 poc_lt_foll: [i32; MAX_DPB_SIZE], 154 155 ref_pic_set_lt_curr: [Option<DpbEntry<T>>; MAX_DPB_SIZE], 156 ref_pic_set_st_curr_after: [Option<DpbEntry<T>>; MAX_DPB_SIZE], 157 ref_pic_set_st_curr_before: [Option<DpbEntry<T>>; MAX_DPB_SIZE], 158 ref_pic_set_st_foll: [Option<DpbEntry<T>>; MAX_DPB_SIZE], 159 ref_pic_set_lt_foll: [Option<DpbEntry<T>>; MAX_DPB_SIZE], 160 } 161 162 impl<T> RefPicSet<T> 163 where 164 T: Clone, 165 { 166 // See 8.3.4. 167 // Builds the reference picture list for `hdr` for P and B slices. build_ref_pic_lists( &self, hdr: &SliceHeader, pps: &Pps, cur_pic: &PictureData, ) -> anyhow::Result<ReferencePicLists<T>>168 fn build_ref_pic_lists( 169 &self, 170 hdr: &SliceHeader, 171 pps: &Pps, 172 cur_pic: &PictureData, 173 ) -> anyhow::Result<ReferencePicLists<T>> { 174 let mut ref_pic_lists = ReferencePicLists::default(); 175 176 // I slices do not use inter prediction. 177 if !hdr.type_.is_p() && !hdr.type_.is_b() { 178 return Ok(ref_pic_lists); 179 } 180 181 if self.num_poc_st_curr_before == 0 182 && self.num_poc_st_curr_after == 0 183 && self.num_poc_lt_curr == 0 184 && pps.scc_extension_flag 185 && !pps.scc_extension.curr_pic_ref_enabled_flag 186 { 187 // Let's try and keep going, if it is a broken stream then maybe it 188 // will sort itself out as we go. In any case, we must not loop 189 // infinitely here. 190 log::error!("Bug or broken stream: out of pictures and can't build ref pic lists."); 191 return Ok(ref_pic_lists); 192 } 193 194 let rplm = &hdr.ref_pic_list_modification; 195 196 let num_rps_curr_temp_list0 = 197 std::cmp::max(u32::from(hdr.num_ref_idx_l0_active_minus1) + 1, hdr.num_pic_total_curr); 198 199 // This could be simplified using a Vec, but lets not change the 200 // algorithm from the spec too much. 201 let mut ref_pic_list_temp0: [Option<RefPicListEntry<T>>; MAX_DPB_SIZE] = Default::default(); 202 203 // Equation 8-8 204 let mut r_idx = 0; 205 assert!(num_rps_curr_temp_list0 as usize <= MAX_DPB_SIZE); 206 while r_idx < num_rps_curr_temp_list0 { 207 let mut i = 0; 208 while i < self.num_poc_st_curr_before && r_idx < num_rps_curr_temp_list0 { 209 ref_pic_list_temp0[r_idx as usize] = 210 self.ref_pic_set_st_curr_before[i].clone().map(RefPicListEntry::DpbEntry); 211 212 i += 1; 213 r_idx += 1; 214 } 215 216 let mut i = 0; 217 while i < self.num_poc_st_curr_after && r_idx < num_rps_curr_temp_list0 { 218 ref_pic_list_temp0[r_idx as usize] = 219 self.ref_pic_set_st_curr_after[i].clone().map(RefPicListEntry::DpbEntry); 220 221 i += 1; 222 r_idx += 1; 223 } 224 225 let mut i = 0; 226 while i < self.num_poc_lt_curr && r_idx < num_rps_curr_temp_list0 { 227 ref_pic_list_temp0[r_idx as usize] = 228 self.ref_pic_set_lt_curr[i].clone().map(RefPicListEntry::DpbEntry); 229 230 i += 1; 231 r_idx += 1; 232 } 233 234 if pps.scc_extension.curr_pic_ref_enabled_flag { 235 ref_pic_list_temp0[r_idx as usize] = 236 Some(RefPicListEntry::CurrentPicture(cur_pic.clone())); 237 238 r_idx += 1; 239 } 240 } 241 242 // Equation 8-9 243 for r_idx in 0..=usize::from(hdr.num_ref_idx_l0_active_minus1) { 244 let entry = if rplm.ref_pic_list_modification_flag_l0 { 245 let idx = rplm.list_entry_l0[r_idx]; 246 ref_pic_list_temp0[idx as usize].clone() 247 } else { 248 ref_pic_list_temp0[r_idx].clone() 249 }; 250 251 ref_pic_lists.ref_pic_list0[r_idx] = entry; 252 } 253 254 if pps.scc_extension.curr_pic_ref_enabled_flag 255 && !rplm.ref_pic_list_modification_flag_l0 256 && num_rps_curr_temp_list0 > (u32::from(hdr.num_ref_idx_l0_active_minus1) + 1) 257 { 258 ref_pic_lists.ref_pic_list0[r_idx as usize] = 259 Some(RefPicListEntry::CurrentPicture(cur_pic.clone())); 260 } 261 262 if hdr.type_.is_b() { 263 let mut ref_pic_list_temp1: [Option<RefPicListEntry<T>>; MAX_DPB_SIZE] = 264 Default::default(); 265 266 let num_rps_curr_temp_list1 = std::cmp::max( 267 u32::from(hdr.num_ref_idx_l1_active_minus1) + 1, 268 hdr.num_pic_total_curr, 269 ); 270 271 // Equation 8-10 272 let mut r_idx = 0; 273 assert!(num_rps_curr_temp_list1 as usize <= MAX_DPB_SIZE); 274 while r_idx < num_rps_curr_temp_list1 { 275 let mut i = 0; 276 while i < self.num_poc_st_curr_after && r_idx < num_rps_curr_temp_list1 { 277 ref_pic_list_temp1[r_idx as usize] = 278 self.ref_pic_set_st_curr_after[i].clone().map(RefPicListEntry::DpbEntry); 279 i += 1; 280 r_idx += 1; 281 } 282 283 let mut i = 0; 284 while i < self.num_poc_st_curr_before && r_idx < num_rps_curr_temp_list1 { 285 ref_pic_list_temp1[r_idx as usize] = 286 self.ref_pic_set_st_curr_before[i].clone().map(RefPicListEntry::DpbEntry); 287 i += 1; 288 r_idx += 1; 289 } 290 291 let mut i = 0; 292 while i < self.num_poc_lt_curr && r_idx < num_rps_curr_temp_list1 { 293 ref_pic_list_temp1[r_idx as usize] = 294 self.ref_pic_set_lt_curr[i].clone().map(RefPicListEntry::DpbEntry); 295 i += 1; 296 r_idx += 1; 297 } 298 299 if pps.scc_extension.curr_pic_ref_enabled_flag { 300 ref_pic_list_temp1[r_idx as usize] = 301 Some(RefPicListEntry::CurrentPicture(cur_pic.clone())); 302 303 r_idx += 1; 304 } 305 } 306 307 // Equation 8-11 308 for r_idx in 0..=usize::from(hdr.num_ref_idx_l1_active_minus1) { 309 let entry = if rplm.ref_pic_list_modification_flag_l1 { 310 let idx = rplm.list_entry_l1[r_idx]; 311 ref_pic_list_temp1[idx as usize].clone() 312 } else { 313 ref_pic_list_temp1[r_idx].clone() 314 }; 315 316 ref_pic_lists.ref_pic_list1[r_idx] = entry; 317 } 318 } 319 320 Ok(ref_pic_lists) 321 } 322 } 323 324 impl<T: Clone> Default for RefPicSet<T> { default() -> Self325 fn default() -> Self { 326 Self { 327 curr_delta_poc_msb_present_flag: Default::default(), 328 foll_delta_poc_msb_present_flag: Default::default(), 329 num_poc_st_curr_before: Default::default(), 330 num_poc_st_curr_after: Default::default(), 331 num_poc_st_foll: Default::default(), 332 num_poc_lt_curr: Default::default(), 333 num_poc_lt_foll: Default::default(), 334 poc_st_curr_before: Default::default(), 335 poc_st_curr_after: Default::default(), 336 poc_st_foll: Default::default(), 337 poc_lt_curr: Default::default(), 338 poc_lt_foll: Default::default(), 339 ref_pic_set_lt_curr: Default::default(), 340 ref_pic_set_st_curr_after: Default::default(), 341 ref_pic_set_st_curr_before: Default::default(), 342 ref_pic_set_st_foll: Default::default(), 343 ref_pic_set_lt_foll: Default::default(), 344 } 345 } 346 } 347 348 /// State of the picture being currently decoded. 349 /// 350 /// Stored between calls to [`StatelessDecoder::handle_slice`] that belong to the same picture. 351 struct CurrentPicState<H: DecodedHandle, P> { 352 /// Data for the current picture as extracted from the stream. 353 pic: PictureData, 354 /// Backend-specific data for that picture. 355 backend_pic: P, 356 /// List of reference pictures, used once per slice. 357 ref_pic_lists: ReferencePicLists<H>, 358 } 359 360 /// All the reference picture lists used to decode a stream. 361 struct ReferencePicLists<T> { 362 /// Reference picture list 0 for P and B slices. Retains the same meaning as 363 /// in the specification. Points into the pictures stored in the DPB. 364 /// Derived once per slice. 365 ref_pic_list0: [Option<RefPicListEntry<T>>; MAX_DPB_SIZE], 366 /// Reference picture list 1 for B slices. Retains the same meaning as in 367 /// the specification. Points into the pictures stored in the DPB. Derived 368 /// once per slice. 369 ref_pic_list1: [Option<RefPicListEntry<T>>; MAX_DPB_SIZE], 370 } 371 372 impl<T> Default for ReferencePicLists<T> { default() -> Self373 fn default() -> Self { 374 Self { ref_pic_list0: Default::default(), ref_pic_list1: Default::default() } 375 } 376 } 377 378 pub struct H265DecoderState<H: DecodedHandle, P> { 379 /// A parser to extract bitstream metadata 380 parser: Parser, 381 382 /// Keeps track of the last values seen for negotiation purposes. 383 negotiation_info: NegotiationInfo, 384 /// The set of reference pictures. 385 rps: RefPicSet<H>, 386 387 /// The decoded picture buffer 388 dpb: Dpb<H>, 389 390 /// The current active SPS id. 391 cur_sps_id: u8, 392 393 /// Used to identify first picture in decoding order or first picture that 394 /// follows an EOS NALU. 395 first_picture_after_eos: bool, 396 397 /// Whether this is the first picture in the bitstream in decoding order. 398 first_picture_in_bitstream: bool, 399 // Same as PrevTid0Pic in the specification. 400 prev_tid_0_pic: Option<PictureData>, 401 402 /// A H.265 syntax element. 403 max_pic_order_cnt_lsb: i32, 404 /// The value of NoRaslOutputFlag for the last IRAP picture. 405 irap_no_rasl_output_flag: bool, 406 407 /// We keep track of the last independent header so we can copy that into 408 /// dependent slices. 409 last_independent_slice_header: Option<SliceHeader>, 410 411 /// The picture currently being decoded. We need to preserve it between 412 /// calls to `decode` because multiple slices will be processed in different 413 /// calls to `decode`. 414 current_pic: Option<CurrentPicState<H, P>>, 415 416 pending_pps: Vec<Nalu<'static>>, 417 } 418 419 impl<H, P> Default for H265DecoderState<H, P> 420 where 421 H: DecodedHandle + Clone, 422 { default() -> Self423 fn default() -> Self { 424 H265DecoderState { 425 parser: Default::default(), 426 negotiation_info: Default::default(), 427 rps: Default::default(), 428 dpb: Default::default(), 429 cur_sps_id: Default::default(), 430 first_picture_after_eos: true, 431 first_picture_in_bitstream: true, 432 prev_tid_0_pic: Default::default(), 433 max_pic_order_cnt_lsb: Default::default(), 434 irap_no_rasl_output_flag: Default::default(), 435 last_independent_slice_header: Default::default(), 436 current_pic: Default::default(), 437 pending_pps: Default::default(), 438 } 439 } 440 } 441 442 /// [`StatelessCodec`] structure to use in order to create a H.265 stateless decoder. 443 /// 444 /// # Accepted input 445 /// 446 /// A decoder using this codec processes exactly one NAL unit of input per call to 447 /// [`StatelessDecoder::decode`], and returns the number of bytes until the end of this NAL unit. 448 /// This makes it possible to call [`Decode`](StatelessDecoder::decode) repeatedly on some unsplit 449 /// Annex B stream and shrinking it by the number of bytes processed after each call, until the 450 /// stream ends up being empty. 451 pub struct H265; 452 453 impl StatelessCodec for H265 { 454 type FormatInfo = Sps; 455 type DecoderState<H: DecodedHandle, P> = H265DecoderState<H, P>; 456 } 457 458 impl<B> StatelessDecoder<H265, B> 459 where 460 B: StatelessH265DecoderBackend, 461 B::Handle: Clone, 462 { 463 /// Whether the stream parameters have changed, indicating that a negotiation window has opened. negotiation_possible( sps: &Sps, dpb: &Dpb<B::Handle>, old_negotiation_info: &NegotiationInfo, ) -> bool464 fn negotiation_possible( 465 sps: &Sps, 466 dpb: &Dpb<B::Handle>, 467 old_negotiation_info: &NegotiationInfo, 468 ) -> bool { 469 let negotiation_info = NegotiationInfo::from(sps); 470 let max_dpb_size = std::cmp::min(sps.max_dpb_size(), 16); 471 let prev_max_dpb_size = dpb.max_num_pics(); 472 473 *old_negotiation_info != negotiation_info || prev_max_dpb_size != max_dpb_size 474 } 475 476 /// Apply the parameters of `sps` to the decoder. apply_sps(&mut self, sps: &Sps) -> anyhow::Result<()>477 fn apply_sps(&mut self, sps: &Sps) -> anyhow::Result<()> { 478 self.drain()?; 479 self.codec.negotiation_info = NegotiationInfo::from(sps); 480 481 let max_dpb_size = std::cmp::min(sps.max_dpb_size(), 16); 482 self.codec.dpb.set_max_num_pics(max_dpb_size); 483 self.coded_resolution = 484 Resolution { width: u32::from(sps.width()), height: u32::from(sps.height()) }; 485 Ok(()) 486 } 487 488 // See 8.3.2, Note 2. st_ref_pic_set<'a>(hdr: &'a SliceHeader, sps: &'a Sps) -> &'a ShortTermRefPicSet489 fn st_ref_pic_set<'a>(hdr: &'a SliceHeader, sps: &'a Sps) -> &'a ShortTermRefPicSet { 490 if hdr.curr_rps_idx == sps.num_short_term_ref_pic_sets { 491 &hdr.short_term_ref_pic_set 492 } else { 493 &sps.short_term_ref_pic_set[usize::from(hdr.curr_rps_idx)] 494 } 495 } 496 497 // See 8.3.2. decode_rps( &mut self, slice: &Slice, sps: &Sps, cur_pic: &PictureData, ) -> anyhow::Result<()>498 fn decode_rps( 499 &mut self, 500 slice: &Slice, 501 sps: &Sps, 502 cur_pic: &PictureData, 503 ) -> anyhow::Result<()> { 504 let hdr = &slice.header; 505 506 if cur_pic.nalu_type.is_irap() && cur_pic.no_rasl_output_flag { 507 self.codec.dpb.mark_all_as_unused_for_ref(); 508 } 509 510 if slice.nalu.header.type_.is_idr() { 511 self.codec.rps.poc_st_curr_before = Default::default(); 512 self.codec.rps.poc_st_curr_after = Default::default(); 513 self.codec.rps.poc_st_foll = Default::default(); 514 self.codec.rps.poc_lt_curr = Default::default(); 515 self.codec.rps.poc_lt_foll = Default::default(); 516 517 self.codec.rps.num_poc_st_curr_before = 0; 518 self.codec.rps.num_poc_st_curr_after = 0; 519 self.codec.rps.num_poc_st_foll = 0; 520 self.codec.rps.num_poc_lt_curr = 0; 521 self.codec.rps.num_poc_lt_foll = 0; 522 } else { 523 let curr_st_rps = Self::st_ref_pic_set(hdr, sps); 524 let mut j = 0; 525 let mut k = 0; 526 for i in 0..usize::from(curr_st_rps.num_negative_pics) { 527 let poc = cur_pic.pic_order_cnt_val + curr_st_rps.delta_poc_s0[i]; 528 529 if curr_st_rps.used_by_curr_pic_s0[i] { 530 self.codec.rps.poc_st_curr_before[j] = poc; 531 j += 1; 532 } else { 533 self.codec.rps.poc_st_foll[k] = poc; 534 k += 1; 535 } 536 } 537 538 self.codec.rps.num_poc_st_curr_before = j as _; 539 540 let mut j = 0; 541 for i in 0..usize::from(curr_st_rps.num_positive_pics) { 542 let poc = cur_pic.pic_order_cnt_val + curr_st_rps.delta_poc_s1[i]; 543 544 if curr_st_rps.used_by_curr_pic_s1[i] { 545 self.codec.rps.poc_st_curr_after[j] = poc; 546 j += 1; 547 } else { 548 self.codec.rps.poc_st_foll[k] = poc; 549 k += 1; 550 } 551 } 552 553 self.codec.rps.num_poc_st_curr_after = j as _; 554 self.codec.rps.num_poc_st_foll = k as _; 555 556 let mut j = 0; 557 let mut k = 0; 558 for i in 0..usize::from(hdr.num_long_term_sps + hdr.num_long_term_pics) { 559 let mut poc_lt = hdr.poc_lsb_lt[i] as i32; 560 if hdr.delta_poc_msb_present_flag[i] { 561 poc_lt += cur_pic.pic_order_cnt_val; 562 let delta_poc = 563 hdr.delta_poc_msb_cycle_lt[i] as i32 * self.codec.max_pic_order_cnt_lsb; 564 565 poc_lt -= delta_poc; 566 poc_lt -= cur_pic.pic_order_cnt_val & (self.codec.max_pic_order_cnt_lsb - 1); 567 } 568 569 if hdr.used_by_curr_pic_lt[i] { 570 self.codec.rps.poc_lt_curr[j] = poc_lt; 571 self.codec.rps.curr_delta_poc_msb_present_flag[j] = 572 hdr.delta_poc_msb_present_flag[i]; 573 j += 1; 574 } else { 575 self.codec.rps.poc_lt_foll[k] = poc_lt; 576 self.codec.rps.foll_delta_poc_msb_present_flag[k] = 577 hdr.delta_poc_msb_present_flag[i]; 578 k += 1; 579 } 580 } 581 582 self.codec.rps.num_poc_lt_curr = j as _; 583 self.codec.rps.num_poc_lt_foll = k as _; 584 } 585 586 self.derive_and_mark_rps()?; 587 Ok(()) 588 } 589 590 // See the derivation process in the second half of 8.3.2. derive_and_mark_rps(&mut self) -> anyhow::Result<()>591 fn derive_and_mark_rps(&mut self) -> anyhow::Result<()> { 592 let max_pic_order_cnt_lsb = self.codec.max_pic_order_cnt_lsb; 593 594 // Equation 8-6 595 for i in 0..self.codec.rps.num_poc_lt_curr { 596 if !self.codec.rps.curr_delta_poc_msb_present_flag[i] { 597 let poc = self.codec.rps.poc_lt_curr[i]; 598 let mask = max_pic_order_cnt_lsb - 1; 599 let reference = self.codec.dpb.find_ref_by_poc_masked(poc, mask); 600 601 if reference.is_none() { 602 log::warn!("No reference found for poc {} and mask {}", poc, mask); 603 } 604 605 self.codec.rps.ref_pic_set_lt_curr[i] = reference; 606 } else { 607 let poc = self.codec.rps.poc_lt_curr[i]; 608 let reference = self.codec.dpb.find_ref_by_poc(poc); 609 610 if reference.is_none() { 611 log::warn!("No reference found for poc {}", poc); 612 } 613 614 self.codec.rps.ref_pic_set_lt_curr[i] = reference; 615 } 616 } 617 618 for i in 0..self.codec.rps.num_poc_lt_foll { 619 if !self.codec.rps.foll_delta_poc_msb_present_flag[i] { 620 let poc = self.codec.rps.poc_lt_foll[i]; 621 let mask = max_pic_order_cnt_lsb - 1; 622 let reference = self.codec.dpb.find_ref_by_poc_masked(poc, mask); 623 624 if reference.is_none() { 625 log::warn!("No reference found for poc {} and mask {}", poc, mask); 626 } 627 628 self.codec.rps.ref_pic_set_lt_foll[i] = reference; 629 } else { 630 let poc = self.codec.rps.poc_lt_foll[i]; 631 let reference = self.codec.dpb.find_ref_by_poc(poc); 632 633 if reference.is_none() { 634 log::warn!("No reference found for poc {}", poc); 635 } 636 637 self.codec.rps.ref_pic_set_lt_foll[i] = reference; 638 } 639 } 640 641 for pic in self.codec.rps.ref_pic_set_lt_curr.iter().flatten() { 642 pic.0.borrow_mut().set_reference(Reference::LongTerm); 643 } 644 645 for pic in self.codec.rps.ref_pic_set_lt_foll.iter().flatten() { 646 pic.0.borrow_mut().set_reference(Reference::LongTerm); 647 } 648 649 // Equation 8-7 650 for i in 0..self.codec.rps.num_poc_st_curr_before { 651 let poc = self.codec.rps.poc_st_curr_before[i]; 652 let reference = self.codec.dpb.find_short_term_ref_by_poc(poc); 653 654 if reference.is_none() { 655 log::warn!("No reference found for poc {}", poc); 656 } 657 658 self.codec.rps.ref_pic_set_st_curr_before[i] = reference; 659 } 660 661 for i in 0..self.codec.rps.num_poc_st_curr_after { 662 let poc = self.codec.rps.poc_st_curr_after[i]; 663 let reference = self.codec.dpb.find_short_term_ref_by_poc(poc); 664 665 if reference.is_none() { 666 log::warn!("No reference found for poc {}", poc); 667 } 668 669 self.codec.rps.ref_pic_set_st_curr_after[i] = reference; 670 } 671 672 for i in 0..self.codec.rps.num_poc_st_foll { 673 let poc = self.codec.rps.poc_st_foll[i]; 674 let reference = self.codec.dpb.find_short_term_ref_by_poc(poc); 675 676 if reference.is_none() { 677 log::warn!("No reference found for poc {}", poc); 678 } 679 680 self.codec.rps.ref_pic_set_st_foll[i] = reference; 681 } 682 683 // 4. All reference pictures in the DPB that are not included in 684 // RefPicSetLtCurr, RefPicSetLtFoll, RefPicSetStCurrBefore, 685 // RefPicSetStCurrAfter, or RefPicSetStFoll and have nuh_layer_id equal 686 // to currPicLayerId are marked as "unused for reference" 687 for dpb_pic in self.codec.dpb.entries() { 688 let find_predicate = |p: &Option<DpbEntry<B::Handle>>| match p { 689 Some(p) => p.0.borrow().pic_order_cnt_val == dpb_pic.0.borrow().pic_order_cnt_val, 690 None => false, 691 }; 692 693 if !self.codec.rps.ref_pic_set_lt_curr[0..self.codec.rps.num_poc_lt_curr] 694 .iter() 695 .any(find_predicate) 696 && !self.codec.rps.ref_pic_set_lt_foll[0..self.codec.rps.num_poc_lt_foll] 697 .iter() 698 .any(find_predicate) 699 && !self.codec.rps.ref_pic_set_st_curr_after 700 [0..self.codec.rps.num_poc_st_curr_after] 701 .iter() 702 .any(find_predicate) 703 && !self.codec.rps.ref_pic_set_st_curr_before 704 [0..self.codec.rps.num_poc_st_curr_before] 705 .iter() 706 .any(find_predicate) 707 && !self.codec.rps.ref_pic_set_st_foll[0..self.codec.rps.num_poc_st_foll] 708 .iter() 709 .any(find_predicate) 710 { 711 dpb_pic.0.borrow_mut().set_reference(Reference::None); 712 } 713 } 714 715 let total_rps_len = self.codec.rps.ref_pic_set_lt_curr[0..self.codec.rps.num_poc_lt_curr] 716 .len() 717 + self.codec.rps.ref_pic_set_lt_foll[0..self.codec.rps.num_poc_lt_foll].len() 718 + self.codec.rps.ref_pic_set_st_curr_after[0..self.codec.rps.num_poc_st_curr_after] 719 .len() 720 + self.codec.rps.ref_pic_set_st_curr_before[0..self.codec.rps.num_poc_st_curr_before] 721 .len() 722 + self.codec.rps.ref_pic_set_st_foll[0..self.codec.rps.num_poc_st_foll].len(); 723 724 let dpb_len = self.codec.dpb.entries().len(); 725 if dpb_len != total_rps_len { 726 log::warn!( 727 "The total RPS length {} is not the same as the DPB length {}", 728 total_rps_len, 729 dpb_len 730 ); 731 log::warn!("A reference pic may be in more than one RPS list. This is against the specification. See 8.3.2. NOTE 5") 732 } 733 734 // According to Chromium, unavailable reference pictures are handled by 735 // the accelerators internally. 736 Ok(()) 737 } 738 739 /// Drain the decoder, processing all pending frames. drain(&mut self) -> anyhow::Result<()>740 fn drain(&mut self) -> anyhow::Result<()> { 741 log::debug!("Draining the decoder"); 742 743 // Finish the current picture if there is one pending. 744 if let Some(cur_pic) = self.codec.current_pic.take() { 745 self.finish_picture(cur_pic)?; 746 } 747 748 let pics = self.codec.dpb.drain(); 749 750 log::debug!( 751 "Adding POCs {:?} to the ready queue while draining", 752 pics.iter().map(|p| p.0.borrow().pic_order_cnt_val).collect::<Vec<_>>() 753 ); 754 755 log::trace!("{:#?}", pics.iter().map(|p| p.0.borrow()).collect::<Vec<_>>()); 756 757 self.ready_queue.extend(pics.into_iter().map(|h| h.1)); 758 self.codec.dpb.clear(); 759 760 Ok(()) 761 } 762 clear_ref_lists(&mut self)763 fn clear_ref_lists(&mut self) { 764 if let Some(pic) = self.codec.current_pic.as_mut() { 765 pic.ref_pic_lists = Default::default(); 766 } 767 768 self.codec.rps.ref_pic_set_lt_curr = Default::default(); 769 self.codec.rps.ref_pic_set_st_curr_after = Default::default(); 770 self.codec.rps.ref_pic_set_st_curr_before = Default::default(); 771 772 self.codec.rps.num_poc_lt_curr = Default::default(); 773 self.codec.rps.num_poc_lt_foll = Default::default(); 774 self.codec.rps.num_poc_st_curr_after = Default::default(); 775 self.codec.rps.num_poc_st_curr_before = Default::default(); 776 self.codec.rps.num_poc_st_foll = Default::default(); 777 } 778 779 /// Bumps the DPB if needed. bump_as_needed( &mut self, bumping_type: BumpingType, ) -> anyhow::Result<Vec<DpbEntry<B::Handle>>>780 fn bump_as_needed( 781 &mut self, 782 bumping_type: BumpingType, 783 ) -> anyhow::Result<Vec<DpbEntry<B::Handle>>> { 784 let mut pics = vec![]; 785 786 let needs_bumping = match bumping_type { 787 BumpingType::BeforeDecoding => Dpb::<B::Handle>::needs_bumping, 788 BumpingType::AfterDecoding => Dpb::<B::Handle>::needs_additional_bumping, 789 }; 790 791 let sps = self.codec.parser.get_sps(self.codec.cur_sps_id).context("Invalid SPS id")?; 792 793 while needs_bumping(&mut self.codec.dpb, sps) { 794 match self.codec.dpb.bump(false) { 795 Some(pic) => pics.push(pic), 796 None => return Ok(pics), 797 } 798 } 799 800 Ok(pics) 801 } 802 803 // See C.5.2.2 update_dpb_before_decoding(&mut self, cur_pic: &PictureData) -> anyhow::Result<()>804 fn update_dpb_before_decoding(&mut self, cur_pic: &PictureData) -> anyhow::Result<()> { 805 if cur_pic.nalu_type.is_irap() 806 && cur_pic.no_rasl_output_flag 807 && !self.codec.first_picture_after_eos 808 { 809 if cur_pic.no_output_of_prior_pics_flag { 810 self.codec.dpb.clear(); 811 } else { 812 self.drain()?; 813 } 814 } else { 815 self.codec.dpb.remove_unused(); 816 let bumped = self.bump_as_needed(BumpingType::BeforeDecoding)?; 817 818 log::debug!( 819 "Adding POCs {:?} to the ready queue before decoding", 820 bumped.iter().map(|p| p.0.borrow().pic_order_cnt_val).collect::<Vec<_>>() 821 ); 822 823 log::trace!("{:#?}", bumped.iter().map(|p| p.0.borrow()).collect::<Vec<_>>()); 824 825 let bumped = bumped.into_iter().map(|p| p.1).collect::<Vec<_>>(); 826 self.ready_queue.extend(bumped); 827 } 828 829 Ok(()) 830 } 831 832 /// Called once per picture to start it. 833 #[allow(clippy::type_complexity)] begin_picture( &mut self, timestamp: u64, slice: &Slice, alloc_cb: &mut dyn FnMut() -> Option<<B::Handle as DecodedHandle>::Frame>, ) -> Result<Option<CurrentPicState<B::Handle, B::Picture>>, DecodeError>834 fn begin_picture( 835 &mut self, 836 timestamp: u64, 837 slice: &Slice, 838 alloc_cb: &mut dyn FnMut() -> Option<<B::Handle as DecodedHandle>::Frame>, 839 ) -> Result<Option<CurrentPicState<B::Handle, B::Picture>>, DecodeError> { 840 self.update_current_set_ids(slice.header.pic_parameter_set_id)?; 841 self.renegotiate_if_needed(RenegotiationType::CurrentSps)?; 842 // We renegotiated and must return the NALU and wait. 843 if matches!(self.decoding_state, DecodingState::AwaitingFormat(_)) { 844 return Err(DecodeError::CheckEvents); 845 } 846 847 let mut backend_pic = self.backend.new_picture(timestamp, alloc_cb)?; 848 849 let pic = PictureData::new_from_slice( 850 slice, 851 self.codec.first_picture_in_bitstream, 852 self.codec.first_picture_after_eos, 853 self.codec.prev_tid_0_pic.as_ref(), 854 self.codec.max_pic_order_cnt_lsb, 855 ); 856 857 self.codec.first_picture_after_eos = false; 858 self.codec.first_picture_in_bitstream = false; 859 860 if pic.nalu_type.is_irap() { 861 self.codec.irap_no_rasl_output_flag = pic.no_rasl_output_flag; 862 } else if pic.nalu_type.is_rasl() && self.codec.irap_no_rasl_output_flag { 863 // NOTE – All RASL pictures are leading pictures of an associated 864 // BLA or CRA picture. When the associated IRAP picture has 865 // NoRaslOutputFlag equal to 1, the RASL picture is not output and 866 // may not be correctly decodable, as the RASL picture may contain 867 // references to pictures that are not present in the bitstream. 868 // RASL pictures are not used as reference pictures for the decoding 869 // process of non-RASL pictures. 870 log::debug!( 871 "Dropping POC {}, as it may not be decodable according to the specification", 872 pic.pic_order_cnt_val 873 ); 874 875 return Ok(None); 876 } 877 878 log::debug!("Decode picture POC {}", pic.pic_order_cnt_val); 879 880 let pps = self 881 .codec 882 .parser 883 .get_pps(slice.header.pic_parameter_set_id) 884 .context("invalid PPS ID")? 885 .clone(); 886 887 self.decode_rps(slice, &pps.sps, &pic)?; 888 self.update_dpb_before_decoding(&pic)?; 889 890 self.backend.begin_picture( 891 &mut backend_pic, 892 &pic, 893 pps.sps.as_ref(), 894 &pps, 895 &self.codec.dpb, 896 &self.codec.rps, 897 slice, 898 )?; 899 900 Ok(Some(CurrentPicState { pic, backend_pic, ref_pic_lists: Default::default() })) 901 } 902 update_current_set_ids(&mut self, pps_id: u8) -> anyhow::Result<()>903 fn update_current_set_ids(&mut self, pps_id: u8) -> anyhow::Result<()> { 904 let pps = self.codec.parser.get_pps(pps_id).context("Invalid PPS")?; 905 906 self.codec.cur_sps_id = pps.seq_parameter_set_id; 907 Ok(()) 908 } 909 910 /// Handle a slice. Called once per slice NALU. handle_slice( &mut self, pic: &mut CurrentPicState<B::Handle, B::Picture>, slice: &Slice, ) -> anyhow::Result<()>911 fn handle_slice( 912 &mut self, 913 pic: &mut CurrentPicState<B::Handle, B::Picture>, 914 slice: &Slice, 915 ) -> anyhow::Result<()> { 916 // A dependent slice may refer to a previous SPS which 917 // is not the one currently in use. 918 self.update_current_set_ids(slice.header.pic_parameter_set_id)?; 919 920 let pps = self 921 .codec 922 .parser 923 .get_pps(slice.header.pic_parameter_set_id) 924 .context("invalid PPS ID")?; 925 let sps = pps.sps.as_ref(); 926 927 pic.ref_pic_lists = self.codec.rps.build_ref_pic_lists(&slice.header, pps, &pic.pic)?; 928 929 // Make sure that no negotiation is possible mid-picture. How could it? 930 // We'd lose the context with the previous slices on it. 931 if Self::negotiation_possible(sps, &self.codec.dpb, &self.codec.negotiation_info) { 932 anyhow::bail!("invalid stream: mid-frame format negotiation requested"); 933 } 934 935 self.backend.decode_slice( 936 &mut pic.backend_pic, 937 slice, 938 sps, 939 pps, 940 &pic.ref_pic_lists.ref_pic_list0, 941 &pic.ref_pic_lists.ref_pic_list1, 942 )?; 943 944 Ok(()) 945 } 946 finish_picture( &mut self, pic: CurrentPicState<B::Handle, B::Picture>, ) -> anyhow::Result<()>947 fn finish_picture( 948 &mut self, 949 pic: CurrentPicState<B::Handle, B::Picture>, 950 ) -> anyhow::Result<()> { 951 log::debug!("Finishing picture POC {:?}", pic.pic.pic_order_cnt_val); 952 953 // Submit the picture to the backend. 954 let handle = self.submit_picture(pic.backend_pic)?; 955 let pic = pic.pic; 956 957 // 8.3.1 958 if pic.valid_for_prev_tid0_pic { 959 self.codec.prev_tid_0_pic = Some(pic.clone()); 960 } 961 962 self.clear_ref_lists(); 963 964 // First store the current picture in the DPB, only then we should 965 // decide whether to bump. 966 self.codec 967 .dpb 968 .store_picture(Rc::new(RefCell::new(pic)), handle) 969 .map_err(|err| anyhow!(err))?; 970 let bumped = self.bump_as_needed(BumpingType::AfterDecoding)?; 971 972 log::debug!( 973 "Adding POCs {:?} to the ready queue after decoding", 974 bumped.iter().map(|p| p.0.borrow().pic_order_cnt_val).collect::<Vec<_>>() 975 ); 976 977 log::trace!("{:#?}", bumped.iter().map(|p| p.0.borrow()).collect::<Vec<_>>()); 978 979 let bumped = bumped.into_iter().map(|p| p.1).collect::<Vec<_>>(); 980 self.ready_queue.extend(bumped); 981 982 Ok(()) 983 } 984 renegotiate_if_needed( &mut self, renegotiation_type: RenegotiationType, ) -> anyhow::Result<()>985 fn renegotiate_if_needed( 986 &mut self, 987 renegotiation_type: RenegotiationType, 988 ) -> anyhow::Result<()> { 989 let sps = match renegotiation_type { 990 RenegotiationType::CurrentSps => { 991 self.codec.parser.get_sps(self.codec.cur_sps_id).context("Invalid SPS")? 992 } 993 RenegotiationType::NewSps(sps) => sps, 994 }; 995 996 if Self::negotiation_possible(sps, &self.codec.dpb, &self.codec.negotiation_info) { 997 // Make sure all the frames we decoded so far are in the ready queue. 998 self.drain()?; 999 let sps = match renegotiation_type { 1000 RenegotiationType::CurrentSps => { 1001 self.codec.parser.get_sps(self.codec.cur_sps_id).context("Invalid SPS")? 1002 } 1003 RenegotiationType::NewSps(sps) => sps, 1004 }; 1005 self.backend.new_sequence(sps)?; 1006 self.await_format_change(sps.clone()); 1007 } 1008 1009 Ok(()) 1010 } 1011 process_nalu( &mut self, timestamp: u64, nalu: Nalu, alloc_cb: &mut dyn FnMut() -> Option<<B::Handle as DecodedHandle>::Frame>, ) -> Result<(), DecodeError>1012 fn process_nalu( 1013 &mut self, 1014 timestamp: u64, 1015 nalu: Nalu, 1016 alloc_cb: &mut dyn FnMut() -> Option<<B::Handle as DecodedHandle>::Frame>, 1017 ) -> Result<(), DecodeError> { 1018 log::debug!("Processing NALU {:?}, length is {}", nalu.header.type_, nalu.size); 1019 1020 match nalu.header.type_ { 1021 NaluType::VpsNut => { 1022 self.codec 1023 .parser 1024 .parse_vps(&nalu) 1025 .map_err(|err| DecodeError::ParseFrameError(err))?; 1026 } 1027 NaluType::SpsNut => { 1028 let sps = self 1029 .codec 1030 .parser 1031 .parse_sps(&nalu) 1032 .map_err(|err| DecodeError::ParseFrameError(err))?; 1033 self.codec.max_pic_order_cnt_lsb = 1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4); 1034 1035 let pending_pps = std::mem::take(&mut self.codec.pending_pps); 1036 1037 // Try parsing the PPS again. 1038 for pps in pending_pps.into_iter() { 1039 if self.codec.parser.parse_pps(&pps).is_err() { 1040 // Failed to parse PPS add it again to pending 1041 self.codec.pending_pps.push(pps); 1042 } 1043 } 1044 } 1045 1046 NaluType::PpsNut => { 1047 if self.codec.parser.parse_pps(&nalu).is_err() { 1048 self.codec.pending_pps.push(nalu.into_owned()) 1049 } 1050 } 1051 1052 NaluType::BlaWLp 1053 | NaluType::BlaWRadl 1054 | NaluType::BlaNLp 1055 | NaluType::IdrWRadl 1056 | NaluType::IdrNLp 1057 | NaluType::TrailN 1058 | NaluType::TrailR 1059 | NaluType::TsaN 1060 | NaluType::TsaR 1061 | NaluType::StsaN 1062 | NaluType::StsaR 1063 | NaluType::RadlN 1064 | NaluType::RadlR 1065 | NaluType::RaslN 1066 | NaluType::RaslR 1067 | NaluType::CraNut => { 1068 let mut slice = self 1069 .codec 1070 .parser 1071 .parse_slice_header(nalu) 1072 .map_err(|err| DecodeError::ParseFrameError(err))?; 1073 1074 let first_slice_segment_in_pic_flag = slice.header.first_slice_segment_in_pic_flag; 1075 1076 if slice.header.dependent_slice_segment_flag { 1077 let previous_independent_header = self.codec.last_independent_slice_header.as_ref().ok_or(anyhow!("Cannot process an dependent slice without first processing and independent one"))?.clone(); 1078 slice 1079 .replace_header(previous_independent_header) 1080 .map_err(|err| DecodeError::ParseFrameError(err))?; 1081 } else { 1082 self.codec.last_independent_slice_header = Some(slice.header.clone()); 1083 } 1084 1085 let cur_pic = match self.codec.current_pic.take() { 1086 // No current picture, start a new one. 1087 None => self.begin_picture(timestamp, &slice, alloc_cb)?, 1088 Some(cur_pic) if first_slice_segment_in_pic_flag => { 1089 self.finish_picture(cur_pic)?; 1090 self.begin_picture(timestamp, &slice, alloc_cb)? 1091 } 1092 Some(cur_pic) => Some(cur_pic), 1093 }; 1094 1095 // Picture may have been dropped during begin_picture() 1096 if let Some(mut cur_pic) = cur_pic { 1097 self.handle_slice(&mut cur_pic, &slice)?; 1098 self.codec.current_pic = Some(cur_pic); 1099 } 1100 } 1101 1102 NaluType::EosNut => { 1103 self.codec.first_picture_after_eos = true; 1104 } 1105 1106 NaluType::EobNut => { 1107 self.codec.first_picture_in_bitstream = true; 1108 } 1109 1110 other => { 1111 log::debug!("Unsupported NAL unit type {:?}", other,); 1112 } 1113 } 1114 1115 Ok(()) 1116 } 1117 1118 /// Submits the picture to the accelerator. submit_picture(&mut self, backend_pic: B::Picture) -> Result<B::Handle, DecodeError>1119 fn submit_picture(&mut self, backend_pic: B::Picture) -> Result<B::Handle, DecodeError> { 1120 let handle = self.backend.submit_picture(backend_pic)?; 1121 1122 if self.blocking_mode == BlockingMode::Blocking { 1123 handle.sync()?; 1124 } 1125 1126 Ok(handle) 1127 } 1128 } 1129 1130 impl<B> StatelessVideoDecoder for StatelessDecoder<H265, B> 1131 where 1132 B: StatelessH265DecoderBackend, 1133 B::Handle: Clone + 'static, 1134 { 1135 type Handle = B::Handle; 1136 decode( &mut self, timestamp: u64, bitstream: &[u8], alloc_cb: &mut dyn FnMut() -> Option< <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, >, ) -> Result<usize, DecodeError>1137 fn decode( 1138 &mut self, 1139 timestamp: u64, 1140 bitstream: &[u8], 1141 alloc_cb: &mut dyn FnMut() -> Option< 1142 <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, 1143 >, 1144 ) -> Result<usize, DecodeError> { 1145 let mut cursor = Cursor::new(bitstream); 1146 let nalu = Nalu::next(&mut cursor).map_err(|err| DecodeError::ParseFrameError(err))?; 1147 1148 if nalu.header.type_ == NaluType::SpsNut { 1149 let sps = self 1150 .codec 1151 .parser 1152 .parse_sps(&nalu) 1153 .map_err(|err| DecodeError::ParseFrameError(err))? 1154 .clone(); 1155 if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) { 1156 // If more SPS come along we will renegotiate in begin_picture(). 1157 self.renegotiate_if_needed(RenegotiationType::NewSps(&sps))?; 1158 } else if matches!(self.decoding_state, DecodingState::Reset) { 1159 // We can resume decoding since the decoding parameters have not changed. 1160 self.decoding_state = DecodingState::Decoding; 1161 } 1162 } else if matches!(self.decoding_state, DecodingState::Reset) { 1163 let mut cursor = Cursor::new(bitstream); 1164 1165 while let Ok(nalu) = Nalu::next(&mut cursor) { 1166 // In the Reset state we can resume decoding from any key frame. 1167 if nalu.header.type_.is_idr() { 1168 self.decoding_state = DecodingState::Decoding; 1169 break; 1170 } 1171 } 1172 } 1173 1174 let nalu_len = nalu.offset + nalu.size; 1175 1176 match &mut self.decoding_state { 1177 // Process parameter sets, but skip input until we get information 1178 // from the stream. 1179 DecodingState::AwaitingStreamInfo | DecodingState::Reset => { 1180 if matches!( 1181 nalu.header.type_, 1182 NaluType::VpsNut | NaluType::SpsNut | NaluType::PpsNut 1183 ) { 1184 self.process_nalu(timestamp, nalu, alloc_cb)?; 1185 } 1186 } 1187 // Ask the client to confirm the format before we can process this. 1188 DecodingState::FlushingForDRC | DecodingState::AwaitingFormat(_) => { 1189 // Start signaling the awaiting format event to process a format change. 1190 self.awaiting_format_event.write(1).unwrap(); 1191 return Err(DecodeError::CheckEvents); 1192 } 1193 DecodingState::Decoding => { 1194 self.process_nalu(timestamp, nalu, alloc_cb)?; 1195 } 1196 } 1197 1198 Ok(nalu_len) 1199 } 1200 flush(&mut self) -> Result<(), DecodeError>1201 fn flush(&mut self) -> Result<(), DecodeError> { 1202 self.drain()?; 1203 self.decoding_state = DecodingState::Reset; 1204 1205 Ok(()) 1206 } 1207 next_event(&mut self) -> Option<DecoderEvent<B::Handle>>1208 fn next_event(&mut self) -> Option<DecoderEvent<B::Handle>> { 1209 self.query_next_event(|decoder, sps| { 1210 // Apply the SPS settings to the decoder so we don't enter the AwaitingFormat state 1211 // on the next decode() call. 1212 // TODO: unwrap this for now, but ideally change this closure to return Result 1213 decoder.apply_sps(sps).unwrap(); 1214 }) 1215 } 1216 stream_info(&self) -> Option<&StreamInfo>1217 fn stream_info(&self) -> Option<&StreamInfo> { 1218 self.backend.stream_info() 1219 } 1220 poll_fd(&self) -> BorrowedFd1221 fn poll_fd(&self) -> BorrowedFd { 1222 self.epoll_fd.0.as_fd() 1223 } 1224 } 1225 1226 #[cfg(test)] 1227 pub mod tests { 1228 1229 use crate::bitstream_utils::NalIterator; 1230 use crate::codec::h265::parser::Nalu; 1231 use crate::decoder::stateless::h265::H265; 1232 use crate::decoder::stateless::tests::test_decode_stream; 1233 use crate::decoder::stateless::tests::TestStream; 1234 use crate::decoder::stateless::StatelessDecoder; 1235 use crate::decoder::BlockingMode; 1236 use crate::utils::simple_playback_loop; 1237 use crate::utils::simple_playback_loop_owned_frames; 1238 use crate::DecodedFormat; 1239 1240 /// Run `test` using the dummy decoder, in both blocking and non-blocking modes. test_decoder_dummy(test: &TestStream, blocking_mode: BlockingMode)1241 fn test_decoder_dummy(test: &TestStream, blocking_mode: BlockingMode) { 1242 let decoder = StatelessDecoder::<H265, _>::new_dummy(blocking_mode).unwrap(); 1243 1244 test_decode_stream( 1245 |d, s, f| { 1246 simple_playback_loop( 1247 d, 1248 NalIterator::<Nalu>::new(s), 1249 f, 1250 &mut simple_playback_loop_owned_frames, 1251 DecodedFormat::NV12, 1252 blocking_mode, 1253 ) 1254 }, 1255 decoder, 1256 test, 1257 false, 1258 false, 1259 ); 1260 } 1261 1262 /// A 64x64 progressive byte-stream encoded I-frame to make it easier to 1263 /// spot errors on the libva trace. 1264 /// Encoded with the following GStreamer pipeline: 1265 /// 1266 /// gst-launch-1.0 videotestsrc num-buffers=1 ! video/x-raw,format=I420,width=64,height=64 ! x265enc ! video/x-h265,profile=main ! filesink location="64x64-I.h265" 1267 pub const DECODE_64X64_PROGRESSIVE_I: TestStream = TestStream { 1268 stream: include_bytes!("../../codec/h265/test_data/64x64-I.h265"), 1269 crcs: include_str!("../../codec/h265/test_data/64x64-I.h265.crc"), 1270 }; 1271 1272 #[test] test_64x64_progressive_i_block()1273 fn test_64x64_progressive_i_block() { 1274 test_decoder_dummy(&DECODE_64X64_PROGRESSIVE_I, BlockingMode::Blocking); 1275 } 1276 1277 #[test] test_64x64_progressive_i_nonblock()1278 fn test_64x64_progressive_i_nonblock() { 1279 test_decoder_dummy(&DECODE_64X64_PROGRESSIVE_I, BlockingMode::NonBlocking); 1280 } 1281 1282 /// A 64x64 progressive byte-stream encoded I-frame and P-frame to make 1283 /// it easier to spot errors on the libva trace. 1284 /// Encoded with the following GStreamer pipeline: 1285 /// gst-launch-1.0 videotestsrc num-buffers=2 ! video/x-raw,format=I420,width=64,height=64 ! x265enc option-string="b-adapt=0" ! video/x-h265,profile=main ! filesink location="64x64-I-P.h265" 1286 pub const DECODE_64X64_PROGRESSIVE_I_P: TestStream = TestStream { 1287 stream: include_bytes!("../../codec/h265/test_data/64x64-I-P.h265"), 1288 crcs: include_str!("../../codec/h265/test_data/64x64-I-P.h265.crc"), 1289 }; 1290 1291 #[test] test_64x64_progressive_i_p_block()1292 fn test_64x64_progressive_i_p_block() { 1293 test_decoder_dummy(&DECODE_64X64_PROGRESSIVE_I_P, BlockingMode::Blocking); 1294 } 1295 1296 #[test] test_64x64_progressive_i_p_nonblock()1297 fn test_64x64_progressive_i_p_nonblock() { 1298 test_decoder_dummy(&DECODE_64X64_PROGRESSIVE_I_P, BlockingMode::NonBlocking); 1299 } 1300 1301 /// A 64x64 progressive byte-stream encoded I-P-B-P sequence to make it 1302 /// easier to it easier to spot errors on the libva trace. 1303 /// Encoded with the following GStreamer pipeline: 1304 /// gst-launch-1.0 videotestsrc num-buffers=3 ! video/x-raw,format=I420,width=64,height=64 ! x265enc option-string="b-adapt=0:bframes=1" ! video/x-h265,profile=main ! filesink location="64x64-I-P-B-P.h265" 1305 pub const DECODE_64X64_PROGRESSIVE_I_P_B_P: TestStream = TestStream { 1306 stream: include_bytes!("../../codec/h265/test_data/64x64-I-P-B-P.h265"), 1307 crcs: include_str!("../../codec/h265/test_data/64x64-I-P-B-P.h265.crc"), 1308 }; 1309 1310 #[test] test_64x64_progressive_i_p_b_p_block()1311 fn test_64x64_progressive_i_p_b_p_block() { 1312 test_decoder_dummy(&DECODE_64X64_PROGRESSIVE_I_P_B_P, BlockingMode::Blocking); 1313 } 1314 1315 #[test] test_64x64_progressive_i_p_b_p_nonblock()1316 fn test_64x64_progressive_i_p_b_p_nonblock() { 1317 test_decoder_dummy(&DECODE_64X64_PROGRESSIVE_I_P_B_P, BlockingMode::NonBlocking); 1318 } 1319 1320 /// Same as Chromium's test-25fps.h265 1321 pub const DECODE_TEST_25FPS: TestStream = TestStream { 1322 stream: include_bytes!("../../codec/h265/test_data/test-25fps.h265"), 1323 crcs: include_str!("../../codec/h265/test_data/test-25fps.h265.crc"), 1324 }; 1325 1326 #[test] test_25fps_block()1327 fn test_25fps_block() { 1328 test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::Blocking); 1329 } 1330 1331 #[test] test_25fps_nonblock()1332 fn test_25fps_nonblock() { 1333 test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::NonBlocking); 1334 } 1335 1336 /// Same as Chromium's bear.h265 1337 pub const DECODE_BEAR: TestStream = TestStream { 1338 stream: include_bytes!("../../codec/h265/test_data/bear.h265"), 1339 crcs: include_str!("../../codec/h265/test_data/bear.h265.crc"), 1340 }; 1341 1342 #[test] test_bear_block()1343 fn test_bear_block() { 1344 test_decoder_dummy(&DECODE_BEAR, BlockingMode::Blocking); 1345 } 1346 1347 #[test] test_bear_nonblock()1348 fn test_bear_nonblock() { 1349 test_decoder_dummy(&DECODE_BEAR, BlockingMode::NonBlocking); 1350 } 1351 1352 /// Same as Chromium's bbb.h265 1353 pub const DECODE_BBB: TestStream = TestStream { 1354 stream: include_bytes!("../../codec/h265/test_data/bbb.h265"), 1355 crcs: include_str!("../../codec/h265/test_data/bbb.h265.crc"), 1356 }; 1357 1358 #[test] test_bbb_block()1359 fn test_bbb_block() { 1360 test_decoder_dummy(&DECODE_BBB, BlockingMode::Blocking); 1361 } 1362 1363 #[test] test_bbb_nonblock()1364 fn test_bbb_nonblock() { 1365 test_decoder_dummy(&DECODE_BBB, BlockingMode::NonBlocking); 1366 } 1367 } 1368