1 // Copyright 2023 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 use std::os::fd::AsFd; 6 use std::os::fd::BorrowedFd; 7 8 use anyhow::anyhow; 9 10 use crate::codec::av1::parser::FrameHeaderObu; 11 use crate::codec::av1::parser::FrameObu; 12 use crate::codec::av1::parser::FrameType; 13 use crate::codec::av1::parser::ObuAction; 14 use crate::codec::av1::parser::ObuType; 15 use crate::codec::av1::parser::ParsedObu; 16 use crate::codec::av1::parser::Parser; 17 use crate::codec::av1::parser::StreamInfo; 18 use crate::codec::av1::parser::TileGroupObu; 19 use crate::codec::av1::parser::NUM_REF_FRAMES; 20 use crate::decoder::stateless::DecodeError; 21 use crate::decoder::stateless::DecodingState; 22 use crate::decoder::stateless::NewPictureResult; 23 use crate::decoder::stateless::StatelessBackendResult; 24 use crate::decoder::stateless::StatelessCodec; 25 use crate::decoder::stateless::StatelessDecoder; 26 use crate::decoder::stateless::StatelessDecoderBackend; 27 use crate::decoder::stateless::StatelessDecoderBackendPicture; 28 use crate::decoder::stateless::StatelessVideoDecoder; 29 use crate::decoder::BlockingMode; 30 use crate::decoder::DecodedHandle; 31 use crate::Resolution; 32 33 #[cfg(test)] 34 mod dummy; 35 #[cfg(feature = "v4l2")] 36 mod v4l2; 37 #[cfg(feature = "vaapi")] 38 mod vaapi; 39 40 /// Stateless backend methods specific to AV1. 41 pub trait StatelessAV1DecoderBackend: 42 StatelessDecoderBackend + StatelessDecoderBackendPicture<Av1> 43 { 44 /// Called when a new Sequence Header OBU is parsed. The 45 /// `highest_spatial_layer` argument refers to the maximum layer selected by 46 /// the client through `set_operating_point()` and the scalability 47 /// information present in the stream, if any. change_stream_info(&mut self, stream_info: &StreamInfo) -> StatelessBackendResult<()>48 fn change_stream_info(&mut self, stream_info: &StreamInfo) -> StatelessBackendResult<()>; 49 50 /// Called when the decoder determines that a new picture was found. The backend allocates all 51 /// the resources it needs to process that picture. new_picture( &mut self, hdr: &FrameHeaderObu, timestamp: u64, alloc_cb: &mut dyn FnMut() -> Option< <<Self as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, >, ) -> NewPictureResult<Self::Picture>52 fn new_picture( 53 &mut self, 54 hdr: &FrameHeaderObu, 55 timestamp: u64, 56 alloc_cb: &mut dyn FnMut() -> Option< 57 <<Self as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, 58 >, 59 ) -> NewPictureResult<Self::Picture>; 60 61 /// Called to set the global parameters of a picture. begin_picture( &mut self, picture: &mut Self::Picture, stream_info: &StreamInfo, hdr: &FrameHeaderObu, reference_frames: &[Option<Self::Handle>; NUM_REF_FRAMES], ) -> StatelessBackendResult<()>62 fn begin_picture( 63 &mut self, 64 picture: &mut Self::Picture, 65 stream_info: &StreamInfo, 66 hdr: &FrameHeaderObu, 67 reference_frames: &[Option<Self::Handle>; NUM_REF_FRAMES], 68 ) -> StatelessBackendResult<()>; 69 70 /// Called to dispatch a decode operation to the backend. 71 #[allow(clippy::too_many_arguments)] decode_tile_group( &mut self, picture: &mut Self::Picture, tile_group: TileGroupObu, ) -> StatelessBackendResult<()>72 fn decode_tile_group( 73 &mut self, 74 picture: &mut Self::Picture, 75 tile_group: TileGroupObu, 76 ) -> StatelessBackendResult<()>; 77 78 /// Called when the decoder wants the backend to finish the decoding 79 /// operations for `picture`. At this point, `decode_tile` has been called 80 /// for all tiles. submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle>81 fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle>; 82 } 83 84 /// State of the picture being currently decoded. 85 /// 86 /// Stored between calls to [`StatelessDecoder::decode_tile_group`] that belong to the same 87 /// picture. 88 enum CurrentPicState<H: DecodedHandle, P> { 89 /// A regular frame 90 RegularFrame { 91 /// Data for the current picture as extracted from the stream. 92 header: FrameHeaderObu, 93 /// Backend-specific data for that picture. 94 backend_picture: P, 95 }, 96 97 /// A frame that has 'show_existing_frame' set. 98 ShowExistingFrame { 99 /// Data for the current picture as extracted from the stream. 100 header: FrameHeaderObu, 101 /// The handle of the reference frame that this frame points to. 102 handle: H, 103 }, 104 } 105 106 pub struct AV1DecoderState<H: DecodedHandle, P> { 107 /// AV1 bitstream parser. 108 parser: Parser, 109 110 /// The reference frames in use. 111 reference_frames: [Option<H>; NUM_REF_FRAMES], 112 113 /// Keeps track of the last values seen for negotiation purposes. 114 stream_info: Option<StreamInfo>, 115 116 /// The picture currently being decoded. We need to preserve it between 117 /// calls to `decode` because multiple tiles will be processed in different 118 /// calls to `decode`. 119 current_pic: Option<CurrentPicState<H, P>>, 120 121 /// Keep track of the number of frames we've processed for logging purposes. 122 frame_count: u32, 123 124 /// For SVC streams, we only want to output the highest layer possible given 125 /// the choice of operating point. 126 highest_spatial_layer: Option<u32>, 127 } 128 129 impl<H, P> Default for AV1DecoderState<H, P> 130 where 131 H: DecodedHandle, 132 { default() -> Self133 fn default() -> Self { 134 Self { 135 parser: Default::default(), 136 reference_frames: Default::default(), 137 stream_info: Default::default(), 138 current_pic: Default::default(), 139 frame_count: Default::default(), 140 highest_spatial_layer: Default::default(), 141 } 142 } 143 } 144 145 /// [`StatelessCodec`] structure to use in order to create a AV1 stateless decoder. 146 /// 147 /// # Accepted input 148 /// 149 /// the VP9 specification requires the last byte of the chunk to contain the superframe marker. 150 /// Thus, a decoder using this codec processes exactly one encoded chunk per call to 151 /// [`StatelessDecoder::decode`], and always returns the size of the passed input if successful. 152 pub struct Av1; 153 154 impl StatelessCodec for Av1 { 155 type FormatInfo = StreamInfo; 156 type DecoderState<H: DecodedHandle, P> = AV1DecoderState<H, P>; 157 } 158 159 impl<B> StatelessDecoder<Av1, B> 160 where 161 B: StatelessAV1DecoderBackend, 162 B::Handle: Clone, 163 { decode_frame_header( &mut self, frame_header: FrameHeaderObu, timestamp: u64, alloc_cb: &mut dyn FnMut() -> Option< <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, >, ) -> Result<(), DecodeError>164 fn decode_frame_header( 165 &mut self, 166 frame_header: FrameHeaderObu, 167 timestamp: u64, 168 alloc_cb: &mut dyn FnMut() -> Option< 169 <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, 170 >, 171 ) -> Result<(), DecodeError> { 172 log::debug!("Processing frame {} with timestamp {}", self.codec.frame_count, timestamp); 173 174 if frame_header.show_existing_frame { 175 let ref_frame = self.codec.reference_frames 176 [frame_header.frame_to_show_map_idx as usize] 177 .as_ref() 178 .ok_or(anyhow!("Broken stream: no reference picture to display"))?; 179 self.codec.current_pic = Some(CurrentPicState::ShowExistingFrame { 180 header: frame_header, 181 handle: ref_frame.clone(), 182 }); 183 } else if let Some(stream_info) = &self.codec.stream_info { 184 let mut backend_picture = 185 self.backend.new_picture(&frame_header, timestamp, alloc_cb)?; 186 187 self.backend.begin_picture( 188 &mut backend_picture, 189 stream_info, 190 &frame_header, 191 &self.codec.reference_frames, 192 )?; 193 194 self.codec.current_pic = Some(CurrentPicState::RegularFrame { 195 header: frame_header.clone(), 196 backend_picture, 197 }); 198 } else { 199 log::warn!("invalid stream: frame header received while no valid sequence ongoing"); 200 } 201 202 Ok(()) 203 } 204 decode_tile_group(&mut self, tile_group: TileGroupObu) -> anyhow::Result<()>205 fn decode_tile_group(&mut self, tile_group: TileGroupObu) -> anyhow::Result<()> { 206 let picture = match self.codec.current_pic.as_mut() { 207 Some(CurrentPicState::RegularFrame { backend_picture, .. }) => backend_picture, 208 Some(CurrentPicState::ShowExistingFrame { .. }) => { 209 return Err(anyhow!("Broken stream: cannot decode a tile group for a frame with show_existing_frame set")); 210 } 211 None => { 212 return Err(anyhow!( 213 "Broken stream: cannot decode a tile group without first decoding a frame header" 214 )) 215 } 216 }; 217 218 self.backend.decode_tile_group(picture, tile_group)?; 219 Ok(()) 220 } 221 decode_frame( &mut self, frame: FrameObu, timestamp: u64, alloc_cb: &mut dyn FnMut() -> Option< <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, >, ) -> Result<(), DecodeError>222 fn decode_frame( 223 &mut self, 224 frame: FrameObu, 225 timestamp: u64, 226 alloc_cb: &mut dyn FnMut() -> Option< 227 <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, 228 >, 229 ) -> Result<(), DecodeError> { 230 let FrameObu { header, tile_group } = frame; 231 self.decode_frame_header(header, timestamp, alloc_cb)?; 232 self.decode_tile_group(tile_group)?; 233 Ok(()) 234 } 235 submit_frame(&mut self, timestamp: u64) -> anyhow::Result<()>236 fn submit_frame(&mut self, timestamp: u64) -> anyhow::Result<()> { 237 log::debug!("Finishing frame {} with timestamp: {}", self.codec.frame_count, timestamp); 238 239 let picture = self.codec.current_pic.take(); 240 241 let (handle, header) = match picture { 242 Some(CurrentPicState::RegularFrame { header, backend_picture }) => { 243 let handle = self.backend.submit_picture(backend_picture)?; 244 245 if self.blocking_mode == BlockingMode::Blocking { 246 handle.sync()?; 247 } 248 (handle, header) 249 } 250 Some(CurrentPicState::ShowExistingFrame { header, handle }) => (handle, header), 251 None => return Err(anyhow!("Broken stream: no picture to submit")), 252 }; 253 254 let update_refs = if header.show_existing_frame { 255 header.frame_type == FrameType::KeyFrame 256 } else { 257 true 258 }; 259 260 if update_refs { 261 let mut refresh_frame_flags = header.refresh_frame_flags; 262 263 #[allow(clippy::needless_range_loop)] 264 for i in 0..NUM_REF_FRAMES { 265 if (refresh_frame_flags & 1) == 1 { 266 log::debug!( 267 "Replacing reference frame {} to new timestamp {} on frame count: {}", 268 i, 269 timestamp, 270 self.codec.frame_count 271 ); 272 self.codec.reference_frames[i] = Some(handle.clone()); 273 } 274 275 refresh_frame_flags >>= 1; 276 } 277 } 278 279 let show_existing_frame = header.show_existing_frame; 280 if header.show_frame || show_existing_frame { 281 match self.codec.highest_spatial_layer { 282 None => self.ready_queue.push(handle), 283 Some(highest_spatial_layer) => { 284 if header.obu_header.spatial_id >= highest_spatial_layer { 285 self.ready_queue.push(handle); 286 } else { 287 log::debug!( 288 "Dropping frame with spatial_id {}", 289 header.obu_header.spatial_id 290 ); 291 } 292 } 293 } 294 } 295 296 self.codec.parser.ref_frame_update(&header).map_err(|err| anyhow!(err))?; 297 self.codec.frame_count += 1; 298 Ok(()) 299 } 300 } 301 302 impl<B> StatelessVideoDecoder for StatelessDecoder<Av1, B> 303 where 304 B: StatelessAV1DecoderBackend, 305 B::Handle: Clone + 'static, 306 { 307 type Handle = B::Handle; 308 309 /// Decode an AV1 stream. 310 /// 311 /// `bitstream` should initially be submitted as a whole temporal unit, however a call to this 312 /// method will only consume a single OBU. The caller must be careful to check the return value 313 /// and resubmit the remainder if the whole bitstream has not been consumed. decode( &mut self, timestamp: u64, bitstream: &[u8], alloc_cb: &mut dyn FnMut() -> Option< <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, >, ) -> Result<usize, DecodeError>314 fn decode( 315 &mut self, 316 timestamp: u64, 317 bitstream: &[u8], 318 alloc_cb: &mut dyn FnMut() -> Option< 319 <<B as StatelessDecoderBackend>::Handle as DecodedHandle>::Frame, 320 >, 321 ) -> Result<usize, DecodeError> { 322 let obu = match self 323 .codec 324 .parser 325 .read_obu(bitstream) 326 .map_err(|err| DecodeError::ParseFrameError(err))? 327 { 328 ObuAction::Process(obu) => obu, 329 // This OBU should be dropped. 330 ObuAction::Drop(length) => return Ok(length as usize), 331 }; 332 let obu_length = obu.bytes_used; 333 334 let is_decode_op = matches!( 335 obu.header.obu_type, 336 ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup 337 ); 338 339 if is_decode_op { 340 match self.decoding_state { 341 /* we want to be here */ 342 DecodingState::Decoding => (), 343 344 /* otherwise... */ 345 DecodingState::AwaitingStreamInfo => { 346 /* Skip input until we get information from the stream. */ 347 return Ok(obu_length); 348 } 349 /* Ask the client to confirm the format before we can process this. */ 350 DecodingState::FlushingForDRC | DecodingState::AwaitingFormat(_) => { 351 // Start signaling the awaiting format event to process a format change. 352 self.awaiting_format_event.write(1).unwrap(); 353 return Err(DecodeError::CheckEvents); 354 } 355 356 DecodingState::Reset => { 357 let mut parser = self.codec.parser.clone(); 358 359 let is_key_frame = match obu.header.obu_type { 360 ObuType::Frame | ObuType::FrameHeader => { 361 let fh = parser 362 .parse_frame_header_obu(&obu) 363 .map_err(|err| DecodeError::ParseFrameError(err))?; 364 fh.frame_type == FrameType::KeyFrame 365 } 366 _ => false, 367 }; 368 369 /* we can only resume from key frames */ 370 if !is_key_frame { 371 return Ok(obu_length); 372 } else { 373 self.decoding_state = DecodingState::Decoding; 374 } 375 } 376 } 377 } 378 379 /* We are in `Decoding` state if we reached here */ 380 381 match self.codec.parser.parse_obu(obu).map_err(|err| DecodeError::ParseFrameError(err))? { 382 ParsedObu::SequenceHeader(sequence) => { 383 let sequence_differs = match &self.codec.stream_info { 384 Some(old_stream_info) => *old_stream_info.seq_header != *sequence, 385 None => true, 386 }; 387 388 if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) 389 || sequence_differs 390 { 391 if self.codec.current_pic.is_some() { 392 return Err(DecodeError::DecoderError(anyhow!( 393 "broken stream: a picture is being decoded while a new sequence header is encountered" 394 ))); 395 } 396 397 /* make sure we sync *before* we clear any state in the backend */ 398 for f in &mut self.ready_queue.queue { 399 /* TODO: this fixes av1-1-b8-03-sizeup on Intel 400 * gen12, but we apparently do not do the same in 401 * VP9. How is it that we do not get similar crashes there? 402 * 403 * TODO: syncing before calling new_sequence() in VP9 may fix some tests 404 */ 405 f.sync()?; 406 } 407 408 log::debug!( 409 "found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}", 410 Resolution::from(( 411 sequence.max_frame_width_minus_1 as u32 + 1, 412 sequence.max_frame_height_minus_1 as u32 + 1 413 )), 414 sequence.seq_profile, 415 sequence.bit_depth 416 ); 417 /* there is nothing to drain, much like vp8 and vp9 */ 418 self.codec.highest_spatial_layer = self.codec.parser.highest_operating_point(); 419 420 let stream_info = match &self.codec.parser.last_frame_header { 421 Some(fh) => StreamInfo { 422 seq_header: sequence.clone(), 423 render_width: fh.render_width, 424 render_height: fh.render_height, 425 }, 426 None => StreamInfo { 427 seq_header: sequence.clone(), 428 render_width: sequence.max_frame_width_minus_1 as u32 + 1, 429 render_height: sequence.max_frame_height_minus_1 as u32 + 1, 430 }, 431 }; 432 self.backend.change_stream_info(&stream_info)?; 433 self.await_format_change(stream_info); 434 } 435 } 436 ParsedObu::FrameHeader(frame_header) => { 437 if self.codec.current_pic.is_some() { 438 /* submit this frame immediately, as we need to update the 439 * DPB and the reference info state *before* processing the 440 * next frame */ 441 self.submit_frame(timestamp)?; 442 } 443 self.decode_frame_header(frame_header, timestamp, alloc_cb)?; 444 } 445 ParsedObu::TileGroup(tile_group) => { 446 self.decode_tile_group(tile_group)?; 447 } 448 ParsedObu::Frame(frame) => { 449 let stream_info = 450 self.codec.stream_info.as_ref().ok_or(DecodeError::DecoderError(anyhow!( 451 "broken stream: a picture is being decoded without a sequence header" 452 )))?; 453 if stream_info.render_width != frame.header.render_width 454 || stream_info.render_height != frame.header.render_height 455 { 456 let new_stream_info = StreamInfo { 457 seq_header: stream_info.seq_header.clone(), 458 render_width: frame.header.render_width, 459 render_height: frame.header.render_height, 460 }; 461 self.backend.change_stream_info(&new_stream_info)?; 462 } 463 if self.codec.current_pic.is_some() { 464 /* submit this frame immediately, as we need to update the 465 * DPB and the reference info state *before* processing the 466 * next frame */ 467 self.submit_frame(timestamp)?; 468 } 469 self.decode_frame(frame, timestamp, alloc_cb)?; 470 /* submit this frame immediately, as we need to update the 471 * DPB and the reference info state *before* processing the 472 * next frame */ 473 self.submit_frame(timestamp)?; 474 } 475 ParsedObu::TileList => { 476 return Err(DecodeError::DecoderError(anyhow!( 477 "large tile scale mode is not supported" 478 ))); 479 } 480 other => { 481 log::debug!("skipping OBU of type {:?}", other.obu_type()); 482 } 483 } 484 485 /* Submit the last frame if we have reached the end of the temporal unit. */ 486 if bitstream.len() == obu_length && self.codec.current_pic.is_some() { 487 self.submit_frame(timestamp)?; 488 } 489 490 Ok(obu_length) 491 } 492 flush(&mut self) -> Result<(), super::DecodeError>493 fn flush(&mut self) -> Result<(), super::DecodeError> { 494 // Note: all the submitted frames are already in the ready queue. 495 self.codec.reference_frames = Default::default(); 496 self.decoding_state = DecodingState::Reset; 497 498 Ok(()) 499 } 500 stream_info(&self) -> Option<&crate::decoder::StreamInfo>501 fn stream_info(&self) -> Option<&crate::decoder::StreamInfo> { 502 self.backend.stream_info() 503 } 504 next_event(&mut self) -> Option<crate::decoder::DecoderEvent<B::Handle>>505 fn next_event(&mut self) -> Option<crate::decoder::DecoderEvent<B::Handle>> { 506 self.query_next_event(|decoder, stream_info| { 507 decoder.codec.stream_info = Some(stream_info.clone()); 508 }) 509 } 510 poll_fd(&self) -> BorrowedFd511 fn poll_fd(&self) -> BorrowedFd { 512 self.epoll_fd.0.as_fd() 513 } 514 } 515 516 #[cfg(test)] 517 pub mod tests { 518 use crate::bitstream_utils::IvfIterator; 519 use crate::decoder::stateless::av1::Av1; 520 use crate::decoder::stateless::tests::test_decode_stream; 521 use crate::decoder::stateless::tests::TestStream; 522 use crate::decoder::stateless::StatelessDecoder; 523 use crate::decoder::BlockingMode; 524 use crate::utils::simple_playback_loop; 525 use crate::utils::simple_playback_loop_owned_frames; 526 use crate::DecodedFormat; 527 528 /// Run `test` using the dummy decoder, in both blocking and non-blocking modes. test_decoder_dummy(test: &TestStream, blocking_mode: BlockingMode)529 fn test_decoder_dummy(test: &TestStream, blocking_mode: BlockingMode) { 530 let decoder = StatelessDecoder::<Av1, _>::new_dummy(blocking_mode).unwrap(); 531 532 test_decode_stream( 533 |d, s, f| { 534 simple_playback_loop( 535 d, 536 IvfIterator::new(s), 537 f, 538 &mut simple_playback_loop_owned_frames, 539 DecodedFormat::NV12, 540 blocking_mode, 541 ) 542 }, 543 decoder, 544 test, 545 false, 546 false, 547 ); 548 } 549 550 /// Same as Chromium's test-25fps.av1.ivf 551 pub const DECODE_TEST_25FPS: TestStream = TestStream { 552 stream: include_bytes!("../../codec/av1/test_data/test-25fps.ivf.av1"), 553 crcs: include_str!("../../codec/av1/test_data/test-25fps.ivf.av1.crc"), 554 }; 555 556 #[test] test_25fps_block()557 fn test_25fps_block() { 558 test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::Blocking); 559 } 560 561 #[test] test_25fps_nonblock()562 fn test_25fps_nonblock() { 563 test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::NonBlocking); 564 } 565 } 566