1 // Copyright 2024 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 pub mod gainmap; 16 pub mod item; 17 pub mod tile; 18 pub mod track; 19 20 use crate::decoder::gainmap::*; 21 use crate::decoder::item::*; 22 use crate::decoder::tile::*; 23 use crate::decoder::track::*; 24 25 #[cfg(feature = "dav1d")] 26 use crate::codecs::dav1d::Dav1d; 27 28 #[cfg(feature = "libgav1")] 29 use crate::codecs::libgav1::Libgav1; 30 31 #[cfg(feature = "android_mediacodec")] 32 use crate::codecs::android_mediacodec::MediaCodec; 33 34 use crate::codecs::DecoderConfig; 35 use crate::image::*; 36 use crate::internal_utils::io::*; 37 use crate::internal_utils::*; 38 use crate::parser::exif; 39 use crate::parser::mp4box; 40 use crate::parser::mp4box::*; 41 use crate::parser::obu::Av1SequenceHeader; 42 use crate::*; 43 44 use std::cmp::max; 45 use std::cmp::min; 46 use std::num::NonZero; 47 48 pub trait IO { read(&mut self, offset: u64, max_read_size: usize) -> AvifResult<&[u8]>49 fn read(&mut self, offset: u64, max_read_size: usize) -> AvifResult<&[u8]>; size_hint(&self) -> u6450 fn size_hint(&self) -> u64; persistent(&self) -> bool51 fn persistent(&self) -> bool; 52 } 53 54 impl dyn IO { read_exact(&mut self, offset: u64, read_size: usize) -> AvifResult<&[u8]>55 pub(crate) fn read_exact(&mut self, offset: u64, read_size: usize) -> AvifResult<&[u8]> { 56 let result = self.read(offset, read_size)?; 57 if result.len() < read_size { 58 Err(AvifError::TruncatedData) 59 } else { 60 assert!(result.len() == read_size); 61 Ok(result) 62 } 63 } 64 } 65 66 pub type GenericIO = Box<dyn IO>; 67 pub type Codec = Box<dyn crate::codecs::Decoder>; 68 69 #[derive(Debug, Default, PartialEq)] 70 pub enum CodecChoice { 71 #[default] 72 Auto, 73 Dav1d, 74 Libgav1, 75 MediaCodec, 76 } 77 78 impl CodecChoice { get_codec(&self, is_avif: bool) -> AvifResult<Codec>79 fn get_codec(&self, is_avif: bool) -> AvifResult<Codec> { 80 match self { 81 CodecChoice::Auto => { 82 // Preferred order of codecs in Auto mode: Android MediaCodec, Dav1d, Libgav1. 83 CodecChoice::MediaCodec 84 .get_codec(is_avif) 85 .or_else(|_| CodecChoice::Dav1d.get_codec(is_avif)) 86 .or_else(|_| CodecChoice::Libgav1.get_codec(is_avif)) 87 } 88 CodecChoice::Dav1d => { 89 if !is_avif { 90 return Err(AvifError::NoCodecAvailable); 91 } 92 #[cfg(feature = "dav1d")] 93 return Ok(Box::<Dav1d>::default()); 94 #[cfg(not(feature = "dav1d"))] 95 return Err(AvifError::NoCodecAvailable); 96 } 97 CodecChoice::Libgav1 => { 98 if !is_avif { 99 return Err(AvifError::NoCodecAvailable); 100 } 101 #[cfg(feature = "libgav1")] 102 return Ok(Box::<Libgav1>::default()); 103 #[cfg(not(feature = "libgav1"))] 104 return Err(AvifError::NoCodecAvailable); 105 } 106 CodecChoice::MediaCodec => { 107 #[cfg(feature = "android_mediacodec")] 108 return Ok(Box::<MediaCodec>::default()); 109 #[cfg(not(feature = "android_mediacodec"))] 110 return Err(AvifError::NoCodecAvailable); 111 } 112 } 113 } 114 } 115 116 #[repr(C)] 117 #[derive(Clone, Copy, Debug, Default, PartialEq)] 118 pub enum Source { 119 #[default] 120 Auto = 0, 121 PrimaryItem = 1, 122 Tracks = 2, 123 // TODO: Thumbnail, 124 } 125 126 pub const DEFAULT_IMAGE_SIZE_LIMIT: u32 = 16384 * 16384; 127 pub const DEFAULT_IMAGE_DIMENSION_LIMIT: u32 = 32768; 128 pub const DEFAULT_IMAGE_COUNT_LIMIT: u32 = 12 * 3600 * 60; 129 130 #[derive(Debug, PartialEq)] 131 pub enum ImageContentType { 132 None, 133 ColorAndAlpha, 134 GainMap, 135 All, 136 } 137 138 impl ImageContentType { categories(&self) -> Vec<Category>139 pub(crate) fn categories(&self) -> Vec<Category> { 140 match self { 141 Self::None => vec![], 142 Self::ColorAndAlpha => vec![Category::Color, Category::Alpha], 143 Self::GainMap => vec![Category::Gainmap], 144 Self::All => Category::ALL.to_vec(), 145 } 146 } 147 gainmap(&self) -> bool148 pub(crate) fn gainmap(&self) -> bool { 149 matches!(self, Self::GainMap | Self::All) 150 } 151 } 152 153 #[derive(Debug)] 154 pub struct Settings { 155 pub source: Source, 156 pub ignore_exif: bool, 157 pub ignore_xmp: bool, 158 pub strictness: Strictness, 159 pub allow_progressive: bool, 160 pub allow_incremental: bool, 161 pub image_content_to_decode: ImageContentType, 162 pub codec_choice: CodecChoice, 163 pub image_size_limit: Option<NonZero<u32>>, 164 pub image_dimension_limit: Option<NonZero<u32>>, 165 pub image_count_limit: Option<NonZero<u32>>, 166 pub max_threads: u32, 167 pub android_mediacodec_output_color_format: AndroidMediaCodecOutputColorFormat, 168 } 169 170 impl Default for Settings { default() -> Self171 fn default() -> Self { 172 Self { 173 source: Default::default(), 174 ignore_exif: false, 175 ignore_xmp: false, 176 strictness: Default::default(), 177 allow_progressive: false, 178 allow_incremental: false, 179 image_content_to_decode: ImageContentType::ColorAndAlpha, 180 codec_choice: Default::default(), 181 image_size_limit: NonZero::new(DEFAULT_IMAGE_SIZE_LIMIT), 182 image_dimension_limit: NonZero::new(DEFAULT_IMAGE_DIMENSION_LIMIT), 183 image_count_limit: NonZero::new(DEFAULT_IMAGE_COUNT_LIMIT), 184 max_threads: 1, 185 android_mediacodec_output_color_format: AndroidMediaCodecOutputColorFormat::default(), 186 } 187 } 188 } 189 190 #[derive(Clone, Copy, Debug, Default)] 191 #[repr(C)] 192 pub struct Extent { 193 pub offset: u64, 194 pub size: usize, 195 } 196 197 impl Extent { merge(&mut self, extent: &Extent) -> AvifResult<()>198 fn merge(&mut self, extent: &Extent) -> AvifResult<()> { 199 if self.size == 0 { 200 *self = *extent; 201 return Ok(()); 202 } 203 if extent.size == 0 { 204 return Ok(()); 205 } 206 let max_extent_1 = checked_add!(self.offset, u64_from_usize(self.size)?)?; 207 let max_extent_2 = checked_add!(extent.offset, u64_from_usize(extent.size)?)?; 208 self.offset = min(self.offset, extent.offset); 209 // The extents may not be contiguous. It does not matter for nth_image_max_extent(). 210 self.size = usize_from_u64(checked_sub!(max(max_extent_1, max_extent_2), self.offset)?)?; 211 Ok(()) 212 } 213 } 214 215 #[derive(Debug)] 216 pub enum StrictnessFlag { 217 PixiRequired, 218 ClapValid, 219 AlphaIspeRequired, 220 } 221 222 #[derive(Debug, Default)] 223 pub enum Strictness { 224 None, 225 #[default] 226 All, 227 SpecificInclude(Vec<StrictnessFlag>), 228 SpecificExclude(Vec<StrictnessFlag>), 229 } 230 231 impl Strictness { pixi_required(&self) -> bool232 pub(crate) fn pixi_required(&self) -> bool { 233 match self { 234 Strictness::All => true, 235 Strictness::SpecificInclude(flags) => flags 236 .iter() 237 .any(|x| matches!(x, StrictnessFlag::PixiRequired)), 238 Strictness::SpecificExclude(flags) => !flags 239 .iter() 240 .any(|x| matches!(x, StrictnessFlag::PixiRequired)), 241 _ => false, 242 } 243 } 244 alpha_ispe_required(&self) -> bool245 pub(crate) fn alpha_ispe_required(&self) -> bool { 246 match self { 247 Strictness::All => true, 248 Strictness::SpecificInclude(flags) => flags 249 .iter() 250 .any(|x| matches!(x, StrictnessFlag::AlphaIspeRequired)), 251 Strictness::SpecificExclude(flags) => !flags 252 .iter() 253 .any(|x| matches!(x, StrictnessFlag::AlphaIspeRequired)), 254 _ => false, 255 } 256 } 257 } 258 259 #[repr(C)] 260 #[derive(Clone, Copy, Debug, Default)] 261 pub enum ProgressiveState { 262 #[default] 263 Unavailable = 0, 264 Available = 1, 265 Active = 2, 266 } 267 268 #[derive(Default, PartialEq)] 269 enum ParseState { 270 #[default] 271 None, 272 AwaitingSequenceHeader, 273 Complete, 274 } 275 276 /// cbindgen:field-names=[colorOBUSize,alphaOBUSize] 277 #[repr(C)] 278 #[derive(Clone, Copy, Debug, Default)] 279 pub struct IOStats { 280 pub color_obu_size: usize, 281 pub alpha_obu_size: usize, 282 } 283 284 #[derive(Default)] 285 pub struct Decoder { 286 pub settings: Settings, 287 image_count: u32, 288 image_index: i32, 289 image_timing: ImageTiming, 290 timescale: u64, 291 duration_in_timescales: u64, 292 duration: f64, 293 repetition_count: RepetitionCount, 294 gainmap: GainMap, 295 gainmap_present: bool, 296 image: Image, 297 source: Source, 298 tile_info: [TileInfo; Category::COUNT], 299 tiles: [Vec<Tile>; Category::COUNT], 300 items: Items, 301 tracks: Vec<Track>, 302 // To replicate the C-API, we need to keep this optional. Otherwise this 303 // could be part of the initialization. 304 io: Option<GenericIO>, 305 codecs: Vec<Codec>, 306 color_track_id: Option<u32>, 307 parse_state: ParseState, 308 io_stats: IOStats, 309 compression_format: CompressionFormat, 310 } 311 312 #[repr(C)] 313 #[derive(Clone, Copy, Debug, Default, PartialEq)] 314 pub enum CompressionFormat { 315 #[default] 316 Avif = 0, 317 Heic = 1, 318 } 319 320 pub struct GridImageHelper<'a> { 321 grid: &'a Grid, 322 image: &'a mut Image, 323 pub(crate) category: Category, 324 cell_index: usize, 325 codec_config: &'a CodecConfiguration, 326 first_cell_image: Option<Image>, 327 tile_width: u32, 328 tile_height: u32, 329 } 330 331 // These functions are not used in all configurations. 332 #[allow(unused)] 333 impl GridImageHelper<'_> { is_grid_complete(&self) -> AvifResult<bool>334 pub(crate) fn is_grid_complete(&self) -> AvifResult<bool> { 335 Ok(self.cell_index as u32 == checked_mul!(self.grid.rows, self.grid.columns)?) 336 } 337 copy_from_cell_image(&mut self, cell_image: &mut Image) -> AvifResult<()>338 pub(crate) fn copy_from_cell_image(&mut self, cell_image: &mut Image) -> AvifResult<()> { 339 if self.is_grid_complete()? { 340 return Ok(()); 341 } 342 if self.category == Category::Alpha && cell_image.yuv_range == YuvRange::Limited { 343 cell_image.alpha_to_full_range()?; 344 } 345 cell_image.scale(self.tile_width, self.tile_height, self.category)?; 346 if self.cell_index == 0 { 347 validate_grid_image_dimensions(cell_image, self.grid)?; 348 if self.category != Category::Alpha { 349 self.image.width = self.grid.width; 350 self.image.height = self.grid.height; 351 self.image 352 .copy_properties_from(cell_image, self.codec_config); 353 } 354 self.image.allocate_planes(self.category)?; 355 } else if !cell_image.has_same_properties_and_cicp(self.first_cell_image.unwrap_ref()) { 356 return Err(AvifError::InvalidImageGrid( 357 "grid image contains mismatched tiles".into(), 358 )); 359 } 360 self.image 361 .copy_from_tile(cell_image, self.grid, self.cell_index as u32, self.category)?; 362 if self.cell_index == 0 { 363 self.first_cell_image = Some(cell_image.shallow_clone()); 364 } 365 self.cell_index += 1; 366 Ok(()) 367 } 368 } 369 370 impl Decoder { image_count(&self) -> u32371 pub fn image_count(&self) -> u32 { 372 self.image_count 373 } image_index(&self) -> i32374 pub fn image_index(&self) -> i32 { 375 self.image_index 376 } image_timing(&self) -> ImageTiming377 pub fn image_timing(&self) -> ImageTiming { 378 self.image_timing 379 } timescale(&self) -> u64380 pub fn timescale(&self) -> u64 { 381 self.timescale 382 } duration_in_timescales(&self) -> u64383 pub fn duration_in_timescales(&self) -> u64 { 384 self.duration_in_timescales 385 } duration(&self) -> f64386 pub fn duration(&self) -> f64 { 387 self.duration 388 } repetition_count(&self) -> RepetitionCount389 pub fn repetition_count(&self) -> RepetitionCount { 390 self.repetition_count 391 } gainmap(&self) -> &GainMap392 pub fn gainmap(&self) -> &GainMap { 393 &self.gainmap 394 } gainmap_present(&self) -> bool395 pub fn gainmap_present(&self) -> bool { 396 self.gainmap_present 397 } io_stats(&self) -> IOStats398 pub fn io_stats(&self) -> IOStats { 399 self.io_stats 400 } compression_format(&self) -> CompressionFormat401 pub fn compression_format(&self) -> CompressionFormat { 402 self.compression_format 403 } 404 parsing_complete(&self) -> bool405 fn parsing_complete(&self) -> bool { 406 self.parse_state == ParseState::Complete 407 } 408 set_io_file(&mut self, filename: &String) -> AvifResult<()>409 pub fn set_io_file(&mut self, filename: &String) -> AvifResult<()> { 410 self.io = Some(Box::new(DecoderFileIO::create(filename)?)); 411 self.parse_state = ParseState::None; 412 Ok(()) 413 } 414 set_io_vec(&mut self, data: Vec<u8>)415 pub fn set_io_vec(&mut self, data: Vec<u8>) { 416 self.io = Some(Box::new(DecoderMemoryIO { data })); 417 self.parse_state = ParseState::None; 418 } 419 420 /// # Safety 421 /// 422 /// This function is intended for use only from the C API. The assumption is that the caller 423 /// will always pass in a valid pointer and size. set_io_raw(&mut self, data: *const u8, size: usize) -> AvifResult<()>424 pub unsafe fn set_io_raw(&mut self, data: *const u8, size: usize) -> AvifResult<()> { 425 self.io = Some(Box::new(unsafe { DecoderRawIO::create(data, size) })); 426 self.parse_state = ParseState::None; 427 Ok(()) 428 } 429 set_io(&mut self, io: GenericIO)430 pub fn set_io(&mut self, io: GenericIO) { 431 self.io = Some(io); 432 self.parse_state = ParseState::None; 433 } 434 find_alpha_item(&mut self, color_item_index: u32) -> AvifResult<Option<u32>>435 fn find_alpha_item(&mut self, color_item_index: u32) -> AvifResult<Option<u32>> { 436 let color_item = self.items.get(&color_item_index).unwrap(); 437 if let Some(item) = self.items.iter().find(|x| { 438 !x.1.should_skip() && x.1.aux_for_id == color_item.id && x.1.is_auxiliary_alpha() 439 }) { 440 return Ok(Some(*item.0)); 441 } 442 if !color_item.is_grid_item() || color_item.source_item_ids.is_empty() { 443 return Ok(None); 444 } 445 // If color item is a grid, check if there is an alpha channel which is represented as an 446 // auxl item to each color tile item. 447 let mut alpha_item_indices: Vec<u32> = create_vec_exact(color_item.source_item_ids.len())?; 448 for color_grid_item_id in &color_item.source_item_ids { 449 match self 450 .items 451 .iter() 452 .find(|x| x.1.aux_for_id == *color_grid_item_id && x.1.is_auxiliary_alpha()) 453 { 454 Some(item) => alpha_item_indices.push(*item.0), 455 None => { 456 if alpha_item_indices.is_empty() { 457 return Ok(None); 458 } else { 459 return Err(AvifError::BmffParseFailed( 460 "Some tiles but not all have an alpha auxiliary image item".into(), 461 )); 462 } 463 } 464 } 465 } 466 467 // Make up an alpha item for convenience. For the item_id, choose the first id that is not 468 // found in the actual image. In the very unlikely case that all the item ids are used, 469 // treat this as an image without alpha channel. 470 let alpha_item_id = match (1..u32::MAX).find(|&id| !self.items.contains_key(&id)) { 471 Some(id) => id, 472 None => return Ok(None), 473 }; 474 let first_item = self.items.get(&alpha_item_indices[0]).unwrap(); 475 let properties = match first_item.codec_config() { 476 Some(config) => vec![ItemProperty::CodecConfiguration(config.clone())], 477 None => return Ok(None), 478 }; 479 let alpha_item = Item { 480 id: alpha_item_id, 481 item_type: String::from("grid"), 482 width: color_item.width, 483 height: color_item.height, 484 source_item_ids: alpha_item_indices, 485 properties, 486 is_made_up: true, 487 ..Item::default() 488 }; 489 self.tile_info[Category::Alpha.usize()].grid = self.tile_info[Category::Color.usize()].grid; 490 self.items.insert(alpha_item_id, alpha_item); 491 Ok(Some(alpha_item_id)) 492 } 493 494 // returns (tone_mapped_image_item_id, gain_map_item_id) if found find_tone_mapped_image_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>>495 fn find_tone_mapped_image_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>> { 496 let tmap_items: Vec<_> = self.items.values().filter(|x| x.is_tmap()).collect(); 497 for item in tmap_items { 498 let dimg_items: Vec<_> = self 499 .items 500 .values() 501 .filter(|x| x.dimg_for_id == item.id) 502 .collect(); 503 if dimg_items.len() != 2 { 504 return Err(AvifError::InvalidToneMappedImage( 505 "Expected tmap to have 2 dimg items".into(), 506 )); 507 } 508 let item0 = if dimg_items[0].dimg_index == 0 { dimg_items[0] } else { dimg_items[1] }; 509 if item0.id != color_item_id { 510 continue; 511 } 512 let item1 = if dimg_items[0].dimg_index == 0 { dimg_items[1] } else { dimg_items[0] }; 513 return Ok(Some((item.id, item1.id))); 514 } 515 Ok(None) 516 } 517 518 // returns (tone_mapped_image_item_id, gain_map_item_id) if found find_gainmap_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>>519 fn find_gainmap_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>> { 520 if let Some((tonemap_id, gainmap_id)) = self.find_tone_mapped_image_item(color_item_id)? { 521 let gainmap_item = self 522 .items 523 .get(&gainmap_id) 524 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 525 if gainmap_item.should_skip() { 526 return Err(AvifError::InvalidToneMappedImage("".into())); 527 } 528 Ok(Some((tonemap_id, gainmap_id))) 529 } else { 530 Ok(None) 531 } 532 } 533 validate_gainmap_item(&mut self, gainmap_id: u32, tonemap_id: u32) -> AvifResult<()>534 fn validate_gainmap_item(&mut self, gainmap_id: u32, tonemap_id: u32) -> AvifResult<()> { 535 let gainmap_item = self 536 .items 537 .get(&gainmap_id) 538 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 539 // Find and adopt all colr boxes "at most one for a given value of colour type" 540 // (HEIF 6.5.5.1, from Amendment 3). Accept one of each type, and bail out if more than one 541 // of a given type is provided. 542 if let Some(nclx) = find_nclx(&gainmap_item.properties)? { 543 self.gainmap.image.color_primaries = nclx.color_primaries; 544 self.gainmap.image.transfer_characteristics = nclx.transfer_characteristics; 545 self.gainmap.image.matrix_coefficients = nclx.matrix_coefficients; 546 self.gainmap.image.yuv_range = nclx.yuv_range; 547 } 548 if tonemap_id == 0 { 549 return Ok(()); 550 } 551 // Find and adopt all colr boxes "at most one for a given value of colour type" 552 // (HEIF 6.5.5.1, from Amendment 3). Accept one of each type, and bail out if more than one 553 // of a given type is provided. 554 let tonemap_item = self 555 .items 556 .get(&tonemap_id) 557 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 558 if let Some(nclx) = find_nclx(&tonemap_item.properties)? { 559 self.gainmap.alt_color_primaries = nclx.color_primaries; 560 self.gainmap.alt_transfer_characteristics = nclx.transfer_characteristics; 561 self.gainmap.alt_matrix_coefficients = nclx.matrix_coefficients; 562 self.gainmap.alt_yuv_range = nclx.yuv_range; 563 } 564 if let Some(icc) = find_icc(&tonemap_item.properties)? { 565 self.gainmap.alt_icc.clone_from(icc); 566 } 567 if let Some(clli) = tonemap_item.clli() { 568 self.gainmap.alt_clli = *clli; 569 } 570 if let Some(pixi) = tonemap_item.pixi() { 571 self.gainmap.alt_plane_count = pixi.plane_depths.len() as u8; 572 self.gainmap.alt_plane_depth = pixi.plane_depths[0]; 573 } 574 // HEIC files created by Apple have some of these properties set in the Tonemap item. So do 575 // not perform this validation when HEIC is enabled. 576 #[cfg(not(feature = "heic"))] 577 if find_property!(tonemap_item.properties, PixelAspectRatio).is_some() 578 || find_property!(tonemap_item.properties, CleanAperture).is_some() 579 || find_property!(tonemap_item.properties, ImageRotation).is_some() 580 || find_property!(tonemap_item.properties, ImageMirror).is_some() 581 { 582 return Err(AvifError::InvalidToneMappedImage("".into())); 583 } 584 Ok(()) 585 } 586 search_exif_or_xmp_metadata( items: &mut Items, color_item_index: Option<u32>, settings: &Settings, io: &mut GenericIO, image: &mut Image, ) -> AvifResult<()>587 fn search_exif_or_xmp_metadata( 588 items: &mut Items, 589 color_item_index: Option<u32>, 590 settings: &Settings, 591 io: &mut GenericIO, 592 image: &mut Image, 593 ) -> AvifResult<()> { 594 if !settings.ignore_exif { 595 if let Some(exif) = items.iter_mut().rfind(|x| x.1.is_exif(color_item_index)) { 596 let mut stream = exif.1.stream(io)?; 597 exif::parse(&mut stream)?; 598 image 599 .exif 600 .extend_from_slice(stream.get_slice(stream.bytes_left()?)?); 601 } 602 } 603 if !settings.ignore_xmp { 604 if let Some(xmp) = items.iter_mut().rfind(|x| x.1.is_xmp(color_item_index)) { 605 let mut stream = xmp.1.stream(io)?; 606 image 607 .xmp 608 .extend_from_slice(stream.get_slice(stream.bytes_left()?)?); 609 } 610 } 611 Ok(()) 612 } 613 generate_tiles(&mut self, item_id: u32, category: Category) -> AvifResult<Vec<Tile>>614 fn generate_tiles(&mut self, item_id: u32, category: Category) -> AvifResult<Vec<Tile>> { 615 let mut tiles: Vec<Tile> = Vec::new(); 616 let item = self 617 .items 618 .get(&item_id) 619 .ok_or(AvifError::MissingImageItem)?; 620 if item.source_item_ids.is_empty() { 621 if item.size == 0 { 622 return Err(AvifError::MissingImageItem); 623 } 624 let mut tile = Tile::create_from_item( 625 self.items.get_mut(&item_id).unwrap(), 626 self.settings.allow_progressive, 627 self.settings.image_count_limit, 628 self.io.unwrap_ref().size_hint(), 629 )?; 630 tile.input.category = category; 631 tiles.push(tile); 632 } else { 633 if !self.tile_info[category.usize()].is_derived_image() { 634 return Err(AvifError::InvalidImageGrid( 635 "dimg items were found but image is not a derived image.".into(), 636 )); 637 } 638 let mut progressive = true; 639 for derived_item_id in item.source_item_ids.clone() { 640 let derived_item = self 641 .items 642 .get_mut(&derived_item_id) 643 .ok_or(AvifError::InvalidImageGrid("missing derived item".into()))?; 644 let mut tile = Tile::create_from_item( 645 derived_item, 646 self.settings.allow_progressive, 647 self.settings.image_count_limit, 648 self.io.unwrap_ref().size_hint(), 649 )?; 650 tile.input.category = category; 651 tiles.push(tile); 652 progressive = progressive && derived_item.progressive; 653 } 654 655 if category == Category::Color && progressive { 656 // Propagate the progressive status to the top-level item. 657 self.items.get_mut(&item_id).unwrap().progressive = true; 658 } 659 } 660 self.tile_info[category.usize()].tile_count = u32_from_usize(tiles.len())?; 661 Ok(tiles) 662 } 663 harvest_cicp_from_sequence_header(&mut self) -> AvifResult<()>664 fn harvest_cicp_from_sequence_header(&mut self) -> AvifResult<()> { 665 let category = Category::Color; 666 if self.tiles[category.usize()].is_empty() { 667 return Ok(()); 668 } 669 let mut search_size = 64; 670 while search_size < 4096 { 671 let tile_index = 0; 672 self.prepare_sample( 673 /*image_index=*/ 0, 674 category, 675 tile_index, 676 Some(search_size), 677 )?; 678 let io = &mut self.io.unwrap_mut(); 679 let sample = &self.tiles[category.usize()][tile_index].input.samples[0]; 680 let item_data_buffer = if sample.item_id == 0 { 681 &None 682 } else { 683 &self.items.get(&sample.item_id).unwrap().data_buffer 684 }; 685 if let Ok(sequence_header) = Av1SequenceHeader::parse_from_obus(sample.partial_data( 686 io, 687 item_data_buffer, 688 min(search_size, sample.size), 689 )?) { 690 self.image.color_primaries = sequence_header.color_primaries; 691 self.image.transfer_characteristics = sequence_header.transfer_characteristics; 692 self.image.matrix_coefficients = sequence_header.matrix_coefficients; 693 self.image.yuv_range = sequence_header.yuv_range; 694 break; 695 } 696 search_size += 64; 697 } 698 Ok(()) 699 } 700 701 // Populates the source item ids for a derived image item. 702 // These are the ids that are in the item's `dimg` box. populate_source_item_ids(&mut self, item_id: u32) -> AvifResult<()>703 fn populate_source_item_ids(&mut self, item_id: u32) -> AvifResult<()> { 704 if !self.items.get(&item_id).unwrap().is_derived_image_item() { 705 return Ok(()); 706 } 707 708 let mut source_item_ids: Vec<u32> = vec![]; 709 let mut first_codec_config: Option<CodecConfiguration> = None; 710 let mut first_icc: Option<Vec<u8>> = None; 711 // Collect all the dimg items. 712 for dimg_item_id in self.items.keys() { 713 if *dimg_item_id == item_id { 714 continue; 715 } 716 let dimg_item = self 717 .items 718 .get(dimg_item_id) 719 .ok_or(AvifError::InvalidImageGrid("".into()))?; 720 if dimg_item.dimg_for_id != item_id { 721 continue; 722 } 723 if !dimg_item.is_image_codec_item() || dimg_item.has_unsupported_essential_property { 724 return Err(AvifError::InvalidImageGrid( 725 "invalid input item in dimg".into(), 726 )); 727 } 728 if first_codec_config.is_none() { 729 // Adopt the configuration property of the first tile. 730 // validate_properties() makes sure they are all equal. 731 first_codec_config = Some( 732 dimg_item 733 .codec_config() 734 .ok_or(AvifError::BmffParseFailed( 735 "missing codec config property".into(), 736 ))? 737 .clone(), 738 ); 739 } 740 if dimg_item.is_image_codec_item() && first_icc.is_none() { 741 first_icc = find_icc(&dimg_item.properties)?.cloned(); 742 } 743 source_item_ids.push(*dimg_item_id); 744 } 745 if first_codec_config.is_none() { 746 // No derived images were found. 747 return Ok(()); 748 } 749 // The order of derived item ids matters: sort them by dimg_index, which is the order that 750 // items appear in the 'iref' box. 751 source_item_ids.sort_by_key(|k| self.items.get(k).unwrap().dimg_index); 752 let item = self.items.get_mut(&item_id).unwrap(); 753 item.properties.push(ItemProperty::CodecConfiguration( 754 first_codec_config.unwrap(), 755 )); 756 if (item.is_grid_item() || item.is_overlay_item()) 757 && first_icc.is_some() 758 && find_icc(&item.properties)?.is_none() 759 { 760 // For grid and overlay items, adopt the icc color profile of the first tile if it is 761 // not explicitly specified for the overall grid. 762 item.properties 763 .push(ItemProperty::ColorInformation(ColorInformation::Icc( 764 first_icc.unwrap().clone(), 765 ))); 766 } 767 item.source_item_ids = source_item_ids; 768 Ok(()) 769 } 770 validate_source_item_counts(&self, item_id: u32, tile_info: &TileInfo) -> AvifResult<()>771 fn validate_source_item_counts(&self, item_id: u32, tile_info: &TileInfo) -> AvifResult<()> { 772 let item = self.items.get(&item_id).unwrap(); 773 if item.is_grid_item() { 774 let tile_count = tile_info.grid_tile_count()? as usize; 775 if item.source_item_ids.len() != tile_count { 776 return Err(AvifError::InvalidImageGrid( 777 "Expected number of tiles not found".into(), 778 )); 779 } 780 } else if item.is_overlay_item() && item.source_item_ids.is_empty() { 781 return Err(AvifError::BmffParseFailed( 782 "No dimg items found for iovl".into(), 783 )); 784 } else if item.is_tmap() && item.source_item_ids.len() != 2 { 785 return Err(AvifError::InvalidToneMappedImage( 786 "Expected tmap to have 2 dimg items".into(), 787 )); 788 } 789 Ok(()) 790 } 791 reset(&mut self)792 fn reset(&mut self) { 793 let decoder = Decoder::default(); 794 // Reset all fields to default except the following: settings, io, source. 795 self.image_count = decoder.image_count; 796 self.image_timing = decoder.image_timing; 797 self.timescale = decoder.timescale; 798 self.duration_in_timescales = decoder.duration_in_timescales; 799 self.duration = decoder.duration; 800 self.repetition_count = decoder.repetition_count; 801 self.gainmap = decoder.gainmap; 802 self.gainmap_present = decoder.gainmap_present; 803 self.image = decoder.image; 804 self.tile_info = decoder.tile_info; 805 self.tiles = decoder.tiles; 806 self.image_index = decoder.image_index; 807 self.items = decoder.items; 808 self.tracks = decoder.tracks; 809 self.codecs = decoder.codecs; 810 self.color_track_id = decoder.color_track_id; 811 self.parse_state = decoder.parse_state; 812 self.compression_format = decoder.compression_format; 813 } 814 parse(&mut self) -> AvifResult<()>815 pub fn parse(&mut self) -> AvifResult<()> { 816 if self.parsing_complete() { 817 // Parse was called again. Reset the data and start over. 818 self.parse_state = ParseState::None; 819 } 820 if self.io.is_none() { 821 return Err(AvifError::IoNotSet); 822 } 823 824 if self.parse_state == ParseState::None { 825 self.reset(); 826 let avif_boxes = mp4box::parse(self.io.unwrap_mut())?; 827 self.tracks = avif_boxes.tracks; 828 if !self.tracks.is_empty() { 829 self.image.image_sequence_track_present = true; 830 for track in &self.tracks { 831 if track.is_video_handler() 832 && !track.check_limits( 833 self.settings.image_size_limit, 834 self.settings.image_dimension_limit, 835 ) 836 { 837 return Err(AvifError::BmffParseFailed( 838 "track dimension too large".into(), 839 )); 840 } 841 } 842 } 843 self.items = construct_items(&avif_boxes.meta)?; 844 if avif_boxes.ftyp.has_tmap() && !self.items.values().any(|x| x.item_type == "tmap") { 845 return Err(AvifError::BmffParseFailed( 846 "tmap was required but not found".into(), 847 )); 848 } 849 for item in self.items.values_mut() { 850 item.harvest_ispe( 851 self.settings.strictness.alpha_ispe_required(), 852 self.settings.image_size_limit, 853 self.settings.image_dimension_limit, 854 )?; 855 } 856 857 self.source = match self.settings.source { 858 // Decide the source based on the major brand. 859 Source::Auto => match avif_boxes.ftyp.major_brand.as_str() { 860 "avis" => Source::Tracks, 861 "avif" => Source::PrimaryItem, 862 _ => { 863 if self.tracks.is_empty() { 864 Source::PrimaryItem 865 } else { 866 Source::Tracks 867 } 868 } 869 }, 870 Source::Tracks => Source::Tracks, 871 Source::PrimaryItem => Source::PrimaryItem, 872 }; 873 874 let color_properties: &Vec<ItemProperty>; 875 let gainmap_properties: Option<&Vec<ItemProperty>>; 876 if self.source == Source::Tracks { 877 let color_track = self 878 .tracks 879 .iter() 880 .find(|x| x.is_color()) 881 .ok_or(AvifError::NoContent)?; 882 if let Some(meta) = &color_track.meta { 883 let mut color_track_items = construct_items(meta)?; 884 Self::search_exif_or_xmp_metadata( 885 &mut color_track_items, 886 None, 887 &self.settings, 888 self.io.unwrap_mut(), 889 &mut self.image, 890 )?; 891 } 892 self.color_track_id = Some(color_track.id); 893 color_properties = color_track 894 .get_properties() 895 .ok_or(AvifError::BmffParseFailed("".into()))?; 896 gainmap_properties = None; 897 898 self.tiles[Category::Color.usize()].push(Tile::create_from_track( 899 color_track, 900 self.settings.image_count_limit, 901 self.io.unwrap_ref().size_hint(), 902 Category::Color, 903 )?); 904 self.tile_info[Category::Color.usize()].tile_count = 1; 905 906 if let Some(alpha_track) = self 907 .tracks 908 .iter() 909 .find(|x| x.is_aux(color_track.id) && x.is_auxiliary_alpha()) 910 { 911 self.tiles[Category::Alpha.usize()].push(Tile::create_from_track( 912 alpha_track, 913 self.settings.image_count_limit, 914 self.io.unwrap_ref().size_hint(), 915 Category::Alpha, 916 )?); 917 self.tile_info[Category::Alpha.usize()].tile_count = 1; 918 self.image.alpha_present = true; 919 self.image.alpha_premultiplied = color_track.prem_by_id == Some(alpha_track.id); 920 } 921 922 self.image_index = -1; 923 self.image_count = 924 self.tiles[Category::Color.usize()][0].input.samples.len() as u32; 925 self.timescale = color_track.media_timescale as u64; 926 self.duration_in_timescales = color_track.media_duration; 927 if self.timescale != 0 { 928 self.duration = (self.duration_in_timescales as f64) / (self.timescale as f64); 929 } else { 930 self.duration = 0.0; 931 } 932 self.repetition_count = color_track.repetition_count()?; 933 self.image_timing = Default::default(); 934 935 self.image.width = color_track.width; 936 self.image.height = color_track.height; 937 } else { 938 assert_eq!(self.source, Source::PrimaryItem); 939 let mut item_ids: [u32; Category::COUNT] = [0; Category::COUNT]; 940 941 // Mandatory color item (primary item). 942 let color_item_id = self 943 .items 944 .iter() 945 .find(|x| { 946 !x.1.should_skip() 947 && x.1.id != 0 948 && x.1.id == avif_boxes.meta.primary_item_id 949 }) 950 .map(|it| *it.0); 951 952 item_ids[Category::Color.usize()] = color_item_id.ok_or(AvifError::NoContent)?; 953 self.read_and_parse_item(item_ids[Category::Color.usize()], Category::Color)?; 954 955 // Find exif/xmp from meta if any. 956 Self::search_exif_or_xmp_metadata( 957 &mut self.items, 958 Some(item_ids[Category::Color.usize()]), 959 &self.settings, 960 self.io.unwrap_mut(), 961 &mut self.image, 962 )?; 963 964 // Optional alpha auxiliary item 965 if let Some(alpha_item_id) = 966 self.find_alpha_item(item_ids[Category::Color.usize()])? 967 { 968 if !self.items.get(&alpha_item_id).unwrap().is_made_up { 969 self.read_and_parse_item(alpha_item_id, Category::Alpha)?; 970 } 971 item_ids[Category::Alpha.usize()] = alpha_item_id; 972 } 973 974 // Optional gainmap item 975 if avif_boxes.ftyp.has_tmap() { 976 if let Some((tonemap_id, gainmap_id)) = 977 self.find_gainmap_item(item_ids[Category::Color.usize()])? 978 { 979 self.validate_gainmap_item(gainmap_id, tonemap_id)?; 980 self.read_and_parse_item(gainmap_id, Category::Gainmap)?; 981 let tonemap_item = self 982 .items 983 .get_mut(&tonemap_id) 984 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 985 let mut stream = tonemap_item.stream(self.io.unwrap_mut())?; 986 if let Some(metadata) = mp4box::parse_tmap(&mut stream)? { 987 self.gainmap.metadata = metadata; 988 self.gainmap_present = true; 989 if self.settings.image_content_to_decode.gainmap() { 990 item_ids[Category::Gainmap.usize()] = gainmap_id; 991 } 992 } 993 } 994 } 995 996 self.image_index = -1; 997 self.image_count = 1; 998 self.timescale = 1; 999 self.duration = 1.0; 1000 self.duration_in_timescales = 1; 1001 self.image_timing.timescale = 1; 1002 self.image_timing.duration = 1.0; 1003 self.image_timing.duration_in_timescales = 1; 1004 1005 for category in Category::ALL { 1006 let item_id = item_ids[category.usize()]; 1007 if item_id == 0 { 1008 continue; 1009 } 1010 1011 let item = self.items.get(&item_id).unwrap(); 1012 if category == Category::Alpha && item.width == 0 && item.height == 0 { 1013 // NON-STANDARD: Alpha subimage does not have an ispe property; adopt 1014 // width/height from color item. 1015 assert!(!self.settings.strictness.alpha_ispe_required()); 1016 let color_item = 1017 self.items.get(&item_ids[Category::Color.usize()]).unwrap(); 1018 let width = color_item.width; 1019 let height = color_item.height; 1020 let alpha_item = self.items.get_mut(&item_id).unwrap(); 1021 // Note: We cannot directly use color_item.width here because borrow 1022 // checker won't allow that. 1023 alpha_item.width = width; 1024 alpha_item.height = height; 1025 } 1026 1027 self.tiles[category.usize()] = self.generate_tiles(item_id, category)?; 1028 let item = self.items.get(&item_id).unwrap(); 1029 // Made up alpha item does not contain the pixi property. So do not try to 1030 // validate it. 1031 let pixi_required = 1032 self.settings.strictness.pixi_required() && !item.is_made_up; 1033 item.validate_properties(&self.items, pixi_required)?; 1034 } 1035 1036 let color_item = self.items.get(&item_ids[Category::Color.usize()]).unwrap(); 1037 self.image.width = color_item.width; 1038 self.image.height = color_item.height; 1039 let alpha_item_id = item_ids[Category::Alpha.usize()]; 1040 self.image.alpha_present = alpha_item_id != 0; 1041 self.image.alpha_premultiplied = 1042 alpha_item_id != 0 && color_item.prem_by_id == alpha_item_id; 1043 1044 if color_item.progressive { 1045 self.image.progressive_state = ProgressiveState::Available; 1046 let sample_count = self.tiles[Category::Color.usize()][0].input.samples.len(); 1047 if sample_count > 1 { 1048 self.image.progressive_state = ProgressiveState::Active; 1049 self.image_count = sample_count as u32; 1050 } 1051 } 1052 1053 if item_ids[Category::Gainmap.usize()] != 0 { 1054 let gainmap_item = self 1055 .items 1056 .get(&item_ids[Category::Gainmap.usize()]) 1057 .unwrap(); 1058 self.gainmap.image.width = gainmap_item.width; 1059 self.gainmap.image.height = gainmap_item.height; 1060 let codec_config = gainmap_item 1061 .codec_config() 1062 .ok_or(AvifError::BmffParseFailed("".into()))?; 1063 self.gainmap.image.depth = codec_config.depth(); 1064 self.gainmap.image.yuv_format = codec_config.pixel_format(); 1065 self.gainmap.image.chroma_sample_position = 1066 codec_config.chroma_sample_position(); 1067 } 1068 1069 // This borrow has to be in the end of this branch. 1070 color_properties = &self 1071 .items 1072 .get(&item_ids[Category::Color.usize()]) 1073 .unwrap() 1074 .properties; 1075 gainmap_properties = if item_ids[Category::Gainmap.usize()] != 0 { 1076 Some( 1077 &self 1078 .items 1079 .get(&item_ids[Category::Gainmap.usize()]) 1080 .unwrap() 1081 .properties, 1082 ) 1083 } else { 1084 None 1085 }; 1086 } 1087 1088 // Check validity of samples. 1089 for tiles in &self.tiles { 1090 for tile in tiles { 1091 for sample in &tile.input.samples { 1092 if sample.size == 0 { 1093 return Err(AvifError::BmffParseFailed( 1094 "sample has invalid size.".into(), 1095 )); 1096 } 1097 match tile.input.category { 1098 Category::Color => { 1099 checked_incr!(self.io_stats.color_obu_size, sample.size) 1100 } 1101 Category::Alpha => { 1102 checked_incr!(self.io_stats.alpha_obu_size, sample.size) 1103 } 1104 _ => {} 1105 } 1106 } 1107 } 1108 } 1109 1110 // Find and adopt all colr boxes "at most one for a given value of colour type" 1111 // (HEIF 6.5.5.1, from Amendment 3) Accept one of each type, and bail out if more than one 1112 // of a given type is provided. 1113 let mut cicp_set = false; 1114 1115 if let Some(nclx) = find_nclx(color_properties)? { 1116 self.image.color_primaries = nclx.color_primaries; 1117 self.image.transfer_characteristics = nclx.transfer_characteristics; 1118 self.image.matrix_coefficients = nclx.matrix_coefficients; 1119 self.image.yuv_range = nclx.yuv_range; 1120 cicp_set = true; 1121 } 1122 if let Some(icc) = find_icc(color_properties)? { 1123 self.image.icc.clone_from(icc); 1124 } 1125 1126 self.image.clli = find_property!(color_properties, ContentLightLevelInformation); 1127 self.image.pasp = find_property!(color_properties, PixelAspectRatio); 1128 self.image.clap = find_property!(color_properties, CleanAperture); 1129 self.image.irot_angle = find_property!(color_properties, ImageRotation); 1130 self.image.imir_axis = find_property!(color_properties, ImageMirror); 1131 1132 if let Some(gainmap_properties) = gainmap_properties { 1133 // Ensure that the bitstream contains the same 'pasp', 'clap', 'irot and 'imir' 1134 // properties for both the base and gain map image items. 1135 if self.image.pasp != find_property!(gainmap_properties, PixelAspectRatio) 1136 || self.image.clap != find_property!(gainmap_properties, CleanAperture) 1137 || self.image.irot_angle != find_property!(gainmap_properties, ImageRotation) 1138 || self.image.imir_axis != find_property!(gainmap_properties, ImageMirror) 1139 { 1140 return Err(AvifError::DecodeGainMapFailed); 1141 } 1142 } 1143 1144 let codec_config = find_property!(color_properties, CodecConfiguration) 1145 .ok_or(AvifError::BmffParseFailed("".into()))?; 1146 self.image.depth = codec_config.depth(); 1147 self.image.yuv_format = codec_config.pixel_format(); 1148 self.image.chroma_sample_position = codec_config.chroma_sample_position(); 1149 self.compression_format = if codec_config.is_avif() { 1150 CompressionFormat::Avif 1151 } else { 1152 CompressionFormat::Heic 1153 }; 1154 1155 if cicp_set { 1156 self.parse_state = ParseState::Complete; 1157 return Ok(()); 1158 } 1159 self.parse_state = ParseState::AwaitingSequenceHeader; 1160 } 1161 1162 // If cicp was not set, try to harvest it from the sequence header. 1163 self.harvest_cicp_from_sequence_header()?; 1164 self.parse_state = ParseState::Complete; 1165 1166 Ok(()) 1167 } 1168 read_and_parse_item(&mut self, item_id: u32, category: Category) -> AvifResult<()>1169 fn read_and_parse_item(&mut self, item_id: u32, category: Category) -> AvifResult<()> { 1170 if item_id == 0 { 1171 return Ok(()); 1172 } 1173 self.populate_source_item_ids(item_id)?; 1174 self.items.get_mut(&item_id).unwrap().read_and_parse( 1175 self.io.unwrap_mut(), 1176 &mut self.tile_info[category.usize()].grid, 1177 &mut self.tile_info[category.usize()].overlay, 1178 self.settings.image_size_limit, 1179 self.settings.image_dimension_limit, 1180 )?; 1181 self.validate_source_item_counts(item_id, &self.tile_info[category.usize()]) 1182 } 1183 can_use_single_codec(&self) -> AvifResult<bool>1184 fn can_use_single_codec(&self) -> AvifResult<bool> { 1185 let total_tile_count = checked_add!( 1186 checked_add!(self.tiles[0].len(), self.tiles[1].len())?, 1187 self.tiles[2].len() 1188 )?; 1189 if total_tile_count == 1 { 1190 return Ok(true); 1191 } 1192 if self.image_count != 1 { 1193 return Ok(false); 1194 } 1195 let mut image_buffers = 0; 1196 let mut stolen_image_buffers = 0; 1197 for category in Category::ALL_USIZE { 1198 if self.tile_info[category].tile_count > 0 { 1199 image_buffers += 1; 1200 } 1201 if self.tile_info[category].tile_count == 1 { 1202 stolen_image_buffers += 1; 1203 } 1204 } 1205 if stolen_image_buffers > 0 && image_buffers > 1 { 1206 // Stealing will cause problems. So we need separate codec instances. 1207 return Ok(false); 1208 } 1209 let operating_point = self.tiles[0][0].operating_point; 1210 let all_layers = self.tiles[0][0].input.all_layers; 1211 for tiles in &self.tiles { 1212 for tile in tiles { 1213 if tile.operating_point != operating_point || tile.input.all_layers != all_layers { 1214 return Ok(false); 1215 } 1216 } 1217 } 1218 Ok(true) 1219 } 1220 create_codec(&mut self, category: Category, tile_index: usize) -> AvifResult<()>1221 fn create_codec(&mut self, category: Category, tile_index: usize) -> AvifResult<()> { 1222 let tile = &self.tiles[category.usize()][tile_index]; 1223 let mut codec: Codec = self 1224 .settings 1225 .codec_choice 1226 .get_codec(tile.codec_config.is_avif())?; 1227 let config = DecoderConfig { 1228 operating_point: tile.operating_point, 1229 all_layers: tile.input.all_layers, 1230 width: tile.width, 1231 height: tile.height, 1232 depth: self.image.depth, 1233 max_threads: self.settings.max_threads, 1234 image_size_limit: self.settings.image_size_limit, 1235 max_input_size: tile.max_sample_size(), 1236 codec_config: tile.codec_config.clone(), 1237 category, 1238 android_mediacodec_output_color_format: self 1239 .settings 1240 .android_mediacodec_output_color_format, 1241 }; 1242 codec.initialize(&config)?; 1243 self.codecs.push(codec); 1244 Ok(()) 1245 } 1246 create_codecs(&mut self) -> AvifResult<()>1247 fn create_codecs(&mut self) -> AvifResult<()> { 1248 if !self.codecs.is_empty() { 1249 return Ok(()); 1250 } 1251 if matches!(self.source, Source::Tracks) || cfg!(feature = "android_mediacodec") { 1252 // In this case, there are two possibilities in the following order: 1253 // 1) If source is Tracks, then we will use at most two codec instances (one each for 1254 // Color and Alpha). Gainmap will always be empty. 1255 // 2) If android_mediacodec is true, then we will use at most three codec instances 1256 // (one for each category). 1257 self.codecs = create_vec_exact(3)?; 1258 for category in self.settings.image_content_to_decode.categories() { 1259 if self.tiles[category.usize()].is_empty() { 1260 continue; 1261 } 1262 self.create_codec(category, 0)?; 1263 for tile in &mut self.tiles[category.usize()] { 1264 tile.codec_index = self.codecs.len() - 1; 1265 } 1266 } 1267 } else if self.can_use_single_codec()? { 1268 self.codecs = create_vec_exact(1)?; 1269 self.create_codec(Category::Color, 0)?; 1270 for tiles in &mut self.tiles { 1271 for tile in tiles { 1272 tile.codec_index = 0; 1273 } 1274 } 1275 } else { 1276 self.codecs = create_vec_exact(self.tiles.iter().map(|tiles| tiles.len()).sum())?; 1277 for category in self.settings.image_content_to_decode.categories() { 1278 for tile_index in 0..self.tiles[category.usize()].len() { 1279 self.create_codec(category, tile_index)?; 1280 self.tiles[category.usize()][tile_index].codec_index = self.codecs.len() - 1; 1281 } 1282 } 1283 } 1284 Ok(()) 1285 } 1286 prepare_sample( &mut self, image_index: usize, category: Category, tile_index: usize, max_num_bytes: Option<usize>, ) -> AvifResult<()>1287 fn prepare_sample( 1288 &mut self, 1289 image_index: usize, 1290 category: Category, 1291 tile_index: usize, 1292 max_num_bytes: Option<usize>, // Bytes read past that size will be ignored. 1293 ) -> AvifResult<()> { 1294 let tile = &mut self.tiles[category.usize()][tile_index]; 1295 if tile.input.samples.len() <= image_index { 1296 return Err(AvifError::NoImagesRemaining); 1297 } 1298 let sample = &tile.input.samples[image_index]; 1299 if sample.item_id == 0 { 1300 // Data comes from a track. Nothing to prepare. 1301 return Ok(()); 1302 } 1303 // Data comes from an item. 1304 let item = self 1305 .items 1306 .get_mut(&sample.item_id) 1307 .ok_or(AvifError::BmffParseFailed("".into()))?; 1308 if item.extents.len() == 1 { 1309 // Item has only one extent. Nothing to prepare. 1310 return Ok(()); 1311 } 1312 if let Some(data) = &item.data_buffer { 1313 if data.len() == item.size { 1314 return Ok(()); // All extents have already been merged. 1315 } 1316 if max_num_bytes.is_some_and(|max_num_bytes| data.len() >= max_num_bytes) { 1317 return Ok(()); // Some sufficient extents have already been merged. 1318 } 1319 } 1320 // Item has multiple extents, merge them into a contiguous buffer. 1321 if item.data_buffer.is_none() { 1322 item.data_buffer = Some(create_vec_exact(item.size)?); 1323 } 1324 let data = item.data_buffer.unwrap_mut(); 1325 let mut bytes_to_skip = data.len(); // These extents were already merged. 1326 for extent in &item.extents { 1327 if bytes_to_skip != 0 { 1328 checked_decr!(bytes_to_skip, extent.size); 1329 continue; 1330 } 1331 let io = self.io.unwrap_mut(); 1332 data.extend_from_slice(io.read_exact(extent.offset, extent.size)?); 1333 if max_num_bytes.is_some_and(|max_num_bytes| data.len() >= max_num_bytes) { 1334 return Ok(()); // There are enough merged extents to satisfy max_num_bytes. 1335 } 1336 } 1337 assert_eq!(bytes_to_skip, 0); 1338 assert_eq!(data.len(), item.size); 1339 Ok(()) 1340 } 1341 prepare_samples(&mut self, image_index: usize) -> AvifResult<()>1342 fn prepare_samples(&mut self, image_index: usize) -> AvifResult<()> { 1343 for category in self.settings.image_content_to_decode.categories() { 1344 for tile_index in 0..self.tiles[category.usize()].len() { 1345 self.prepare_sample(image_index, category, tile_index, None)?; 1346 } 1347 } 1348 Ok(()) 1349 } 1350 decode_tile( &mut self, image_index: usize, category: Category, tile_index: usize, ) -> AvifResult<()>1351 fn decode_tile( 1352 &mut self, 1353 image_index: usize, 1354 category: Category, 1355 tile_index: usize, 1356 ) -> AvifResult<()> { 1357 // Split the tiles array into two mutable arrays so that we can validate the 1358 // properties of tiles with index > 0 with that of the first tile. 1359 let (tiles_slice1, tiles_slice2) = self.tiles[category.usize()].split_at_mut(tile_index); 1360 let tile = &mut tiles_slice2[0]; 1361 let sample = &tile.input.samples[image_index]; 1362 let io = &mut self.io.unwrap_mut(); 1363 1364 let codec = &mut self.codecs[tile.codec_index]; 1365 let item_data_buffer = if sample.item_id == 0 { 1366 &None 1367 } else { 1368 &self.items.get(&sample.item_id).unwrap().data_buffer 1369 }; 1370 let data = sample.data(io, item_data_buffer)?; 1371 let next_image_result = 1372 codec.get_next_image(data, sample.spatial_id, &mut tile.image, category); 1373 if next_image_result.is_err() { 1374 if cfg!(feature = "android_mediacodec") 1375 && cfg!(feature = "heic") 1376 && tile.codec_config.is_heic() 1377 && category == Category::Alpha 1378 { 1379 // When decoding HEIC on Android, if the alpha channel decoding fails, simply 1380 // ignore it and return the rest of the image. 1381 checked_incr!(self.tile_info[category.usize()].decoded_tile_count, 1); 1382 return Ok(()); 1383 } else { 1384 return next_image_result; 1385 } 1386 } 1387 1388 checked_incr!(self.tile_info[category.usize()].decoded_tile_count, 1); 1389 1390 if category == Category::Alpha && tile.image.yuv_range == YuvRange::Limited { 1391 tile.image.alpha_to_full_range()?; 1392 } 1393 tile.image.scale(tile.width, tile.height, category)?; 1394 1395 if self.tile_info[category.usize()].is_grid() { 1396 if tile_index == 0 { 1397 let grid = &self.tile_info[category.usize()].grid; 1398 validate_grid_image_dimensions(&tile.image, grid)?; 1399 match category { 1400 Category::Color => { 1401 self.image.width = grid.width; 1402 self.image.height = grid.height; 1403 self.image 1404 .copy_properties_from(&tile.image, &tile.codec_config); 1405 self.image.allocate_planes(category)?; 1406 } 1407 Category::Alpha => { 1408 // Alpha is always just one plane and the depth has been validated 1409 // to be the same as the color planes' depth. 1410 self.image.allocate_planes(category)?; 1411 } 1412 Category::Gainmap => { 1413 self.gainmap.image.width = grid.width; 1414 self.gainmap.image.height = grid.height; 1415 self.gainmap 1416 .image 1417 .copy_properties_from(&tile.image, &tile.codec_config); 1418 self.gainmap.image.allocate_planes(category)?; 1419 } 1420 } 1421 } 1422 if !tiles_slice1.is_empty() 1423 && !tile 1424 .image 1425 .has_same_properties_and_cicp(&tiles_slice1[0].image) 1426 { 1427 return Err(AvifError::InvalidImageGrid( 1428 "grid image contains mismatched tiles".into(), 1429 )); 1430 } 1431 match category { 1432 Category::Gainmap => self.gainmap.image.copy_from_tile( 1433 &tile.image, 1434 &self.tile_info[category.usize()].grid, 1435 tile_index as u32, 1436 category, 1437 )?, 1438 _ => { 1439 self.image.copy_from_tile( 1440 &tile.image, 1441 &self.tile_info[category.usize()].grid, 1442 tile_index as u32, 1443 category, 1444 )?; 1445 } 1446 } 1447 } else if self.tile_info[category.usize()].is_overlay() { 1448 if tile_index == 0 { 1449 let overlay = &self.tile_info[category.usize()].overlay; 1450 let canvas_fill_values = 1451 self.image.convert_rgba16_to_yuva(overlay.canvas_fill_value); 1452 match category { 1453 Category::Color => { 1454 self.image.width = overlay.width; 1455 self.image.height = overlay.height; 1456 self.image 1457 .copy_properties_from(&tile.image, &tile.codec_config); 1458 self.image 1459 .allocate_planes_with_default_values(category, canvas_fill_values)?; 1460 } 1461 Category::Alpha => { 1462 // Alpha is always just one plane and the depth has been validated 1463 // to be the same as the color planes' depth. 1464 self.image 1465 .allocate_planes_with_default_values(category, canvas_fill_values)?; 1466 } 1467 Category::Gainmap => { 1468 self.gainmap.image.width = overlay.width; 1469 self.gainmap.image.height = overlay.height; 1470 self.gainmap 1471 .image 1472 .copy_properties_from(&tile.image, &tile.codec_config); 1473 self.gainmap 1474 .image 1475 .allocate_planes_with_default_values(category, canvas_fill_values)?; 1476 } 1477 } 1478 } 1479 if !tiles_slice1.is_empty() { 1480 let first_tile_image = &tiles_slice1[0].image; 1481 if tile.image.width != first_tile_image.width 1482 || tile.image.height != first_tile_image.height 1483 || tile.image.depth != first_tile_image.depth 1484 || tile.image.yuv_format != first_tile_image.yuv_format 1485 || tile.image.yuv_range != first_tile_image.yuv_range 1486 || tile.image.color_primaries != first_tile_image.color_primaries 1487 || tile.image.transfer_characteristics 1488 != first_tile_image.transfer_characteristics 1489 || tile.image.matrix_coefficients != first_tile_image.matrix_coefficients 1490 { 1491 return Err(AvifError::InvalidImageGrid( 1492 "overlay image contains mismatched tiles".into(), 1493 )); 1494 } 1495 } 1496 match category { 1497 Category::Gainmap => self.gainmap.image.copy_and_overlay_from_tile( 1498 &tile.image, 1499 &self.tile_info[category.usize()], 1500 tile_index as u32, 1501 category, 1502 )?, 1503 _ => { 1504 self.image.copy_and_overlay_from_tile( 1505 &tile.image, 1506 &self.tile_info[category.usize()], 1507 tile_index as u32, 1508 category, 1509 )?; 1510 } 1511 } 1512 } else { 1513 // Non grid/overlay path, steal or copy planes from the only tile. 1514 match category { 1515 Category::Color => { 1516 self.image.width = tile.image.width; 1517 self.image.height = tile.image.height; 1518 self.image 1519 .copy_properties_from(&tile.image, &tile.codec_config); 1520 self.image 1521 .steal_or_copy_planes_from(&tile.image, category)?; 1522 } 1523 Category::Alpha => { 1524 if !self.image.has_same_properties(&tile.image) { 1525 return Err(AvifError::DecodeAlphaFailed); 1526 } 1527 self.image 1528 .steal_or_copy_planes_from(&tile.image, category)?; 1529 } 1530 Category::Gainmap => { 1531 self.gainmap.image.width = tile.image.width; 1532 self.gainmap.image.height = tile.image.height; 1533 self.gainmap 1534 .image 1535 .copy_properties_from(&tile.image, &tile.codec_config); 1536 self.gainmap 1537 .image 1538 .steal_or_copy_planes_from(&tile.image, category)?; 1539 } 1540 } 1541 } 1542 Ok(()) 1543 } 1544 decode_grid(&mut self, image_index: usize, category: Category) -> AvifResult<()>1545 fn decode_grid(&mut self, image_index: usize, category: Category) -> AvifResult<()> { 1546 let tile_count = self.tiles[category.usize()].len(); 1547 if tile_count == 0 { 1548 return Ok(()); 1549 } 1550 let previous_decoded_tile_count = 1551 self.tile_info[category.usize()].decoded_tile_count as usize; 1552 let mut payloads = vec![]; 1553 for tile_index in previous_decoded_tile_count..tile_count { 1554 let tile = &self.tiles[category.usize()][tile_index]; 1555 let sample = &tile.input.samples[image_index]; 1556 let item_data_buffer = if sample.item_id == 0 { 1557 &None 1558 } else { 1559 &self.items.get(&sample.item_id).unwrap().data_buffer 1560 }; 1561 let io = &mut self.io.unwrap_mut(); 1562 let data = sample.data(io, item_data_buffer)?; 1563 payloads.push(data.to_vec()); 1564 } 1565 let grid = &self.tile_info[category.usize()].grid; 1566 if checked_mul!(grid.rows, grid.columns)? != payloads.len() as u32 { 1567 return Err(AvifError::InvalidArgument); 1568 } 1569 let first_tile = &self.tiles[category.usize()][previous_decoded_tile_count]; 1570 let mut grid_image_helper = GridImageHelper { 1571 grid, 1572 image: if category == Category::Gainmap { 1573 &mut self.gainmap.image 1574 } else { 1575 &mut self.image 1576 }, 1577 category, 1578 cell_index: 0, 1579 codec_config: &first_tile.codec_config, 1580 first_cell_image: None, 1581 tile_width: first_tile.width, 1582 tile_height: first_tile.height, 1583 }; 1584 let codec = &mut self.codecs[first_tile.codec_index]; 1585 let next_image_result = codec.get_next_image_grid( 1586 &payloads, 1587 first_tile.input.samples[image_index].spatial_id, 1588 &mut grid_image_helper, 1589 ); 1590 if next_image_result.is_err() { 1591 if cfg!(feature = "android_mediacodec") 1592 && cfg!(feature = "heic") 1593 && first_tile.codec_config.is_heic() 1594 && category == Category::Alpha 1595 { 1596 // When decoding HEIC on Android, if the alpha channel decoding fails, simply 1597 // ignore it and return the rest of the image. 1598 } else { 1599 return next_image_result; 1600 } 1601 } 1602 if !grid_image_helper.is_grid_complete()? { 1603 return Err(AvifError::UnknownError( 1604 "codec did not decode all cells".into(), 1605 )); 1606 } 1607 checked_incr!( 1608 self.tile_info[category.usize()].decoded_tile_count, 1609 u32_from_usize(payloads.len())? 1610 ); 1611 Ok(()) 1612 } 1613 decode_tiles(&mut self, image_index: usize) -> AvifResult<()>1614 fn decode_tiles(&mut self, image_index: usize) -> AvifResult<()> { 1615 let mut decoded_something = false; 1616 for category in self.settings.image_content_to_decode.categories() { 1617 let tile_count = self.tiles[category.usize()].len(); 1618 if tile_count == 0 { 1619 continue; 1620 } 1621 let first_tile = &self.tiles[category.usize()][0]; 1622 let codec = self.codecs[first_tile.codec_index].codec(); 1623 if codec == CodecChoice::MediaCodec 1624 && !self.settings.allow_incremental 1625 && self.tile_info[category.usize()].is_grid() 1626 { 1627 self.decode_grid(image_index, category)?; 1628 decoded_something = true; 1629 } else { 1630 let previous_decoded_tile_count = 1631 self.tile_info[category.usize()].decoded_tile_count as usize; 1632 for tile_index in previous_decoded_tile_count..tile_count { 1633 self.decode_tile(image_index, category, tile_index)?; 1634 decoded_something = true; 1635 } 1636 } 1637 } 1638 if decoded_something { 1639 Ok(()) 1640 } else { 1641 Err(AvifError::NoContent) 1642 } 1643 } 1644 next_image(&mut self) -> AvifResult<()>1645 pub fn next_image(&mut self) -> AvifResult<()> { 1646 if self.io.is_none() { 1647 return Err(AvifError::IoNotSet); 1648 } 1649 if !self.parsing_complete() { 1650 return Err(AvifError::NoContent); 1651 } 1652 if self.is_current_frame_fully_decoded() { 1653 for category in Category::ALL_USIZE { 1654 self.tile_info[category].decoded_tile_count = 0; 1655 } 1656 } 1657 1658 let next_image_index = checked_add!(self.image_index, 1)?; 1659 self.create_codecs()?; 1660 self.prepare_samples(next_image_index as usize)?; 1661 self.decode_tiles(next_image_index as usize)?; 1662 self.image_index = next_image_index; 1663 self.image_timing = self.nth_image_timing(self.image_index as u32)?; 1664 Ok(()) 1665 } 1666 is_current_frame_fully_decoded(&self) -> bool1667 fn is_current_frame_fully_decoded(&self) -> bool { 1668 if !self.parsing_complete() { 1669 return false; 1670 } 1671 for category in self.settings.image_content_to_decode.categories() { 1672 if !self.tile_info[category.usize()].is_fully_decoded() { 1673 return false; 1674 } 1675 } 1676 true 1677 } 1678 nth_image(&mut self, index: u32) -> AvifResult<()>1679 pub fn nth_image(&mut self, index: u32) -> AvifResult<()> { 1680 if !self.parsing_complete() { 1681 return Err(AvifError::NoContent); 1682 } 1683 if index >= self.image_count { 1684 return Err(AvifError::NoImagesRemaining); 1685 } 1686 let requested_index = i32_from_u32(index)?; 1687 if requested_index == checked_add!(self.image_index, 1)? { 1688 return self.next_image(); 1689 } 1690 if requested_index == self.image_index && self.is_current_frame_fully_decoded() { 1691 // Current frame which is already fully decoded has been requested. Do nothing. 1692 return Ok(()); 1693 } 1694 let nearest_keyframe = i32_from_u32(self.nearest_keyframe(index))?; 1695 if nearest_keyframe > checked_add!(self.image_index, 1)? 1696 || requested_index <= self.image_index 1697 { 1698 // Start decoding from the nearest keyframe. 1699 self.image_index = nearest_keyframe - 1; 1700 } 1701 loop { 1702 self.next_image()?; 1703 if requested_index == self.image_index { 1704 break; 1705 } 1706 } 1707 Ok(()) 1708 } 1709 image(&self) -> Option<&Image>1710 pub fn image(&self) -> Option<&Image> { 1711 if self.parsing_complete() { 1712 Some(&self.image) 1713 } else { 1714 None 1715 } 1716 } 1717 nth_image_timing(&self, n: u32) -> AvifResult<ImageTiming>1718 pub fn nth_image_timing(&self, n: u32) -> AvifResult<ImageTiming> { 1719 if !self.parsing_complete() { 1720 return Err(AvifError::NoContent); 1721 } 1722 if let Some(limit) = self.settings.image_count_limit { 1723 if n > limit.get() { 1724 return Err(AvifError::NoImagesRemaining); 1725 } 1726 } 1727 if self.color_track_id.is_none() { 1728 return Ok(self.image_timing); 1729 } 1730 let color_track_id = self.color_track_id.unwrap(); 1731 let color_track = self 1732 .tracks 1733 .iter() 1734 .find(|x| x.id == color_track_id) 1735 .ok_or(AvifError::NoContent)?; 1736 if color_track.sample_table.is_none() { 1737 return Ok(self.image_timing); 1738 } 1739 color_track.image_timing(n) 1740 } 1741 1742 // When next_image() or nth_image() returns AvifResult::WaitingOnIo, this function can be called 1743 // next to retrieve the number of top rows that can be immediately accessed from the luma plane 1744 // of decoder->image, and alpha if any. The corresponding rows from the chroma planes, 1745 // if any, can also be accessed (half rounded up if subsampled, same number of rows otherwise). 1746 // If a gain map is present, and image_content_to_decode contains ImageContentType::GainMap, 1747 // the gain map's planes can also be accessed in the same way. 1748 // The number of available gain map rows is at least: 1749 // decoder.decoded_row_count() * decoder.gainmap.image.height / decoder.image.height 1750 // When gain map scaling is needed, callers might choose to use a few less rows depending on how 1751 // many rows are needed by the scaling algorithm, to avoid the last row(s) changing when more 1752 // data becomes available. allow_incremental must be set to true before calling next_image() or 1753 // nth_image(). Returns decoder.image.height when the last call to next_image() or nth_image() 1754 // returned AvifResult::Ok. Returns 0 in all other cases. decoded_row_count(&self) -> u321755 pub fn decoded_row_count(&self) -> u32 { 1756 let mut min_row_count = self.image.height; 1757 for category in Category::ALL_USIZE { 1758 if self.tiles[category].is_empty() { 1759 continue; 1760 } 1761 let first_tile_height = self.tiles[category][0].height; 1762 let row_count = if category == Category::Gainmap.usize() 1763 && self.gainmap_present() 1764 && self.settings.image_content_to_decode.gainmap() 1765 && self.gainmap.image.height != 0 1766 && self.gainmap.image.height != self.image.height 1767 { 1768 if self.tile_info[category].is_fully_decoded() { 1769 self.image.height 1770 } else { 1771 let gainmap_row_count = self.tile_info[category] 1772 .decoded_row_count(self.gainmap.image.height, first_tile_height); 1773 // row_count fits for sure in 32 bits because heights do. 1774 let row_count = (gainmap_row_count as u64 * self.image.height as u64 1775 / self.gainmap.image.height as u64) 1776 as u32; 1777 1778 // Make sure it satisfies the C API guarantee. 1779 assert!( 1780 gainmap_row_count 1781 >= (row_count as f32 / self.image.height as f32 1782 * self.gainmap.image.height as f32) 1783 .round() as u32 1784 ); 1785 row_count 1786 } 1787 } else { 1788 self.tile_info[category].decoded_row_count(self.image.height, first_tile_height) 1789 }; 1790 min_row_count = std::cmp::min(min_row_count, row_count); 1791 } 1792 min_row_count 1793 } 1794 is_keyframe(&self, index: u32) -> bool1795 pub fn is_keyframe(&self, index: u32) -> bool { 1796 if !self.parsing_complete() { 1797 return false; 1798 } 1799 let index = index as usize; 1800 // All the tiles for the requested index must be a keyframe. 1801 for category in Category::ALL_USIZE { 1802 for tile in &self.tiles[category] { 1803 if index >= tile.input.samples.len() || !tile.input.samples[index].sync { 1804 return false; 1805 } 1806 } 1807 } 1808 true 1809 } 1810 nearest_keyframe(&self, mut index: u32) -> u321811 pub fn nearest_keyframe(&self, mut index: u32) -> u32 { 1812 if !self.parsing_complete() { 1813 return 0; 1814 } 1815 while index != 0 { 1816 if self.is_keyframe(index) { 1817 return index; 1818 } 1819 index -= 1; 1820 } 1821 assert!(self.is_keyframe(0)); 1822 0 1823 } 1824 nth_image_max_extent(&self, index: u32) -> AvifResult<Extent>1825 pub fn nth_image_max_extent(&self, index: u32) -> AvifResult<Extent> { 1826 if !self.parsing_complete() { 1827 return Err(AvifError::NoContent); 1828 } 1829 let mut extent = Extent::default(); 1830 let start_index = self.nearest_keyframe(index) as usize; 1831 let end_index = index as usize; 1832 for current_index in start_index..=end_index { 1833 for category in Category::ALL_USIZE { 1834 for tile in &self.tiles[category] { 1835 if current_index >= tile.input.samples.len() { 1836 return Err(AvifError::NoImagesRemaining); 1837 } 1838 let sample = &tile.input.samples[current_index]; 1839 let sample_extent = if sample.item_id != 0 { 1840 let item = self.items.get(&sample.item_id).unwrap(); 1841 item.max_extent(sample)? 1842 } else { 1843 Extent { 1844 offset: sample.offset, 1845 size: sample.size, 1846 } 1847 }; 1848 extent.merge(&sample_extent)?; 1849 } 1850 } 1851 } 1852 Ok(extent) 1853 } 1854 peek_compatible_file_type(data: &[u8]) -> bool1855 pub fn peek_compatible_file_type(data: &[u8]) -> bool { 1856 mp4box::peek_compatible_file_type(data).unwrap_or(false) 1857 } 1858 } 1859 1860 #[cfg(test)] 1861 mod tests { 1862 use super::*; 1863 use test_case::test_case; 1864 1865 #[test_case(10, 20, 50, 100, 10, 140 ; "case 1")] 1866 #[test_case(100, 20, 50, 100, 50, 100 ; "case 2")] merge_extents( offset1: u64, size1: usize, offset2: u64, size2: usize, expected_offset: u64, expected_size: usize, )1867 fn merge_extents( 1868 offset1: u64, 1869 size1: usize, 1870 offset2: u64, 1871 size2: usize, 1872 expected_offset: u64, 1873 expected_size: usize, 1874 ) { 1875 let mut e1 = Extent { 1876 offset: offset1, 1877 size: size1, 1878 }; 1879 let e2 = Extent { 1880 offset: offset2, 1881 size: size2, 1882 }; 1883 assert!(e1.merge(&e2).is_ok()); 1884 assert_eq!(e1.offset, expected_offset); 1885 assert_eq!(e1.size, expected_size); 1886 } 1887 } 1888