1 // Copyright 2024 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 use crate::decoder::tile::TileInfo; 16 use crate::decoder::ProgressiveState; 17 use crate::internal_utils::pixels::*; 18 use crate::internal_utils::*; 19 use crate::parser::mp4box::CodecConfiguration; 20 use crate::reformat::coeffs::*; 21 use crate::utils::clap::CleanAperture; 22 use crate::*; 23 24 #[derive(Clone, Copy, Debug, PartialEq)] 25 pub enum Plane { 26 Y = 0, 27 U = 1, 28 V = 2, 29 A = 3, 30 } 31 32 impl From<usize> for Plane { from(plane: usize) -> Self33 fn from(plane: usize) -> Self { 34 match plane { 35 1 => Plane::U, 36 2 => Plane::V, 37 3 => Plane::A, 38 _ => Plane::Y, 39 } 40 } 41 } 42 43 impl Plane { as_usize(&self) -> usize44 pub(crate) fn as_usize(&self) -> usize { 45 match self { 46 Plane::Y => 0, 47 Plane::U => 1, 48 Plane::V => 2, 49 Plane::A => 3, 50 } 51 } 52 } 53 54 /// cbindgen:ignore 55 pub const MAX_PLANE_COUNT: usize = 4; 56 pub const YUV_PLANES: [Plane; 3] = [Plane::Y, Plane::U, Plane::V]; 57 pub const A_PLANE: [Plane; 1] = [Plane::A]; 58 pub const ALL_PLANES: [Plane; MAX_PLANE_COUNT] = [Plane::Y, Plane::U, Plane::V, Plane::A]; 59 60 #[repr(C)] 61 #[derive(Clone, Copy, Debug, Default, PartialEq)] 62 // VideoFullRangeFlag as specified in ISO/IEC 23091-2/ITU-T H.273. 63 pub enum YuvRange { 64 Limited = 0, 65 #[default] 66 Full = 1, 67 } 68 69 #[derive(Default)] 70 pub struct Image { 71 pub width: u32, 72 pub height: u32, 73 pub depth: u8, 74 75 pub yuv_format: PixelFormat, 76 pub yuv_range: YuvRange, 77 pub chroma_sample_position: ChromaSamplePosition, 78 79 pub alpha_present: bool, 80 pub alpha_premultiplied: bool, 81 82 pub row_bytes: [u32; MAX_PLANE_COUNT], 83 pub image_owns_planes: [bool; MAX_PLANE_COUNT], 84 85 pub planes: [Option<Pixels>; MAX_PLANE_COUNT], 86 87 pub color_primaries: ColorPrimaries, 88 pub transfer_characteristics: TransferCharacteristics, 89 pub matrix_coefficients: MatrixCoefficients, 90 91 pub clli: Option<ContentLightLevelInformation>, 92 pub pasp: Option<PixelAspectRatio>, 93 pub clap: Option<CleanAperture>, 94 pub irot_angle: Option<u8>, 95 pub imir_axis: Option<u8>, 96 97 pub exif: Vec<u8>, 98 pub icc: Vec<u8>, 99 pub xmp: Vec<u8>, 100 101 pub image_sequence_track_present: bool, 102 pub progressive_state: ProgressiveState, 103 } 104 105 pub struct PlaneData { 106 pub width: u32, 107 pub height: u32, 108 pub row_bytes: u32, 109 pub pixel_size: u32, 110 } 111 112 #[derive(Clone, Copy)] 113 pub enum PlaneRow<'a> { 114 Depth8(&'a [u8]), 115 Depth16(&'a [u16]), 116 } 117 118 impl Image { shallow_clone(&self) -> Self119 pub(crate) fn shallow_clone(&self) -> Self { 120 Self { 121 width: self.width, 122 height: self.height, 123 depth: self.depth, 124 yuv_format: self.yuv_format, 125 yuv_range: self.yuv_range, 126 chroma_sample_position: self.chroma_sample_position, 127 alpha_present: self.alpha_present, 128 alpha_premultiplied: self.alpha_premultiplied, 129 color_primaries: self.color_primaries, 130 transfer_characteristics: self.transfer_characteristics, 131 matrix_coefficients: self.matrix_coefficients, 132 clli: self.clli, 133 pasp: self.pasp, 134 clap: self.clap, 135 irot_angle: self.irot_angle, 136 imir_axis: self.imir_axis, 137 exif: self.exif.clone(), 138 icc: self.icc.clone(), 139 xmp: self.xmp.clone(), 140 image_sequence_track_present: self.image_sequence_track_present, 141 progressive_state: self.progressive_state, 142 ..Default::default() 143 } 144 } 145 depth_valid(&self) -> bool146 pub(crate) fn depth_valid(&self) -> bool { 147 matches!(self.depth, 8 | 10 | 12 | 16) 148 } 149 max_channel(&self) -> u16150 pub fn max_channel(&self) -> u16 { 151 if !self.depth_valid() { 152 0 153 } else { 154 ((1i32 << self.depth) - 1) as u16 155 } 156 } 157 max_channel_f(&self) -> f32158 pub(crate) fn max_channel_f(&self) -> f32 { 159 self.max_channel() as f32 160 } 161 has_plane(&self, plane: Plane) -> bool162 pub fn has_plane(&self, plane: Plane) -> bool { 163 let plane_index = plane.as_usize(); 164 if self.planes[plane_index].is_none() || self.row_bytes[plane_index] == 0 { 165 return false; 166 } 167 self.planes[plane_index].unwrap_ref().has_data() 168 } 169 has_alpha(&self) -> bool170 pub fn has_alpha(&self) -> bool { 171 self.has_plane(Plane::A) 172 } 173 has_same_properties(&self, other: &Image) -> bool174 pub(crate) fn has_same_properties(&self, other: &Image) -> bool { 175 self.width == other.width && self.height == other.height && self.depth == other.depth 176 } 177 has_same_cicp(&self, other: &Image) -> bool178 fn has_same_cicp(&self, other: &Image) -> bool { 179 self.depth == other.depth 180 && self.yuv_format == other.yuv_format 181 && self.yuv_range == other.yuv_range 182 && self.chroma_sample_position == other.chroma_sample_position 183 && self.color_primaries == other.color_primaries 184 && self.transfer_characteristics == other.transfer_characteristics 185 && self.matrix_coefficients == other.matrix_coefficients 186 } 187 has_same_properties_and_cicp(&self, other: &Image) -> bool188 pub(crate) fn has_same_properties_and_cicp(&self, other: &Image) -> bool { 189 self.has_same_properties(other) && self.has_same_cicp(other) 190 } 191 width(&self, plane: Plane) -> usize192 pub fn width(&self, plane: Plane) -> usize { 193 match plane { 194 Plane::Y | Plane::A => self.width as usize, 195 Plane::U => match self.yuv_format { 196 PixelFormat::Yuv444 197 | PixelFormat::AndroidP010 198 | PixelFormat::AndroidNv12 199 | PixelFormat::AndroidNv21 => self.width as usize, 200 PixelFormat::Yuv420 | PixelFormat::Yuv422 => (self.width as usize + 1) / 2, 201 PixelFormat::None | PixelFormat::Yuv400 => 0, 202 }, 203 Plane::V => match self.yuv_format { 204 PixelFormat::Yuv444 => self.width as usize, 205 PixelFormat::Yuv420 | PixelFormat::Yuv422 => (self.width as usize + 1) / 2, 206 PixelFormat::None 207 | PixelFormat::Yuv400 208 | PixelFormat::AndroidP010 209 | PixelFormat::AndroidNv12 210 | PixelFormat::AndroidNv21 => 0, 211 }, 212 } 213 } 214 height(&self, plane: Plane) -> usize215 pub fn height(&self, plane: Plane) -> usize { 216 match plane { 217 Plane::Y | Plane::A => self.height as usize, 218 Plane::U => match self.yuv_format { 219 PixelFormat::Yuv444 | PixelFormat::Yuv422 => self.height as usize, 220 PixelFormat::Yuv420 221 | PixelFormat::AndroidP010 222 | PixelFormat::AndroidNv12 223 | PixelFormat::AndroidNv21 => (self.height as usize + 1) / 2, 224 PixelFormat::None | PixelFormat::Yuv400 => 0, 225 }, 226 Plane::V => match self.yuv_format { 227 PixelFormat::Yuv444 | PixelFormat::Yuv422 => self.height as usize, 228 PixelFormat::Yuv420 => (self.height as usize + 1) / 2, 229 PixelFormat::None 230 | PixelFormat::Yuv400 231 | PixelFormat::AndroidP010 232 | PixelFormat::AndroidNv12 233 | PixelFormat::AndroidNv21 => 0, 234 }, 235 } 236 } 237 plane_data(&self, plane: Plane) -> Option<PlaneData>238 pub fn plane_data(&self, plane: Plane) -> Option<PlaneData> { 239 if !self.has_plane(plane) { 240 return None; 241 } 242 Some(PlaneData { 243 width: self.width(plane) as u32, 244 height: self.height(plane) as u32, 245 row_bytes: self.row_bytes[plane.as_usize()], 246 pixel_size: if self.depth == 8 { 1 } else { 2 }, 247 }) 248 } 249 row(&self, plane: Plane, row: u32) -> AvifResult<&[u8]>250 pub fn row(&self, plane: Plane, row: u32) -> AvifResult<&[u8]> { 251 let plane_data = self.plane_data(plane).ok_or(AvifError::NoContent)?; 252 let start = checked_mul!(row, plane_data.row_bytes)?; 253 self.planes[plane.as_usize()] 254 .unwrap_ref() 255 .slice(start, plane_data.row_bytes) 256 } 257 row_mut(&mut self, plane: Plane, row: u32) -> AvifResult<&mut [u8]>258 pub fn row_mut(&mut self, plane: Plane, row: u32) -> AvifResult<&mut [u8]> { 259 let plane_data = self.plane_data(plane).ok_or(AvifError::NoContent)?; 260 let row_bytes = plane_data.row_bytes; 261 let start = checked_mul!(row, row_bytes)?; 262 self.planes[plane.as_usize()] 263 .unwrap_mut() 264 .slice_mut(start, row_bytes) 265 } 266 row16(&self, plane: Plane, row: u32) -> AvifResult<&[u16]>267 pub fn row16(&self, plane: Plane, row: u32) -> AvifResult<&[u16]> { 268 let plane_data = self.plane_data(plane).ok_or(AvifError::NoContent)?; 269 let row_bytes = plane_data.row_bytes / 2; 270 let start = checked_mul!(row, row_bytes)?; 271 self.planes[plane.as_usize()] 272 .unwrap_ref() 273 .slice16(start, row_bytes) 274 } 275 row16_mut(&mut self, plane: Plane, row: u32) -> AvifResult<&mut [u16]>276 pub fn row16_mut(&mut self, plane: Plane, row: u32) -> AvifResult<&mut [u16]> { 277 let plane_data = self.plane_data(plane).ok_or(AvifError::NoContent)?; 278 let row_bytes = plane_data.row_bytes / 2; 279 let start = checked_mul!(row, row_bytes)?; 280 self.planes[plane.as_usize()] 281 .unwrap_mut() 282 .slice16_mut(start, row_bytes) 283 } 284 row_generic(&self, plane: Plane, row: u32) -> AvifResult<PlaneRow>285 pub(crate) fn row_generic(&self, plane: Plane, row: u32) -> AvifResult<PlaneRow> { 286 Ok(if self.depth == 8 { 287 PlaneRow::Depth8(self.row(plane, row)?) 288 } else { 289 PlaneRow::Depth16(self.row16(plane, row)?) 290 }) 291 } 292 293 #[cfg(any(feature = "dav1d", feature = "libgav1"))] clear_chroma_planes(&mut self)294 pub(crate) fn clear_chroma_planes(&mut self) { 295 for plane in [Plane::U, Plane::V] { 296 let plane = plane.as_usize(); 297 self.planes[plane] = None; 298 self.row_bytes[plane] = 0; 299 self.image_owns_planes[plane] = false; 300 } 301 } 302 allocate_planes_with_default_values( &mut self, category: Category, default_values: [u16; 4], ) -> AvifResult<()>303 pub(crate) fn allocate_planes_with_default_values( 304 &mut self, 305 category: Category, 306 default_values: [u16; 4], 307 ) -> AvifResult<()> { 308 let pixel_size: usize = if self.depth == 8 { 1 } else { 2 }; 309 for plane in category.planes() { 310 let plane = *plane; 311 let plane_index = plane.as_usize(); 312 let width = round2_usize(self.width(plane)); 313 let plane_size = checked_mul!(width, round2_usize(self.height(plane)))?; 314 if self.planes[plane_index].is_some() 315 && self.planes[plane_index].unwrap_ref().size() == plane_size 316 && (self.planes[plane_index].unwrap_ref().pixel_bit_size() == 0 317 || self.planes[plane_index].unwrap_ref().pixel_bit_size() == pixel_size * 8) 318 { 319 continue; 320 } 321 self.planes[plane_index] = Some(if self.depth == 8 { 322 Pixels::Buffer(Vec::new()) 323 } else { 324 Pixels::Buffer16(Vec::new()) 325 }); 326 let pixels = self.planes[plane_index].unwrap_mut(); 327 pixels.resize(plane_size, default_values[plane_index])?; 328 self.row_bytes[plane_index] = u32_from_usize(checked_mul!(width, pixel_size)?)?; 329 self.image_owns_planes[plane_index] = true; 330 } 331 Ok(()) 332 } 333 allocate_planes(&mut self, category: Category) -> AvifResult<()>334 pub(crate) fn allocate_planes(&mut self, category: Category) -> AvifResult<()> { 335 self.allocate_planes_with_default_values(category, [0, 0, 0, self.max_channel()]) 336 } 337 copy_properties_from( &mut self, image: &Image, codec_config: &CodecConfiguration, )338 pub(crate) fn copy_properties_from( 339 &mut self, 340 image: &Image, 341 codec_config: &CodecConfiguration, 342 ) { 343 self.yuv_format = image.yuv_format; 344 self.depth = image.depth; 345 if cfg!(feature = "heic") && codec_config.is_heic() { 346 // For AVIF, the information in the `colr` box takes precedence over what is reported 347 // by the decoder. For HEIC, we always honor what is reported by the decoder. 348 self.yuv_range = image.yuv_range; 349 self.color_primaries = image.color_primaries; 350 self.transfer_characteristics = image.transfer_characteristics; 351 self.matrix_coefficients = image.matrix_coefficients; 352 } 353 } 354 355 // If src contains pointers, this function will simply make a copy of the pointer without 356 // copying the actual pixels (stealing). If src contains buffer, this function will clone the 357 // buffers (copying). steal_or_copy_planes_from( &mut self, src: &Image, category: Category, ) -> AvifResult<()>358 pub(crate) fn steal_or_copy_planes_from( 359 &mut self, 360 src: &Image, 361 category: Category, 362 ) -> AvifResult<()> { 363 for plane in category.planes() { 364 let plane = plane.as_usize(); 365 (self.planes[plane], self.row_bytes[plane]) = match &src.planes[plane] { 366 Some(src_plane) => (Some(src_plane.try_clone()?), src.row_bytes[plane]), 367 None => (None, 0), 368 } 369 } 370 Ok(()) 371 } 372 copy_from_tile( &mut self, tile: &Image, grid: &Grid, tile_index: u32, category: Category, ) -> AvifResult<()>373 pub(crate) fn copy_from_tile( 374 &mut self, 375 tile: &Image, 376 grid: &Grid, 377 tile_index: u32, 378 category: Category, 379 ) -> AvifResult<()> { 380 // This function is used only when |tile| contains pointers and self contains buffers. 381 let row_index = tile_index / grid.columns; 382 let column_index = tile_index % grid.columns; 383 for plane in category.planes() { 384 let plane = *plane; 385 let src_plane = tile.plane_data(plane); 386 if src_plane.is_none() { 387 continue; 388 } 389 let src_plane = src_plane.unwrap(); 390 // If this is the last tile column, clamp to left over width. 391 let src_width_to_copy = if column_index == grid.columns - 1 { 392 let width_so_far = checked_mul!(src_plane.width, column_index)?; 393 checked_sub!(self.width(plane), usize_from_u32(width_so_far)?)? 394 } else { 395 usize_from_u32(src_plane.width)? 396 }; 397 398 // If this is the last tile row, clamp to left over height. 399 let src_height_to_copy = if row_index == grid.rows - 1 { 400 let height_so_far = checked_mul!(src_plane.height, row_index)?; 401 checked_sub!(u32_from_usize(self.height(plane))?, height_so_far)? 402 } else { 403 src_plane.height 404 }; 405 406 let dst_y_start = checked_mul!(row_index, src_plane.height)?; 407 let dst_x_offset = usize_from_u32(checked_mul!(column_index, src_plane.width)?)?; 408 let dst_x_offset_end = checked_add!(dst_x_offset, src_width_to_copy)?; 409 if self.depth == 8 { 410 for y in 0..src_height_to_copy { 411 let src_row = tile.row(plane, y)?; 412 let src_slice = &src_row[0..src_width_to_copy]; 413 let dst_row = self.row_mut(plane, checked_add!(dst_y_start, y)?)?; 414 let dst_slice = &mut dst_row[dst_x_offset..dst_x_offset_end]; 415 dst_slice.copy_from_slice(src_slice); 416 } 417 } else { 418 for y in 0..src_height_to_copy { 419 let src_row = tile.row16(plane, y)?; 420 let src_slice = &src_row[0..src_width_to_copy]; 421 let dst_row = self.row16_mut(plane, checked_add!(dst_y_start, y)?)?; 422 let dst_slice = &mut dst_row[dst_x_offset..dst_x_offset_end]; 423 dst_slice.copy_from_slice(src_slice); 424 } 425 } 426 } 427 Ok(()) 428 } 429 copy_and_overlay_from_tile( &mut self, tile: &Image, tile_info: &TileInfo, tile_index: u32, category: Category, ) -> AvifResult<()>430 pub(crate) fn copy_and_overlay_from_tile( 431 &mut self, 432 tile: &Image, 433 tile_info: &TileInfo, 434 tile_index: u32, 435 category: Category, 436 ) -> AvifResult<()> { 437 // This function is used only when |tile| contains pointers and self contains buffers. 438 for plane in category.planes() { 439 let plane = *plane; 440 let src_plane = tile.plane_data(plane); 441 let dst_plane = self.plane_data(plane); 442 if src_plane.is_none() || dst_plane.is_none() { 443 continue; 444 } 445 let dst_plane = dst_plane.unwrap(); 446 let tile_index = usize_from_u32(tile_index)?; 447 448 let vertical_offset = tile_info.overlay.vertical_offsets[tile_index] as i128; 449 let horizontal_offset = tile_info.overlay.horizontal_offsets[tile_index] as i128; 450 let src_height = tile.height as i128; 451 let src_width = tile.width as i128; 452 let dst_height = dst_plane.height as i128; 453 let dst_width = dst_plane.width as i128; 454 455 if matches!(plane, Plane::Y | Plane::A) 456 && (vertical_offset + src_height < 0 457 || horizontal_offset + src_width < 0 458 || vertical_offset >= dst_height 459 || horizontal_offset >= dst_width) 460 { 461 // Entire tile outside of the canvas. It is sufficient to perform this check only 462 // for Y and A plane since they are never sub-sampled. 463 return Ok(()); 464 } 465 466 let mut src_y_start: u32; 467 let mut src_height_to_copy: u32; 468 let mut dst_y_start: u32; 469 if vertical_offset >= 0 { 470 src_y_start = 0; 471 src_height_to_copy = src_height as u32; 472 dst_y_start = vertical_offset as u32; 473 } else { 474 src_y_start = vertical_offset.unsigned_abs() as u32; 475 src_height_to_copy = (src_height - vertical_offset.abs()) as u32; 476 dst_y_start = 0; 477 } 478 479 let mut src_x_start: u32; 480 let mut src_width_to_copy: u32; 481 let mut dst_x_start: u32; 482 if horizontal_offset >= 0 { 483 src_x_start = 0; 484 src_width_to_copy = src_width as u32; 485 dst_x_start = horizontal_offset as u32; 486 } else { 487 src_x_start = horizontal_offset.unsigned_abs() as u32; 488 src_width_to_copy = (src_width - horizontal_offset.abs()) as u32; 489 dst_x_start = 0; 490 } 491 492 // Clamp width to the canvas width. 493 if self.width - dst_x_start < src_width_to_copy { 494 src_width_to_copy = self.width - dst_x_start; 495 } 496 497 // Clamp height to the canvas height. 498 if self.height - dst_y_start < src_height_to_copy { 499 src_height_to_copy = self.height - dst_y_start; 500 } 501 502 // Apply chroma subsampling to the offsets. 503 if plane == Plane::U || plane == Plane::V { 504 src_y_start = tile.yuv_format.apply_chroma_shift_y(src_y_start); 505 src_height_to_copy = tile.yuv_format.apply_chroma_shift_y(src_height_to_copy); 506 dst_y_start = tile.yuv_format.apply_chroma_shift_y(dst_y_start); 507 src_x_start = tile.yuv_format.apply_chroma_shift_x(src_x_start); 508 src_width_to_copy = tile.yuv_format.apply_chroma_shift_x(src_width_to_copy); 509 dst_x_start = tile.yuv_format.apply_chroma_shift_x(dst_x_start); 510 } 511 512 let src_y_range = src_y_start..checked_add!(src_y_start, src_height_to_copy)?; 513 let dst_x_range = usize_from_u32(dst_x_start)? 514 ..usize_from_u32(checked_add!(dst_x_start, src_width_to_copy)?)?; 515 let src_x_range = usize_from_u32(src_x_start)? 516 ..checked_add!(usize_from_u32(src_x_start)?, dst_x_range.len())?; 517 let mut dst_y = dst_y_start; 518 if self.depth == 8 { 519 for src_y in src_y_range { 520 let src_row = tile.row(plane, src_y)?; 521 let src_slice = &src_row[src_x_range.clone()]; 522 let dst_row = self.row_mut(plane, dst_y)?; 523 let dst_slice = &mut dst_row[dst_x_range.clone()]; 524 dst_slice.copy_from_slice(src_slice); 525 checked_incr!(dst_y, 1); 526 } 527 } else { 528 for src_y in src_y_range { 529 let src_row = tile.row16(plane, src_y)?; 530 let src_slice = &src_row[src_x_range.clone()]; 531 let dst_row = self.row16_mut(plane, dst_y)?; 532 let dst_slice = &mut dst_row[dst_x_range.clone()]; 533 dst_slice.copy_from_slice(src_slice); 534 checked_incr!(dst_y, 1); 535 } 536 } 537 } 538 Ok(()) 539 } 540 convert_rgba16_to_yuva(&self, rgba: [u16; 4]) -> [u16; 4]541 pub(crate) fn convert_rgba16_to_yuva(&self, rgba: [u16; 4]) -> [u16; 4] { 542 let r = rgba[0] as f32 / 65535.0; 543 let g = rgba[1] as f32 / 65535.0; 544 let b = rgba[2] as f32 / 65535.0; 545 let coeffs = calculate_yuv_coefficients(self.color_primaries, self.matrix_coefficients); 546 let y = coeffs[0] * r + coeffs[1] * g + coeffs[2] * b; 547 let u = (b - y) / (2.0 * (1.0 - coeffs[2])); 548 let v = (r - y) / (2.0 * (1.0 - coeffs[0])); 549 let uv_bias = (1 << (self.depth - 1)) as f32; 550 let max_channel = self.max_channel_f(); 551 [ 552 (y * max_channel).clamp(0.0, max_channel) as u16, 553 (u * max_channel + uv_bias).clamp(0.0, max_channel) as u16, 554 (v * max_channel + uv_bias).clamp(0.0, max_channel) as u16, 555 ((rgba[3] as f32) / 65535.0 * max_channel).round() as u16, 556 ] 557 } 558 } 559