1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/base/video_frame.h"
6
7 #include <algorithm>
8
9 #include "base/bind.h"
10 #include "base/callback_helpers.h"
11 #include "base/logging.h"
12 #include "base/memory/aligned_memory.h"
13 #include "base/strings/string_piece.h"
14 #include "media/base/limits.h"
15 #include "media/base/video_util.h"
16 #include "third_party/skia/include/core/SkBitmap.h"
17
18 namespace media {
19
20 // static
CreateFrame(VideoFrame::Format format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp)21 scoped_refptr<VideoFrame> VideoFrame::CreateFrame(
22 VideoFrame::Format format,
23 const gfx::Size& coded_size,
24 const gfx::Rect& visible_rect,
25 const gfx::Size& natural_size,
26 base::TimeDelta timestamp) {
27 DCHECK(IsValidConfig(format, coded_size, visible_rect, natural_size));
28 scoped_refptr<VideoFrame> frame(new VideoFrame(
29 format, coded_size, visible_rect, natural_size, timestamp, false));
30 switch (format) {
31 case VideoFrame::YV12:
32 case VideoFrame::YV12A:
33 case VideoFrame::YV16:
34 case VideoFrame::I420:
35 case VideoFrame::YV12J:
36 frame->AllocateYUV();
37 break;
38 default:
39 LOG(FATAL) << "Unsupported frame format: " << format;
40 }
41 return frame;
42 }
43
44 // static
FormatToString(VideoFrame::Format format)45 std::string VideoFrame::FormatToString(VideoFrame::Format format) {
46 switch (format) {
47 case VideoFrame::UNKNOWN:
48 return "UNKNOWN";
49 case VideoFrame::YV12:
50 return "YV12";
51 case VideoFrame::YV16:
52 return "YV16";
53 case VideoFrame::I420:
54 return "I420";
55 case VideoFrame::NATIVE_TEXTURE:
56 return "NATIVE_TEXTURE";
57 #if defined(VIDEO_HOLE)
58 case VideoFrame::HOLE:
59 return "HOLE";
60 #endif // defined(VIDEO_HOLE)
61 case VideoFrame::YV12A:
62 return "YV12A";
63 case VideoFrame::YV12J:
64 return "YV12J";
65 case VideoFrame::HISTOGRAM_MAX:
66 return "HISTOGRAM_MAX";
67 }
68 NOTREACHED() << "Invalid videoframe format provided: " << format;
69 return "";
70 }
71
72 // static
IsValidConfig(VideoFrame::Format format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size)73 bool VideoFrame::IsValidConfig(VideoFrame::Format format,
74 const gfx::Size& coded_size,
75 const gfx::Rect& visible_rect,
76 const gfx::Size& natural_size) {
77 return (format != VideoFrame::UNKNOWN &&
78 !coded_size.IsEmpty() &&
79 coded_size.GetArea() <= limits::kMaxCanvas &&
80 coded_size.width() <= limits::kMaxDimension &&
81 coded_size.height() <= limits::kMaxDimension &&
82 !visible_rect.IsEmpty() &&
83 visible_rect.x() >= 0 && visible_rect.y() >= 0 &&
84 visible_rect.right() <= coded_size.width() &&
85 visible_rect.bottom() <= coded_size.height() &&
86 !natural_size.IsEmpty() &&
87 natural_size.GetArea() <= limits::kMaxCanvas &&
88 natural_size.width() <= limits::kMaxDimension &&
89 natural_size.height() <= limits::kMaxDimension);
90 }
91
92 // static
WrapNativeTexture(scoped_ptr<MailboxHolder> mailbox_holder,uint32 texture_target,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp,const ReadPixelsCB & read_pixels_cb,const base::Closure & no_longer_needed_cb)93 scoped_refptr<VideoFrame> VideoFrame::WrapNativeTexture(
94 scoped_ptr<MailboxHolder> mailbox_holder,
95 uint32 texture_target,
96 const gfx::Size& coded_size,
97 const gfx::Rect& visible_rect,
98 const gfx::Size& natural_size,
99 base::TimeDelta timestamp,
100 const ReadPixelsCB& read_pixels_cb,
101 const base::Closure& no_longer_needed_cb) {
102 scoped_refptr<VideoFrame> frame(new VideoFrame(NATIVE_TEXTURE,
103 coded_size,
104 visible_rect,
105 natural_size,
106 timestamp,
107 false));
108 frame->texture_mailbox_holder_ = mailbox_holder.Pass();
109 frame->texture_target_ = texture_target;
110 frame->read_pixels_cb_ = read_pixels_cb;
111 frame->no_longer_needed_cb_ = no_longer_needed_cb;
112
113 return frame;
114 }
115
ReadPixelsFromNativeTexture(const SkBitmap & pixels)116 void VideoFrame::ReadPixelsFromNativeTexture(const SkBitmap& pixels) {
117 DCHECK_EQ(format_, NATIVE_TEXTURE);
118 if (!read_pixels_cb_.is_null())
119 read_pixels_cb_.Run(pixels);
120 }
121
122 // static
WrapExternalPackedMemory(Format format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,uint8 * data,size_t data_size,base::SharedMemoryHandle handle,base::TimeDelta timestamp,const base::Closure & no_longer_needed_cb)123 scoped_refptr<VideoFrame> VideoFrame::WrapExternalPackedMemory(
124 Format format,
125 const gfx::Size& coded_size,
126 const gfx::Rect& visible_rect,
127 const gfx::Size& natural_size,
128 uint8* data,
129 size_t data_size,
130 base::SharedMemoryHandle handle,
131 base::TimeDelta timestamp,
132 const base::Closure& no_longer_needed_cb) {
133 if (data_size < AllocationSize(format, coded_size))
134 return NULL;
135
136 switch (format) {
137 case I420: {
138 scoped_refptr<VideoFrame> frame(new VideoFrame(
139 format, coded_size, visible_rect, natural_size, timestamp, false));
140 frame->shared_memory_handle_ = handle;
141 frame->strides_[kYPlane] = coded_size.width();
142 frame->strides_[kUPlane] = coded_size.width() / 2;
143 frame->strides_[kVPlane] = coded_size.width() / 2;
144 frame->data_[kYPlane] = data;
145 frame->data_[kUPlane] = data + coded_size.GetArea();
146 frame->data_[kVPlane] = data + (coded_size.GetArea() * 5 / 4);
147 frame->no_longer_needed_cb_ = no_longer_needed_cb;
148 return frame;
149 }
150 default:
151 NOTIMPLEMENTED();
152 return NULL;
153 }
154 }
155
156 // static
WrapExternalYuvData(Format format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,int32 y_stride,int32 u_stride,int32 v_stride,uint8 * y_data,uint8 * u_data,uint8 * v_data,base::TimeDelta timestamp,const base::Closure & no_longer_needed_cb)157 scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
158 Format format,
159 const gfx::Size& coded_size,
160 const gfx::Rect& visible_rect,
161 const gfx::Size& natural_size,
162 int32 y_stride,
163 int32 u_stride,
164 int32 v_stride,
165 uint8* y_data,
166 uint8* u_data,
167 uint8* v_data,
168 base::TimeDelta timestamp,
169 const base::Closure& no_longer_needed_cb) {
170 DCHECK(format == YV12 || format == YV16 || format == I420) << format;
171 scoped_refptr<VideoFrame> frame(new VideoFrame(
172 format, coded_size, visible_rect, natural_size, timestamp, false));
173 frame->strides_[kYPlane] = y_stride;
174 frame->strides_[kUPlane] = u_stride;
175 frame->strides_[kVPlane] = v_stride;
176 frame->data_[kYPlane] = y_data;
177 frame->data_[kUPlane] = u_data;
178 frame->data_[kVPlane] = v_data;
179 frame->no_longer_needed_cb_ = no_longer_needed_cb;
180 return frame;
181 }
182
183 // static
WrapVideoFrame(const scoped_refptr<VideoFrame> & frame,const base::Closure & no_longer_needed_cb)184 scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame(
185 const scoped_refptr<VideoFrame>& frame,
186 const base::Closure& no_longer_needed_cb) {
187 scoped_refptr<VideoFrame> wrapped_frame(new VideoFrame(
188 frame->format(), frame->coded_size(), frame->visible_rect(),
189 frame->natural_size(), frame->GetTimestamp(), frame->end_of_stream()));
190
191 for (size_t i = 0; i < NumPlanes(frame->format()); ++i) {
192 wrapped_frame->strides_[i] = frame->stride(i);
193 wrapped_frame->data_[i] = frame->data(i);
194 }
195
196 wrapped_frame->no_longer_needed_cb_ = no_longer_needed_cb;
197 return wrapped_frame;
198 }
199
200 // static
CreateEOSFrame()201 scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() {
202 return new VideoFrame(VideoFrame::UNKNOWN,
203 gfx::Size(),
204 gfx::Rect(),
205 gfx::Size(),
206 kNoTimestamp(),
207 true);
208 }
209
210 // static
CreateColorFrame(const gfx::Size & size,uint8 y,uint8 u,uint8 v,base::TimeDelta timestamp)211 scoped_refptr<VideoFrame> VideoFrame::CreateColorFrame(
212 const gfx::Size& size,
213 uint8 y, uint8 u, uint8 v,
214 base::TimeDelta timestamp) {
215 DCHECK(IsValidConfig(VideoFrame::YV12, size, gfx::Rect(size), size));
216 scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
217 VideoFrame::YV12, size, gfx::Rect(size), size, timestamp);
218 FillYUV(frame.get(), y, u, v);
219 return frame;
220 }
221
222 // static
CreateBlackFrame(const gfx::Size & size)223 scoped_refptr<VideoFrame> VideoFrame::CreateBlackFrame(const gfx::Size& size) {
224 const uint8 kBlackY = 0x00;
225 const uint8 kBlackUV = 0x80;
226 const base::TimeDelta kZero;
227 return CreateColorFrame(size, kBlackY, kBlackUV, kBlackUV, kZero);
228 }
229
230 #if defined(VIDEO_HOLE)
231 // This block and other blocks wrapped around #if defined(VIDEO_HOLE) is not
232 // maintained by the general compositor team. Please contact the following
233 // people instead:
234 //
235 // wonsik@chromium.org
236 // ycheo@chromium.org
237
238 // static
CreateHoleFrame(const gfx::Size & size)239 scoped_refptr<VideoFrame> VideoFrame::CreateHoleFrame(
240 const gfx::Size& size) {
241 DCHECK(IsValidConfig(VideoFrame::HOLE, size, gfx::Rect(size), size));
242 scoped_refptr<VideoFrame> frame(new VideoFrame(
243 VideoFrame::HOLE, size, gfx::Rect(size), size, base::TimeDelta(), false));
244 return frame;
245 }
246 #endif // defined(VIDEO_HOLE)
247
248 // static
NumPlanes(Format format)249 size_t VideoFrame::NumPlanes(Format format) {
250 switch (format) {
251 case VideoFrame::NATIVE_TEXTURE:
252 #if defined(VIDEO_HOLE)
253 case VideoFrame::HOLE:
254 #endif // defined(VIDEO_HOLE)
255 return 0;
256 case VideoFrame::YV12:
257 case VideoFrame::YV16:
258 case VideoFrame::I420:
259 case VideoFrame::YV12J:
260 return 3;
261 case VideoFrame::YV12A:
262 return 4;
263 case VideoFrame::UNKNOWN:
264 case VideoFrame::HISTOGRAM_MAX:
265 break;
266 }
267 NOTREACHED() << "Unsupported video frame format: " << format;
268 return 0;
269 }
270
RoundUp(size_t value,size_t alignment)271 static inline size_t RoundUp(size_t value, size_t alignment) {
272 // Check that |alignment| is a power of 2.
273 DCHECK((alignment + (alignment - 1)) == (alignment | (alignment - 1)));
274 return ((value + (alignment - 1)) & ~(alignment-1));
275 }
276
277 // static
AllocationSize(Format format,const gfx::Size & coded_size)278 size_t VideoFrame::AllocationSize(Format format, const gfx::Size& coded_size) {
279 size_t total = 0;
280 for (size_t i = 0; i < NumPlanes(format); ++i)
281 total += PlaneAllocationSize(format, i, coded_size);
282 return total;
283 }
284
285 // static
PlaneAllocationSize(Format format,size_t plane,const gfx::Size & coded_size)286 size_t VideoFrame::PlaneAllocationSize(Format format,
287 size_t plane,
288 const gfx::Size& coded_size) {
289 const size_t area =
290 RoundUp(coded_size.width(), 2) * RoundUp(coded_size.height(), 2);
291 switch (format) {
292 case VideoFrame::YV12:
293 case VideoFrame::YV12J:
294 case VideoFrame::I420: {
295 switch (plane) {
296 case VideoFrame::kYPlane:
297 return area;
298 case VideoFrame::kUPlane:
299 case VideoFrame::kVPlane:
300 return area / 4;
301 default:
302 break;
303 }
304 }
305 case VideoFrame::YV12A: {
306 switch (plane) {
307 case VideoFrame::kYPlane:
308 case VideoFrame::kAPlane:
309 return area;
310 case VideoFrame::kUPlane:
311 case VideoFrame::kVPlane:
312 return area / 4;
313 default:
314 break;
315 }
316 }
317 case VideoFrame::YV16: {
318 switch (plane) {
319 case VideoFrame::kYPlane:
320 return area;
321 case VideoFrame::kUPlane:
322 case VideoFrame::kVPlane:
323 return area / 2;
324 default:
325 break;
326 }
327 }
328 case VideoFrame::UNKNOWN:
329 case VideoFrame::NATIVE_TEXTURE:
330 case VideoFrame::HISTOGRAM_MAX:
331 #if defined(VIDEO_HOLE)
332 case VideoFrame::HOLE:
333 #endif // defined(VIDEO_HOLE)
334 break;
335 }
336 NOTREACHED() << "Unsupported video frame format/plane: "
337 << format << "/" << plane;
338 return 0;
339 }
340
341 // Release data allocated by AllocateYUV().
ReleaseData(uint8 * data)342 static void ReleaseData(uint8* data) {
343 DCHECK(data);
344 base::AlignedFree(data);
345 }
346
AllocateYUV()347 void VideoFrame::AllocateYUV() {
348 DCHECK(format_ == VideoFrame::YV12 || format_ == VideoFrame::YV16 ||
349 format_ == VideoFrame::YV12A || format_ == VideoFrame::I420 ||
350 format_ == VideoFrame::YV12J);
351 // Align Y rows at least at 16 byte boundaries. The stride for both
352 // YV12 and YV16 is 1/2 of the stride of Y. For YV12, every row of bytes for
353 // U and V applies to two rows of Y (one byte of UV for 4 bytes of Y), so in
354 // the case of YV12 the strides are identical for the same width surface, but
355 // the number of bytes allocated for YV12 is 1/2 the amount for U & V as
356 // YV16. We also round the height of the surface allocated to be an even
357 // number to avoid any potential of faulting by code that attempts to access
358 // the Y values of the final row, but assumes that the last row of U & V
359 // applies to a full two rows of Y. YV12A is the same as YV12, but with an
360 // additional alpha plane that has the same size and alignment as the Y plane.
361
362 size_t y_stride = RoundUp(row_bytes(VideoFrame::kYPlane),
363 kFrameSizeAlignment);
364 size_t uv_stride = RoundUp(row_bytes(VideoFrame::kUPlane),
365 kFrameSizeAlignment);
366 // The *2 here is because some formats (e.g. h264) allow interlaced coding,
367 // and then the size needs to be a multiple of two macroblocks (vertically).
368 // See libavcodec/utils.c:avcodec_align_dimensions2().
369 size_t y_height = RoundUp(coded_size_.height(), kFrameSizeAlignment * 2);
370 size_t uv_height =
371 (format_ == VideoFrame::YV12 || format_ == VideoFrame::YV12A ||
372 format_ == VideoFrame::I420)
373 ? y_height / 2
374 : y_height;
375 size_t y_bytes = y_height * y_stride;
376 size_t uv_bytes = uv_height * uv_stride;
377 size_t a_bytes = format_ == VideoFrame::YV12A ? y_bytes : 0;
378
379 // The extra line of UV being allocated is because h264 chroma MC
380 // overreads by one line in some cases, see libavcodec/utils.c:
381 // avcodec_align_dimensions2() and libavcodec/x86/h264_chromamc.asm:
382 // put_h264_chroma_mc4_ssse3().
383 uint8* data = reinterpret_cast<uint8*>(
384 base::AlignedAlloc(
385 y_bytes + (uv_bytes * 2 + uv_stride) + a_bytes + kFrameSizePadding,
386 kFrameAddressAlignment));
387 no_longer_needed_cb_ = base::Bind(&ReleaseData, data);
388 COMPILE_ASSERT(0 == VideoFrame::kYPlane, y_plane_data_must_be_index_0);
389 data_[VideoFrame::kYPlane] = data;
390 data_[VideoFrame::kUPlane] = data + y_bytes;
391 data_[VideoFrame::kVPlane] = data + y_bytes + uv_bytes;
392 strides_[VideoFrame::kYPlane] = y_stride;
393 strides_[VideoFrame::kUPlane] = uv_stride;
394 strides_[VideoFrame::kVPlane] = uv_stride;
395 if (format_ == YV12A) {
396 data_[VideoFrame::kAPlane] = data + y_bytes + (2 * uv_bytes);
397 strides_[VideoFrame::kAPlane] = y_stride;
398 }
399 }
400
VideoFrame(VideoFrame::Format format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp,bool end_of_stream)401 VideoFrame::VideoFrame(VideoFrame::Format format,
402 const gfx::Size& coded_size,
403 const gfx::Rect& visible_rect,
404 const gfx::Size& natural_size,
405 base::TimeDelta timestamp,
406 bool end_of_stream)
407 : format_(format),
408 coded_size_(coded_size),
409 visible_rect_(visible_rect),
410 natural_size_(natural_size),
411 texture_target_(0),
412 shared_memory_handle_(base::SharedMemory::NULLHandle()),
413 timestamp_(timestamp),
414 end_of_stream_(end_of_stream) {
415 memset(&strides_, 0, sizeof(strides_));
416 memset(&data_, 0, sizeof(data_));
417 }
418
~VideoFrame()419 VideoFrame::~VideoFrame() {
420 if (!no_longer_needed_cb_.is_null())
421 base::ResetAndReturn(&no_longer_needed_cb_).Run();
422 }
423
IsValidPlane(size_t plane) const424 bool VideoFrame::IsValidPlane(size_t plane) const {
425 return (plane < NumPlanes(format_));
426 }
427
stride(size_t plane) const428 int VideoFrame::stride(size_t plane) const {
429 DCHECK(IsValidPlane(plane));
430 return strides_[plane];
431 }
432
row_bytes(size_t plane) const433 int VideoFrame::row_bytes(size_t plane) const {
434 DCHECK(IsValidPlane(plane));
435 int width = coded_size_.width();
436 switch (format_) {
437 // Planar, 8bpp.
438 case YV12A:
439 if (plane == kAPlane)
440 return width;
441 // Fallthrough.
442 case YV12:
443 case YV16:
444 case I420:
445 case YV12J:
446 if (plane == kYPlane)
447 return width;
448 return RoundUp(width, 2) / 2;
449
450 default:
451 break;
452 }
453
454 // Intentionally leave out non-production formats.
455 NOTREACHED() << "Unsupported video frame format: " << format_;
456 return 0;
457 }
458
rows(size_t plane) const459 int VideoFrame::rows(size_t plane) const {
460 DCHECK(IsValidPlane(plane));
461 int height = coded_size_.height();
462 switch (format_) {
463 case YV16:
464 return height;
465
466 case YV12A:
467 if (plane == kAPlane)
468 return height;
469 // Fallthrough.
470 case YV12:
471 case I420:
472 if (plane == kYPlane)
473 return height;
474 return RoundUp(height, 2) / 2;
475
476 default:
477 break;
478 }
479
480 // Intentionally leave out non-production formats.
481 NOTREACHED() << "Unsupported video frame format: " << format_;
482 return 0;
483 }
484
data(size_t plane) const485 uint8* VideoFrame::data(size_t plane) const {
486 DCHECK(IsValidPlane(plane));
487 return data_[plane];
488 }
489
texture_mailbox() const490 VideoFrame::MailboxHolder* VideoFrame::texture_mailbox() const {
491 DCHECK_EQ(format_, NATIVE_TEXTURE);
492 return texture_mailbox_holder_.get();
493 }
494
texture_target() const495 uint32 VideoFrame::texture_target() const {
496 DCHECK_EQ(format_, NATIVE_TEXTURE);
497 return texture_target_;
498 }
499
shared_memory_handle() const500 base::SharedMemoryHandle VideoFrame::shared_memory_handle() const {
501 return shared_memory_handle_;
502 }
503
HashFrameForTesting(base::MD5Context * context)504 void VideoFrame::HashFrameForTesting(base::MD5Context* context) {
505 for (int plane = 0; plane < kMaxPlanes; ++plane) {
506 if (!IsValidPlane(plane))
507 break;
508 for (int row = 0; row < rows(plane); ++row) {
509 base::MD5Update(context, base::StringPiece(
510 reinterpret_cast<char*>(data(plane) + stride(plane) * row),
511 row_bytes(plane)));
512 }
513 }
514 }
515
MailboxHolder(const gpu::Mailbox & mailbox,unsigned sync_point,const TextureNoLongerNeededCallback & release_callback)516 VideoFrame::MailboxHolder::MailboxHolder(
517 const gpu::Mailbox& mailbox,
518 unsigned sync_point,
519 const TextureNoLongerNeededCallback& release_callback)
520 : mailbox_(mailbox),
521 sync_point_(sync_point),
522 release_callback_(release_callback) {}
523
~MailboxHolder()524 VideoFrame::MailboxHolder::~MailboxHolder() {
525 if (!release_callback_.is_null())
526 release_callback_.Run(sync_point_);
527 }
528
529 } // namespace media
530