1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "cc/resources/video_resource_updater.h"
6
7 #include "base/bind.h"
8 #include "cc/output/gl_renderer.h"
9 #include "cc/resources/resource_provider.h"
10 #include "gpu/GLES2/gl2extchromium.h"
11 #include "gpu/command_buffer/client/gles2_interface.h"
12 #include "media/base/video_frame.h"
13 #include "media/filters/skcanvas_video_renderer.h"
14 #include "third_party/khronos/GLES2/gl2.h"
15 #include "third_party/khronos/GLES2/gl2ext.h"
16 #include "ui/gfx/size_conversions.h"
17
18 namespace cc {
19
20 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
21 const ResourceFormat kRGBResourceFormat = RGBA_8888;
22
VideoFrameExternalResources()23 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
24
~VideoFrameExternalResources()25 VideoFrameExternalResources::~VideoFrameExternalResources() {}
26
VideoResourceUpdater(ContextProvider * context_provider,ResourceProvider * resource_provider)27 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
28 ResourceProvider* resource_provider)
29 : context_provider_(context_provider),
30 resource_provider_(resource_provider) {
31 }
32
~VideoResourceUpdater()33 VideoResourceUpdater::~VideoResourceUpdater() {
34 while (!all_resources_.empty()) {
35 resource_provider_->DeleteResource(all_resources_.back());
36 all_resources_.pop_back();
37 }
38 }
39
DeleteResource(unsigned resource_id)40 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
41 resource_provider_->DeleteResource(resource_id);
42 all_resources_.erase(std::remove(all_resources_.begin(),
43 all_resources_.end(),
44 resource_id));
45 }
46
47 VideoFrameExternalResources VideoResourceUpdater::
CreateExternalResourcesFromVideoFrame(const scoped_refptr<media::VideoFrame> & video_frame)48 CreateExternalResourcesFromVideoFrame(
49 const scoped_refptr<media::VideoFrame>& video_frame) {
50 if (!VerifyFrame(video_frame))
51 return VideoFrameExternalResources();
52
53 if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
54 return CreateForHardwarePlanes(video_frame);
55 else
56 return CreateForSoftwarePlanes(video_frame);
57 }
58
VerifyFrame(const scoped_refptr<media::VideoFrame> & video_frame)59 bool VideoResourceUpdater::VerifyFrame(
60 const scoped_refptr<media::VideoFrame>& video_frame) {
61 // If these fail, we'll have to add logic that handles offset bitmap/texture
62 // UVs. For now, just expect (0, 0) offset, since all our decoders so far
63 // don't offset.
64 DCHECK_EQ(video_frame->visible_rect().x(), 0);
65 DCHECK_EQ(video_frame->visible_rect().y(), 0);
66
67 switch (video_frame->format()) {
68 // Acceptable inputs.
69 case media::VideoFrame::YV12:
70 case media::VideoFrame::YV12A:
71 case media::VideoFrame::YV16:
72 case media::VideoFrame::YV12J:
73 case media::VideoFrame::NATIVE_TEXTURE:
74 #if defined(VIDEO_HOLE)
75 case media::VideoFrame::HOLE:
76 #endif // defined(VIDEO_HOLE)
77 return true;
78
79 // Unacceptable inputs. ¯\(°_o)/¯
80 case media::VideoFrame::UNKNOWN:
81 case media::VideoFrame::HISTOGRAM_MAX:
82 case media::VideoFrame::I420:
83 break;
84 }
85 return false;
86 }
87
88 // For frames that we receive in software format, determine the dimensions of
89 // each plane in the frame.
SoftwarePlaneDimension(media::VideoFrame::Format input_frame_format,gfx::Size coded_size,ResourceFormat output_resource_format,int plane_index)90 static gfx::Size SoftwarePlaneDimension(
91 media::VideoFrame::Format input_frame_format,
92 gfx::Size coded_size,
93 ResourceFormat output_resource_format,
94 int plane_index) {
95 if (output_resource_format == kYUVResourceFormat) {
96 if (plane_index == media::VideoFrame::kYPlane ||
97 plane_index == media::VideoFrame::kAPlane)
98 return coded_size;
99
100 switch (input_frame_format) {
101 case media::VideoFrame::YV12:
102 case media::VideoFrame::YV12A:
103 case media::VideoFrame::YV12J:
104 return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 0.5f));
105 case media::VideoFrame::YV16:
106 return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 1.f));
107
108 case media::VideoFrame::UNKNOWN:
109 case media::VideoFrame::I420:
110 case media::VideoFrame::NATIVE_TEXTURE:
111 case media::VideoFrame::HISTOGRAM_MAX:
112 #if defined(VIDEO_HOLE)
113 case media::VideoFrame::HOLE:
114 #endif // defined(VIDEO_HOLE)
115 NOTREACHED();
116 }
117 }
118
119 DCHECK_EQ(output_resource_format, kRGBResourceFormat);
120 return coded_size;
121 }
122
CreateForSoftwarePlanes(const scoped_refptr<media::VideoFrame> & video_frame)123 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
124 const scoped_refptr<media::VideoFrame>& video_frame) {
125 media::VideoFrame::Format input_frame_format = video_frame->format();
126
127 #if defined(VIDEO_HOLE)
128 if (input_frame_format == media::VideoFrame::HOLE) {
129 VideoFrameExternalResources external_resources;
130 external_resources.type = VideoFrameExternalResources::HOLE;
131 return external_resources;
132 }
133 #endif // defined(VIDEO_HOLE)
134
135 // Only YUV software video frames are supported.
136 DCHECK(input_frame_format == media::VideoFrame::YV12 ||
137 input_frame_format == media::VideoFrame::YV12A ||
138 input_frame_format == media::VideoFrame::YV12J ||
139 input_frame_format == media::VideoFrame::YV16);
140 if (input_frame_format != media::VideoFrame::YV12 &&
141 input_frame_format != media::VideoFrame::YV12A &&
142 input_frame_format != media::VideoFrame::YV12J &&
143 input_frame_format != media::VideoFrame::YV16)
144 return VideoFrameExternalResources();
145
146 bool software_compositor = context_provider_ == NULL;
147
148 ResourceFormat output_resource_format = kYUVResourceFormat;
149 size_t output_plane_count =
150 (input_frame_format == media::VideoFrame::YV12A) ? 4 : 3;
151
152 // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
153 // conversion here. That involves an extra copy of each frame to a bitmap.
154 // Obviously, this is suboptimal and should be addressed once ubercompositor
155 // starts shaping up.
156 if (software_compositor) {
157 output_resource_format = kRGBResourceFormat;
158 output_plane_count = 1;
159 }
160
161 int max_resource_size = resource_provider_->max_texture_size();
162 gfx::Size coded_frame_size = video_frame->coded_size();
163
164 std::vector<PlaneResource> plane_resources;
165 bool allocation_success = true;
166
167 for (size_t i = 0; i < output_plane_count; ++i) {
168 gfx::Size output_plane_resource_size =
169 SoftwarePlaneDimension(input_frame_format,
170 coded_frame_size,
171 output_resource_format,
172 i);
173 if (output_plane_resource_size.IsEmpty() ||
174 output_plane_resource_size.width() > max_resource_size ||
175 output_plane_resource_size.height() > max_resource_size) {
176 allocation_success = false;
177 break;
178 }
179
180 ResourceProvider::ResourceId resource_id = 0;
181 gpu::Mailbox mailbox;
182
183 // Try recycle a previously-allocated resource.
184 for (size_t i = 0; i < recycled_resources_.size(); ++i) {
185 if (recycled_resources_[i].resource_format == output_resource_format &&
186 recycled_resources_[i].resource_size == output_plane_resource_size) {
187 resource_id = recycled_resources_[i].resource_id;
188 mailbox = recycled_resources_[i].mailbox;
189 recycled_resources_.erase(recycled_resources_.begin() + i);
190 break;
191 }
192 }
193
194 if (resource_id == 0) {
195 // TODO(danakj): Abstract out hw/sw resource create/delete from
196 // ResourceProvider and stop using ResourceProvider in this class.
197 resource_id =
198 resource_provider_->CreateResource(output_plane_resource_size,
199 GL_CLAMP_TO_EDGE,
200 ResourceProvider::TextureUsageAny,
201 output_resource_format);
202
203 DCHECK(mailbox.IsZero());
204
205 if (!software_compositor) {
206 DCHECK(context_provider_);
207
208 gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
209
210 GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
211 if (mailbox.IsZero()) {
212 resource_provider_->DeleteResource(resource_id);
213 resource_id = 0;
214 } else {
215 ResourceProvider::ScopedWriteLockGL lock(
216 resource_provider_, resource_id);
217 GLC(gl, gl->BindTexture(GL_TEXTURE_2D, lock.texture_id()));
218 GLC(gl, gl->ProduceTextureCHROMIUM(GL_TEXTURE_2D, mailbox.name));
219 GLC(gl, gl->BindTexture(GL_TEXTURE_2D, 0));
220 }
221 }
222
223 if (resource_id)
224 all_resources_.push_back(resource_id);
225 }
226
227 if (resource_id == 0) {
228 allocation_success = false;
229 break;
230 }
231
232 DCHECK(software_compositor || !mailbox.IsZero());
233 plane_resources.push_back(PlaneResource(resource_id,
234 output_plane_resource_size,
235 output_resource_format,
236 mailbox));
237 }
238
239 if (!allocation_success) {
240 for (size_t i = 0; i < plane_resources.size(); ++i)
241 DeleteResource(plane_resources[i].resource_id);
242 return VideoFrameExternalResources();
243 }
244
245 VideoFrameExternalResources external_resources;
246
247 if (software_compositor) {
248 DCHECK_EQ(plane_resources.size(), 1u);
249 DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
250 DCHECK(plane_resources[0].mailbox.IsZero());
251
252 if (!video_renderer_)
253 video_renderer_.reset(new media::SkCanvasVideoRenderer);
254
255 {
256 ResourceProvider::ScopedWriteLockSoftware lock(
257 resource_provider_, plane_resources[0].resource_id);
258 video_renderer_->Paint(video_frame.get(),
259 lock.sk_canvas(),
260 video_frame->visible_rect(),
261 0xff);
262 }
263
264 RecycleResourceData recycle_data = {
265 plane_resources[0].resource_id,
266 plane_resources[0].resource_size,
267 plane_resources[0].resource_format,
268 gpu::Mailbox()
269 };
270 base::SharedMemory* shared_memory =
271 resource_provider_->GetSharedMemory(plane_resources[0].resource_id);
272 if (shared_memory) {
273 external_resources.mailboxes.push_back(
274 TextureMailbox(shared_memory, plane_resources[0].resource_size));
275 external_resources.release_callbacks
276 .push_back(base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
277 external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
278 } else {
279 // TODO(jbauman): Remove this path once shared memory is available
280 // everywhere.
281 external_resources.software_resources
282 .push_back(plane_resources[0].resource_id);
283 external_resources.software_release_callback =
284 base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
285 external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
286 }
287
288 return external_resources;
289 }
290
291 for (size_t i = 0; i < plane_resources.size(); ++i) {
292 // Update each plane's resource id with its content.
293 DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
294
295 const uint8_t* input_plane_pixels = video_frame->data(i);
296
297 gfx::Rect image_rect(0,
298 0,
299 video_frame->stride(i),
300 plane_resources[i].resource_size.height());
301 gfx::Rect source_rect(plane_resources[i].resource_size);
302 resource_provider_->SetPixels(plane_resources[i].resource_id,
303 input_plane_pixels,
304 image_rect,
305 source_rect,
306 gfx::Vector2d());
307
308 RecycleResourceData recycle_data = {
309 plane_resources[i].resource_id,
310 plane_resources[i].resource_size,
311 plane_resources[i].resource_format,
312 plane_resources[i].mailbox
313 };
314
315 external_resources.mailboxes.push_back(
316 TextureMailbox(plane_resources[i].mailbox));
317 external_resources.release_callbacks.push_back(
318 base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
319 }
320
321 external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
322 return external_resources;
323 }
324
ReturnTexture(const scoped_refptr<media::VideoFrame> & frame,unsigned sync_point,bool lost_resource)325 static void ReturnTexture(const scoped_refptr<media::VideoFrame>& frame,
326 unsigned sync_point,
327 bool lost_resource) {
328 frame->texture_mailbox()->Resync(sync_point);
329 }
330
CreateForHardwarePlanes(const scoped_refptr<media::VideoFrame> & video_frame)331 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
332 const scoped_refptr<media::VideoFrame>& video_frame) {
333 media::VideoFrame::Format frame_format = video_frame->format();
334
335 DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
336 if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
337 return VideoFrameExternalResources();
338
339 if (!context_provider_)
340 return VideoFrameExternalResources();
341
342 VideoFrameExternalResources external_resources;
343 switch (video_frame->texture_target()) {
344 case GL_TEXTURE_2D:
345 external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
346 break;
347 case GL_TEXTURE_EXTERNAL_OES:
348 external_resources.type =
349 VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
350 break;
351 case GL_TEXTURE_RECTANGLE_ARB:
352 external_resources.type = VideoFrameExternalResources::IO_SURFACE;
353 break;
354 default:
355 NOTREACHED();
356 return VideoFrameExternalResources();
357 }
358
359 media::VideoFrame::MailboxHolder* mailbox_holder =
360 video_frame->texture_mailbox();
361
362 external_resources.mailboxes.push_back(
363 TextureMailbox(mailbox_holder->mailbox(),
364 video_frame->texture_target(),
365 mailbox_holder->sync_point()));
366 external_resources.release_callbacks.push_back(
367 base::Bind(&ReturnTexture, video_frame));
368 return external_resources;
369 }
370
371 // static
RecycleResource(base::WeakPtr<VideoResourceUpdater> updater,RecycleResourceData data,unsigned sync_point,bool lost_resource)372 void VideoResourceUpdater::RecycleResource(
373 base::WeakPtr<VideoResourceUpdater> updater,
374 RecycleResourceData data,
375 unsigned sync_point,
376 bool lost_resource) {
377 if (!updater.get()) {
378 // Resource was already deleted.
379 return;
380 }
381
382 ContextProvider* context_provider = updater->context_provider_;
383 if (context_provider && sync_point) {
384 GLC(context_provider->ContextGL(),
385 context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
386 }
387
388 if (lost_resource) {
389 updater->DeleteResource(data.resource_id);
390 return;
391 }
392
393 // Drop recycled resources that are the wrong format.
394 while (!updater->recycled_resources_.empty() &&
395 updater->recycled_resources_.back().resource_format !=
396 data.resource_format) {
397 updater->DeleteResource(updater->recycled_resources_.back().resource_id);
398 updater->recycled_resources_.pop_back();
399 }
400
401 PlaneResource recycled_resource(data.resource_id,
402 data.resource_size,
403 data.resource_format,
404 data.mailbox);
405 updater->recycled_resources_.push_back(recycled_resource);
406 }
407
408 } // namespace cc
409