1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "cc/resources/video_resource_updater.h"
6
7 #include "base/bind.h"
8 #include "cc/output/gl_renderer.h"
9 #include "cc/resources/resource_provider.h"
10 #include "gpu/GLES2/gl2extchromium.h"
11 #include "gpu/command_buffer/client/gles2_interface.h"
12 #include "media/base/video_frame.h"
13 #include "media/filters/skcanvas_video_renderer.h"
14 #include "third_party/khronos/GLES2/gl2.h"
15 #include "third_party/khronos/GLES2/gl2ext.h"
16 #include "ui/gfx/size_conversions.h"
17
18 namespace cc {
19
20 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
21 const ResourceFormat kRGBResourceFormat = RGBA_8888;
22
VideoFrameExternalResources()23 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
24
~VideoFrameExternalResources()25 VideoFrameExternalResources::~VideoFrameExternalResources() {}
26
VideoResourceUpdater(ContextProvider * context_provider,ResourceProvider * resource_provider)27 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
28 ResourceProvider* resource_provider)
29 : context_provider_(context_provider),
30 resource_provider_(resource_provider) {
31 }
32
~VideoResourceUpdater()33 VideoResourceUpdater::~VideoResourceUpdater() {
34 while (!all_resources_.empty()) {
35 resource_provider_->DeleteResource(all_resources_.back());
36 all_resources_.pop_back();
37 }
38 }
39
DeleteResource(unsigned resource_id)40 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
41 resource_provider_->DeleteResource(resource_id);
42 all_resources_.erase(std::remove(all_resources_.begin(),
43 all_resources_.end(),
44 resource_id));
45 }
46
47 VideoFrameExternalResources VideoResourceUpdater::
CreateExternalResourcesFromVideoFrame(const scoped_refptr<media::VideoFrame> & video_frame)48 CreateExternalResourcesFromVideoFrame(
49 const scoped_refptr<media::VideoFrame>& video_frame) {
50 if (!VerifyFrame(video_frame))
51 return VideoFrameExternalResources();
52
53 if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
54 return CreateForHardwarePlanes(video_frame);
55 else
56 return CreateForSoftwarePlanes(video_frame);
57 }
58
VerifyFrame(const scoped_refptr<media::VideoFrame> & video_frame)59 bool VideoResourceUpdater::VerifyFrame(
60 const scoped_refptr<media::VideoFrame>& video_frame) {
61 switch (video_frame->format()) {
62 // Acceptable inputs.
63 case media::VideoFrame::YV12:
64 case media::VideoFrame::I420:
65 case media::VideoFrame::YV12A:
66 case media::VideoFrame::YV16:
67 case media::VideoFrame::YV12J:
68 case media::VideoFrame::YV24:
69 case media::VideoFrame::NATIVE_TEXTURE:
70 #if defined(VIDEO_HOLE)
71 case media::VideoFrame::HOLE:
72 #endif // defined(VIDEO_HOLE)
73 return true;
74
75 // Unacceptable inputs. ¯\(°_o)/¯
76 case media::VideoFrame::UNKNOWN:
77 case media::VideoFrame::NV12:
78 break;
79 }
80 return false;
81 }
82
83 // For frames that we receive in software format, determine the dimensions of
84 // each plane in the frame.
SoftwarePlaneDimension(const scoped_refptr<media::VideoFrame> & input_frame,ResourceFormat output_resource_format,size_t plane_index)85 static gfx::Size SoftwarePlaneDimension(
86 const scoped_refptr<media::VideoFrame>& input_frame,
87 ResourceFormat output_resource_format,
88 size_t plane_index) {
89 if (output_resource_format == kYUVResourceFormat) {
90 return media::VideoFrame::PlaneSize(
91 input_frame->format(), plane_index, input_frame->coded_size());
92 }
93
94 DCHECK_EQ(output_resource_format, kRGBResourceFormat);
95 return input_frame->coded_size();
96 }
97
CreateForSoftwarePlanes(const scoped_refptr<media::VideoFrame> & video_frame)98 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
99 const scoped_refptr<media::VideoFrame>& video_frame) {
100 media::VideoFrame::Format input_frame_format = video_frame->format();
101
102 #if defined(VIDEO_HOLE)
103 if (input_frame_format == media::VideoFrame::HOLE) {
104 VideoFrameExternalResources external_resources;
105 external_resources.type = VideoFrameExternalResources::HOLE;
106 return external_resources;
107 }
108 #endif // defined(VIDEO_HOLE)
109
110 // Only YUV software video frames are supported.
111 DCHECK(input_frame_format == media::VideoFrame::YV12 ||
112 input_frame_format == media::VideoFrame::I420 ||
113 input_frame_format == media::VideoFrame::YV12A ||
114 input_frame_format == media::VideoFrame::YV12J ||
115 input_frame_format == media::VideoFrame::YV16 ||
116 input_frame_format == media::VideoFrame::YV24);
117 if (input_frame_format != media::VideoFrame::YV12 &&
118 input_frame_format != media::VideoFrame::I420 &&
119 input_frame_format != media::VideoFrame::YV12A &&
120 input_frame_format != media::VideoFrame::YV12J &&
121 input_frame_format != media::VideoFrame::YV16 &&
122 input_frame_format != media::VideoFrame::YV24)
123 return VideoFrameExternalResources();
124
125 bool software_compositor = context_provider_ == NULL;
126
127 ResourceFormat output_resource_format = kYUVResourceFormat;
128 size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
129
130 // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
131 // conversion here. That involves an extra copy of each frame to a bitmap.
132 // Obviously, this is suboptimal and should be addressed once ubercompositor
133 // starts shaping up.
134 if (software_compositor) {
135 output_resource_format = kRGBResourceFormat;
136 output_plane_count = 1;
137 }
138
139 int max_resource_size = resource_provider_->max_texture_size();
140 std::vector<PlaneResource> plane_resources;
141 bool allocation_success = true;
142
143 for (size_t i = 0; i < output_plane_count; ++i) {
144 gfx::Size output_plane_resource_size =
145 SoftwarePlaneDimension(video_frame, output_resource_format, i);
146 if (output_plane_resource_size.IsEmpty() ||
147 output_plane_resource_size.width() > max_resource_size ||
148 output_plane_resource_size.height() > max_resource_size) {
149 allocation_success = false;
150 break;
151 }
152
153 ResourceProvider::ResourceId resource_id = 0;
154 gpu::Mailbox mailbox;
155
156 // Try recycle a previously-allocated resource.
157 for (size_t i = 0; i < recycled_resources_.size(); ++i) {
158 bool resource_matches =
159 recycled_resources_[i].resource_format == output_resource_format &&
160 recycled_resources_[i].resource_size == output_plane_resource_size;
161 bool not_in_use =
162 !software_compositor || !resource_provider_->InUseByConsumer(
163 recycled_resources_[i].resource_id);
164 if (resource_matches && not_in_use) {
165 resource_id = recycled_resources_[i].resource_id;
166 mailbox = recycled_resources_[i].mailbox;
167 recycled_resources_.erase(recycled_resources_.begin() + i);
168 break;
169 }
170 }
171
172 if (resource_id == 0) {
173 // TODO(danakj): Abstract out hw/sw resource create/delete from
174 // ResourceProvider and stop using ResourceProvider in this class.
175 resource_id =
176 resource_provider_->CreateResource(output_plane_resource_size,
177 GL_CLAMP_TO_EDGE,
178 ResourceProvider::TextureUsageAny,
179 output_resource_format);
180
181 DCHECK(mailbox.IsZero());
182
183 if (!software_compositor) {
184 DCHECK(context_provider_);
185
186 gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
187
188 GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
189 ResourceProvider::ScopedWriteLockGL lock(resource_provider_,
190 resource_id);
191 GLC(gl, gl->BindTexture(GL_TEXTURE_2D, lock.texture_id()));
192 GLC(gl, gl->ProduceTextureCHROMIUM(GL_TEXTURE_2D, mailbox.name));
193 GLC(gl, gl->BindTexture(GL_TEXTURE_2D, 0));
194 }
195
196 if (resource_id)
197 all_resources_.push_back(resource_id);
198 }
199
200 if (resource_id == 0) {
201 allocation_success = false;
202 break;
203 }
204
205 DCHECK(software_compositor || !mailbox.IsZero());
206 plane_resources.push_back(PlaneResource(resource_id,
207 output_plane_resource_size,
208 output_resource_format,
209 mailbox));
210 }
211
212 if (!allocation_success) {
213 for (size_t i = 0; i < plane_resources.size(); ++i)
214 DeleteResource(plane_resources[i].resource_id);
215 return VideoFrameExternalResources();
216 }
217
218 VideoFrameExternalResources external_resources;
219
220 if (software_compositor) {
221 DCHECK_EQ(plane_resources.size(), 1u);
222 DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
223 DCHECK(plane_resources[0].mailbox.IsZero());
224
225 if (!video_renderer_)
226 video_renderer_.reset(new media::SkCanvasVideoRenderer);
227
228 {
229 ResourceProvider::ScopedWriteLockSoftware lock(
230 resource_provider_, plane_resources[0].resource_id);
231 video_renderer_->Paint(video_frame.get(),
232 lock.sk_canvas(),
233 video_frame->visible_rect(),
234 0xff);
235 }
236
237 RecycleResourceData recycle_data = {
238 plane_resources[0].resource_id,
239 plane_resources[0].resource_size,
240 plane_resources[0].resource_format,
241 gpu::Mailbox()
242 };
243 external_resources.software_resources.push_back(
244 plane_resources[0].resource_id);
245 external_resources.software_release_callback =
246 base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
247 external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
248
249 return external_resources;
250 }
251
252 for (size_t i = 0; i < plane_resources.size(); ++i) {
253 // Update each plane's resource id with its content.
254 DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
255
256 const uint8_t* input_plane_pixels = video_frame->data(i);
257
258 gfx::Rect image_rect(0,
259 0,
260 video_frame->stride(i),
261 plane_resources[i].resource_size.height());
262 gfx::Rect source_rect(plane_resources[i].resource_size);
263 resource_provider_->SetPixels(plane_resources[i].resource_id,
264 input_plane_pixels,
265 image_rect,
266 source_rect,
267 gfx::Vector2d());
268
269 RecycleResourceData recycle_data = {
270 plane_resources[i].resource_id,
271 plane_resources[i].resource_size,
272 plane_resources[i].resource_format,
273 plane_resources[i].mailbox
274 };
275
276 external_resources.mailboxes.push_back(
277 TextureMailbox(plane_resources[i].mailbox, GL_TEXTURE_2D, 0));
278 external_resources.release_callbacks.push_back(
279 base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
280 }
281
282 external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
283 return external_resources;
284 }
285
ReturnTexture(const scoped_refptr<media::VideoFrame> & frame,uint32 sync_point,bool lost_resource)286 static void ReturnTexture(const scoped_refptr<media::VideoFrame>& frame,
287 uint32 sync_point,
288 bool lost_resource) {
289 frame->AppendReleaseSyncPoint(sync_point);
290 }
291
CreateForHardwarePlanes(const scoped_refptr<media::VideoFrame> & video_frame)292 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
293 const scoped_refptr<media::VideoFrame>& video_frame) {
294 media::VideoFrame::Format frame_format = video_frame->format();
295
296 DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
297 if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
298 return VideoFrameExternalResources();
299
300 if (!context_provider_)
301 return VideoFrameExternalResources();
302
303 const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
304 VideoFrameExternalResources external_resources;
305 switch (mailbox_holder->texture_target) {
306 case GL_TEXTURE_2D:
307 external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
308 break;
309 case GL_TEXTURE_EXTERNAL_OES:
310 external_resources.type =
311 VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
312 break;
313 case GL_TEXTURE_RECTANGLE_ARB:
314 external_resources.type = VideoFrameExternalResources::IO_SURFACE;
315 break;
316 default:
317 NOTREACHED();
318 return VideoFrameExternalResources();
319 }
320
321 external_resources.mailboxes.push_back(
322 TextureMailbox(mailbox_holder->mailbox,
323 mailbox_holder->texture_target,
324 mailbox_holder->sync_point));
325 external_resources.release_callbacks.push_back(
326 base::Bind(&ReturnTexture, video_frame));
327 return external_resources;
328 }
329
330 // static
RecycleResource(base::WeakPtr<VideoResourceUpdater> updater,RecycleResourceData data,uint32 sync_point,bool lost_resource)331 void VideoResourceUpdater::RecycleResource(
332 base::WeakPtr<VideoResourceUpdater> updater,
333 RecycleResourceData data,
334 uint32 sync_point,
335 bool lost_resource) {
336 if (!updater.get()) {
337 // Resource was already deleted.
338 return;
339 }
340
341 ContextProvider* context_provider = updater->context_provider_;
342 if (context_provider && sync_point) {
343 GLC(context_provider->ContextGL(),
344 context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
345 }
346
347 if (lost_resource) {
348 updater->DeleteResource(data.resource_id);
349 return;
350 }
351
352 // Drop recycled resources that are the wrong format.
353 while (!updater->recycled_resources_.empty() &&
354 updater->recycled_resources_.back().resource_format !=
355 data.resource_format) {
356 updater->DeleteResource(updater->recycled_resources_.back().resource_id);
357 updater->recycled_resources_.pop_back();
358 }
359
360 PlaneResource recycled_resource(data.resource_id,
361 data.resource_size,
362 data.resource_format,
363 data.mailbox);
364 updater->recycled_resources_.push_back(recycled_resource);
365 }
366
367 } // namespace cc
368