1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/renderer/media/webmediaplayer_ms.h"
6
7 #include <limits>
8
9 #include "base/bind.h"
10 #include "base/callback.h"
11 #include "base/message_loop/message_loop.h"
12 #include "base/metrics/histogram.h"
13 #include "cc/blink/web_layer_impl.h"
14 #include "cc/layers/video_layer.h"
15 #include "content/public/renderer/render_view.h"
16 #include "content/renderer/media/media_stream_audio_renderer.h"
17 #include "content/renderer/media/media_stream_renderer_factory.h"
18 #include "content/renderer/media/video_frame_provider.h"
19 #include "content/renderer/render_frame_impl.h"
20 #include "media/base/media_log.h"
21 #include "media/base/video_frame.h"
22 #include "media/base/video_rotation.h"
23 #include "media/base/video_util.h"
24 #include "media/blink/webmediaplayer_delegate.h"
25 #include "media/blink/webmediaplayer_util.h"
26 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
27 #include "third_party/WebKit/public/platform/WebRect.h"
28 #include "third_party/WebKit/public/platform/WebSize.h"
29 #include "third_party/WebKit/public/platform/WebURL.h"
30 #include "third_party/WebKit/public/web/WebFrame.h"
31 #include "third_party/WebKit/public/web/WebView.h"
32 #include "third_party/skia/include/core/SkBitmap.h"
33
34 using blink::WebCanvas;
35 using blink::WebMediaPlayer;
36 using blink::WebRect;
37 using blink::WebSize;
38
39 namespace {
40
41 // This function copies a YV12 or NATIVE_TEXTURE to a new YV12
42 // media::VideoFrame.
CopyFrameToYV12(const scoped_refptr<media::VideoFrame> & frame)43 scoped_refptr<media::VideoFrame> CopyFrameToYV12(
44 const scoped_refptr<media::VideoFrame>& frame) {
45 DCHECK(frame->format() == media::VideoFrame::YV12 ||
46 frame->format() == media::VideoFrame::I420 ||
47 frame->format() == media::VideoFrame::NATIVE_TEXTURE);
48 scoped_refptr<media::VideoFrame> new_frame =
49 media::VideoFrame::CreateFrame(media::VideoFrame::YV12,
50 frame->coded_size(),
51 frame->visible_rect(),
52 frame->natural_size(),
53 frame->timestamp());
54
55 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
56 SkBitmap bitmap;
57 bitmap.allocN32Pixels(frame->visible_rect().width(),
58 frame->visible_rect().height());
59 frame->ReadPixelsFromNativeTexture(bitmap);
60
61 media::CopyRGBToVideoFrame(
62 reinterpret_cast<uint8*>(bitmap.getPixels()),
63 bitmap.rowBytes(),
64 frame->visible_rect(),
65 new_frame.get());
66 } else {
67 size_t number_of_planes =
68 media::VideoFrame::NumPlanes(frame->format());
69 for (size_t i = 0; i < number_of_planes; ++i) {
70 media::CopyPlane(i, frame->data(i), frame->stride(i),
71 frame->rows(i), new_frame.get());
72 }
73 }
74 return new_frame;
75 }
76
77 } // anonymous namespace
78
79 namespace content {
80
WebMediaPlayerMS(blink::WebFrame * frame,blink::WebMediaPlayerClient * client,base::WeakPtr<media::WebMediaPlayerDelegate> delegate,media::MediaLog * media_log,scoped_ptr<MediaStreamRendererFactory> factory)81 WebMediaPlayerMS::WebMediaPlayerMS(
82 blink::WebFrame* frame,
83 blink::WebMediaPlayerClient* client,
84 base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
85 media::MediaLog* media_log,
86 scoped_ptr<MediaStreamRendererFactory> factory)
87 : frame_(frame),
88 network_state_(WebMediaPlayer::NetworkStateEmpty),
89 ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
90 buffered_(static_cast<size_t>(1)),
91 volume_(1.0f),
92 client_(client),
93 delegate_(delegate),
94 paused_(true),
95 current_frame_used_(false),
96 pending_repaint_(false),
97 video_frame_provider_client_(NULL),
98 received_first_frame_(false),
99 total_frame_count_(0),
100 dropped_frame_count_(0),
101 media_log_(media_log),
102 renderer_factory_(factory.Pass()) {
103 DVLOG(1) << "WebMediaPlayerMS::ctor";
104 media_log_->AddEvent(
105 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
106 }
107
~WebMediaPlayerMS()108 WebMediaPlayerMS::~WebMediaPlayerMS() {
109 DVLOG(1) << "WebMediaPlayerMS::dtor";
110 DCHECK(thread_checker_.CalledOnValidThread());
111
112 SetVideoFrameProviderClient(NULL);
113 GetClient()->setWebLayer(NULL);
114
115 if (video_frame_provider_.get())
116 video_frame_provider_->Stop();
117
118 if (audio_renderer_.get())
119 audio_renderer_->Stop();
120
121 media_log_->AddEvent(
122 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
123
124 if (delegate_.get())
125 delegate_->PlayerGone(this);
126 }
127
load(LoadType load_type,const blink::WebURL & url,CORSMode cors_mode)128 void WebMediaPlayerMS::load(LoadType load_type,
129 const blink::WebURL& url,
130 CORSMode cors_mode) {
131 DVLOG(1) << "WebMediaPlayerMS::load";
132 DCHECK(thread_checker_.CalledOnValidThread());
133
134 // TODO(acolwell): Change this to DCHECK_EQ(load_type,
135 // LoadTypeMediaStream) once Blink-side changes land.
136 DCHECK_NE(load_type, LoadTypeMediaSource);
137
138 GURL gurl(url);
139
140 SetNetworkState(WebMediaPlayer::NetworkStateLoading);
141 SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
142 media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
143
144 video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
145 url,
146 base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
147 base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
148
149 RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
150 audio_renderer_ = renderer_factory_->GetAudioRenderer(
151 url,
152 frame->GetRenderView()->GetRoutingID(),
153 frame->GetRoutingID());
154
155 if (video_frame_provider_.get() || audio_renderer_.get()) {
156 if (audio_renderer_.get()) {
157 audio_renderer_->SetVolume(volume_);
158 audio_renderer_->Start();
159 }
160
161 if (video_frame_provider_.get()) {
162 video_frame_provider_->Start();
163 } else {
164 // This is audio-only mode.
165 DCHECK(audio_renderer_.get());
166 SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
167 SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
168 }
169 } else {
170 SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
171 }
172 }
173
play()174 void WebMediaPlayerMS::play() {
175 DVLOG(1) << "WebMediaPlayerMS::play";
176 DCHECK(thread_checker_.CalledOnValidThread());
177
178 if (paused_) {
179 if (video_frame_provider_.get())
180 video_frame_provider_->Play();
181
182 if (audio_renderer_.get())
183 audio_renderer_->Play();
184
185 if (delegate_.get())
186 delegate_->DidPlay(this);
187 }
188
189 paused_ = false;
190
191 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
192 }
193
pause()194 void WebMediaPlayerMS::pause() {
195 DVLOG(1) << "WebMediaPlayerMS::pause";
196 DCHECK(thread_checker_.CalledOnValidThread());
197
198 if (video_frame_provider_.get())
199 video_frame_provider_->Pause();
200
201 if (!paused_) {
202 if (audio_renderer_.get())
203 audio_renderer_->Pause();
204
205 if (delegate_.get())
206 delegate_->DidPause(this);
207 }
208
209 paused_ = true;
210
211 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
212
213 if (!current_frame_.get())
214 return;
215
216 // Copy the frame so that rendering can show the last received frame.
217 // The original frame must not be referenced when the player is paused since
218 // there might be a finite number of available buffers. E.g, video that
219 // originates from a video camera.
220 scoped_refptr<media::VideoFrame> new_frame = CopyFrameToYV12(current_frame_);
221 base::AutoLock auto_lock(current_frame_lock_);
222 current_frame_ = new_frame;
223 }
224
supportsSave() const225 bool WebMediaPlayerMS::supportsSave() const {
226 DCHECK(thread_checker_.CalledOnValidThread());
227 return false;
228 }
229
seek(double seconds)230 void WebMediaPlayerMS::seek(double seconds) {
231 DCHECK(thread_checker_.CalledOnValidThread());
232 }
233
setRate(double rate)234 void WebMediaPlayerMS::setRate(double rate) {
235 DCHECK(thread_checker_.CalledOnValidThread());
236 }
237
setVolume(double volume)238 void WebMediaPlayerMS::setVolume(double volume) {
239 DCHECK(thread_checker_.CalledOnValidThread());
240 DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
241 volume_ = volume;
242 if (audio_renderer_.get())
243 audio_renderer_->SetVolume(volume_);
244 }
245
setPreload(WebMediaPlayer::Preload preload)246 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
247 DCHECK(thread_checker_.CalledOnValidThread());
248 }
249
hasVideo() const250 bool WebMediaPlayerMS::hasVideo() const {
251 DCHECK(thread_checker_.CalledOnValidThread());
252 return (video_frame_provider_.get() != NULL);
253 }
254
hasAudio() const255 bool WebMediaPlayerMS::hasAudio() const {
256 DCHECK(thread_checker_.CalledOnValidThread());
257 return (audio_renderer_.get() != NULL);
258 }
259
naturalSize() const260 blink::WebSize WebMediaPlayerMS::naturalSize() const {
261 DCHECK(thread_checker_.CalledOnValidThread());
262
263 gfx::Size size;
264 if (current_frame_.get())
265 size = current_frame_->natural_size();
266 DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
267 return blink::WebSize(size);
268 }
269
paused() const270 bool WebMediaPlayerMS::paused() const {
271 DCHECK(thread_checker_.CalledOnValidThread());
272 return paused_;
273 }
274
seeking() const275 bool WebMediaPlayerMS::seeking() const {
276 DCHECK(thread_checker_.CalledOnValidThread());
277 return false;
278 }
279
duration() const280 double WebMediaPlayerMS::duration() const {
281 DCHECK(thread_checker_.CalledOnValidThread());
282 return std::numeric_limits<double>::infinity();
283 }
284
currentTime() const285 double WebMediaPlayerMS::currentTime() const {
286 DCHECK(thread_checker_.CalledOnValidThread());
287 if (current_time_.ToInternalValue() != 0) {
288 return current_time_.InSecondsF();
289 } else if (audio_renderer_.get()) {
290 return audio_renderer_->GetCurrentRenderTime().InSecondsF();
291 }
292 return 0.0;
293 }
294
networkState() const295 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
296 DCHECK(thread_checker_.CalledOnValidThread());
297 DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
298 return network_state_;
299 }
300
readyState() const301 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
302 DCHECK(thread_checker_.CalledOnValidThread());
303 DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
304 return ready_state_;
305 }
306
buffered() const307 blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
308 DCHECK(thread_checker_.CalledOnValidThread());
309 return buffered_;
310 }
311
maxTimeSeekable() const312 double WebMediaPlayerMS::maxTimeSeekable() const {
313 DCHECK(thread_checker_.CalledOnValidThread());
314 return 0.0;
315 }
316
didLoadingProgress()317 bool WebMediaPlayerMS::didLoadingProgress() {
318 DCHECK(thread_checker_.CalledOnValidThread());
319 return true;
320 }
321
paint(blink::WebCanvas * canvas,const blink::WebRect & rect,unsigned char alpha)322 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
323 const blink::WebRect& rect,
324 unsigned char alpha) {
325 paint(canvas, rect, alpha, SkXfermode::kSrcOver_Mode);
326 }
327
paint(blink::WebCanvas * canvas,const blink::WebRect & rect,unsigned char alpha,SkXfermode::Mode mode)328 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
329 const blink::WebRect& rect,
330 unsigned char alpha,
331 SkXfermode::Mode mode) {
332 DVLOG(3) << "WebMediaPlayerMS::paint";
333 DCHECK(thread_checker_.CalledOnValidThread());
334
335 gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
336 video_renderer_.Paint(
337 current_frame_, canvas, dest_rect, alpha, mode, media::VIDEO_ROTATION_0);
338
339 {
340 base::AutoLock auto_lock(current_frame_lock_);
341 if (current_frame_.get())
342 current_frame_used_ = true;
343 }
344 }
345
hasSingleSecurityOrigin() const346 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
347 DCHECK(thread_checker_.CalledOnValidThread());
348 return true;
349 }
350
didPassCORSAccessCheck() const351 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
352 DCHECK(thread_checker_.CalledOnValidThread());
353 return true;
354 }
355
mediaTimeForTimeValue(double timeValue) const356 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
357 return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
358 }
359
decodedFrameCount() const360 unsigned WebMediaPlayerMS::decodedFrameCount() const {
361 DCHECK(thread_checker_.CalledOnValidThread());
362 DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
363 return total_frame_count_;
364 }
365
droppedFrameCount() const366 unsigned WebMediaPlayerMS::droppedFrameCount() const {
367 DCHECK(thread_checker_.CalledOnValidThread());
368 DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
369 return dropped_frame_count_;
370 }
371
audioDecodedByteCount() const372 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
373 DCHECK(thread_checker_.CalledOnValidThread());
374 NOTIMPLEMENTED();
375 return 0;
376 }
377
videoDecodedByteCount() const378 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
379 DCHECK(thread_checker_.CalledOnValidThread());
380 NOTIMPLEMENTED();
381 return 0;
382 }
383
SetVideoFrameProviderClient(cc::VideoFrameProvider::Client * client)384 void WebMediaPlayerMS::SetVideoFrameProviderClient(
385 cc::VideoFrameProvider::Client* client) {
386 // This is called from both the main renderer thread and the compositor
387 // thread (when the main thread is blocked).
388 if (video_frame_provider_client_)
389 video_frame_provider_client_->StopUsingProvider();
390 video_frame_provider_client_ = client;
391 }
392
GetCurrentFrame()393 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
394 DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
395 base::AutoLock auto_lock(current_frame_lock_);
396 DCHECK(!pending_repaint_);
397 if (!current_frame_.get())
398 return NULL;
399 pending_repaint_ = true;
400 current_frame_used_ = true;
401 return current_frame_;
402 }
403
PutCurrentFrame(const scoped_refptr<media::VideoFrame> & frame)404 void WebMediaPlayerMS::PutCurrentFrame(
405 const scoped_refptr<media::VideoFrame>& frame) {
406 DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
407 DCHECK(pending_repaint_);
408 pending_repaint_ = false;
409 }
410
OnFrameAvailable(const scoped_refptr<media::VideoFrame> & frame)411 void WebMediaPlayerMS::OnFrameAvailable(
412 const scoped_refptr<media::VideoFrame>& frame) {
413 DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
414 DCHECK(thread_checker_.CalledOnValidThread());
415 ++total_frame_count_;
416 if (!received_first_frame_) {
417 received_first_frame_ = true;
418 {
419 base::AutoLock auto_lock(current_frame_lock_);
420 DCHECK(!current_frame_used_);
421 current_frame_ = frame;
422 }
423 SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
424 SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
425 GetClient()->sizeChanged();
426
427 if (video_frame_provider_.get()) {
428 video_weblayer_.reset(new cc_blink::WebLayerImpl(
429 cc::VideoLayer::Create(this, media::VIDEO_ROTATION_0)));
430 video_weblayer_->setOpaque(true);
431 GetClient()->setWebLayer(video_weblayer_.get());
432 }
433 }
434
435 // Do not update |current_frame_| when paused.
436 if (paused_)
437 return;
438
439 bool size_changed = !current_frame_.get() ||
440 current_frame_->natural_size() != frame->natural_size();
441
442 {
443 base::AutoLock auto_lock(current_frame_lock_);
444 if (!current_frame_used_ && current_frame_.get())
445 ++dropped_frame_count_;
446 current_frame_ = frame;
447 current_time_ = frame->timestamp();
448 current_frame_used_ = false;
449 }
450
451 if (size_changed)
452 GetClient()->sizeChanged();
453
454 GetClient()->repaint();
455 }
456
RepaintInternal()457 void WebMediaPlayerMS::RepaintInternal() {
458 DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
459 DCHECK(thread_checker_.CalledOnValidThread());
460 GetClient()->repaint();
461 }
462
OnSourceError()463 void WebMediaPlayerMS::OnSourceError() {
464 DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
465 DCHECK(thread_checker_.CalledOnValidThread());
466 SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
467 RepaintInternal();
468 }
469
SetNetworkState(WebMediaPlayer::NetworkState state)470 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
471 DCHECK(thread_checker_.CalledOnValidThread());
472 network_state_ = state;
473 // Always notify to ensure client has the latest value.
474 GetClient()->networkStateChanged();
475 }
476
SetReadyState(WebMediaPlayer::ReadyState state)477 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
478 DCHECK(thread_checker_.CalledOnValidThread());
479 ready_state_ = state;
480 // Always notify to ensure client has the latest value.
481 GetClient()->readyStateChanged();
482 }
483
GetClient()484 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
485 DCHECK(thread_checker_.CalledOnValidThread());
486 DCHECK(client_);
487 return client_;
488 }
489
490 } // namespace content
491