1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/renderer/media/webmediaplayer_ms.h"
6
7 #include <limits>
8
9 #include "base/bind.h"
10 #include "base/callback.h"
11 #include "base/message_loop/message_loop.h"
12 #include "base/metrics/histogram.h"
13 #include "cc/layers/video_layer.h"
14 #include "content/public/renderer/render_view.h"
15 #include "content/renderer/compositor_bindings/web_layer_impl.h"
16 #include "content/renderer/media/media_stream_audio_renderer.h"
17 #include "content/renderer/media/media_stream_renderer_factory.h"
18 #include "content/renderer/media/video_frame_provider.h"
19 #include "content/renderer/media/webmediaplayer_delegate.h"
20 #include "content/renderer/media/webmediaplayer_util.h"
21 #include "content/renderer/render_frame_impl.h"
22 #include "media/base/media_log.h"
23 #include "media/base/video_frame.h"
24 #include "media/base/video_util.h"
25 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
26 #include "third_party/WebKit/public/platform/WebRect.h"
27 #include "third_party/WebKit/public/platform/WebSize.h"
28 #include "third_party/WebKit/public/platform/WebURL.h"
29 #include "third_party/WebKit/public/web/WebFrame.h"
30 #include "third_party/WebKit/public/web/WebView.h"
31 #include "third_party/skia/include/core/SkBitmap.h"
32
33 using blink::WebCanvas;
34 using blink::WebMediaPlayer;
35 using blink::WebRect;
36 using blink::WebSize;
37
38 namespace {
39
40 // This function copies a YV12 or NATIVE_TEXTURE to a new YV12
41 // media::VideoFrame.
CopyFrameToYV12(const scoped_refptr<media::VideoFrame> & frame)42 scoped_refptr<media::VideoFrame> CopyFrameToYV12(
43 const scoped_refptr<media::VideoFrame>& frame) {
44 DCHECK(frame->format() == media::VideoFrame::YV12 ||
45 frame->format() == media::VideoFrame::I420 ||
46 frame->format() == media::VideoFrame::NATIVE_TEXTURE);
47 scoped_refptr<media::VideoFrame> new_frame =
48 media::VideoFrame::CreateFrame(media::VideoFrame::YV12,
49 frame->coded_size(),
50 frame->visible_rect(),
51 frame->natural_size(),
52 frame->timestamp());
53
54 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
55 SkBitmap bitmap;
56 bitmap.allocN32Pixels(frame->visible_rect().width(),
57 frame->visible_rect().height());
58 frame->ReadPixelsFromNativeTexture(bitmap);
59
60 media::CopyRGBToVideoFrame(
61 reinterpret_cast<uint8*>(bitmap.getPixels()),
62 bitmap.rowBytes(),
63 frame->visible_rect(),
64 new_frame.get());
65 } else {
66 size_t number_of_planes =
67 media::VideoFrame::NumPlanes(frame->format());
68 for (size_t i = 0; i < number_of_planes; ++i) {
69 media::CopyPlane(i, frame->data(i), frame->stride(i),
70 frame->rows(i), new_frame.get());
71 }
72 }
73 return new_frame;
74 }
75
76 } // anonymous namespace
77
78 namespace content {
79
WebMediaPlayerMS(blink::WebFrame * frame,blink::WebMediaPlayerClient * client,base::WeakPtr<WebMediaPlayerDelegate> delegate,media::MediaLog * media_log,scoped_ptr<MediaStreamRendererFactory> factory)80 WebMediaPlayerMS::WebMediaPlayerMS(
81 blink::WebFrame* frame,
82 blink::WebMediaPlayerClient* client,
83 base::WeakPtr<WebMediaPlayerDelegate> delegate,
84 media::MediaLog* media_log,
85 scoped_ptr<MediaStreamRendererFactory> factory)
86 : frame_(frame),
87 network_state_(WebMediaPlayer::NetworkStateEmpty),
88 ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
89 buffered_(static_cast<size_t>(1)),
90 client_(client),
91 delegate_(delegate),
92 paused_(true),
93 current_frame_used_(false),
94 pending_repaint_(false),
95 video_frame_provider_client_(NULL),
96 received_first_frame_(false),
97 total_frame_count_(0),
98 dropped_frame_count_(0),
99 media_log_(media_log),
100 renderer_factory_(factory.Pass()) {
101 DVLOG(1) << "WebMediaPlayerMS::ctor";
102 media_log_->AddEvent(
103 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
104 }
105
~WebMediaPlayerMS()106 WebMediaPlayerMS::~WebMediaPlayerMS() {
107 DVLOG(1) << "WebMediaPlayerMS::dtor";
108 DCHECK(thread_checker_.CalledOnValidThread());
109
110 SetVideoFrameProviderClient(NULL);
111 GetClient()->setWebLayer(NULL);
112
113 if (video_frame_provider_.get())
114 video_frame_provider_->Stop();
115
116 if (audio_renderer_.get())
117 audio_renderer_->Stop();
118
119 media_log_->AddEvent(
120 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
121
122 if (delegate_.get())
123 delegate_->PlayerGone(this);
124 }
125
load(LoadType load_type,const blink::WebURL & url,CORSMode cors_mode)126 void WebMediaPlayerMS::load(LoadType load_type,
127 const blink::WebURL& url,
128 CORSMode cors_mode) {
129 DVLOG(1) << "WebMediaPlayerMS::load";
130 DCHECK(thread_checker_.CalledOnValidThread());
131
132 // TODO(acolwell): Change this to DCHECK_EQ(load_type,
133 // LoadTypeMediaStream) once Blink-side changes land.
134 DCHECK_NE(load_type, LoadTypeMediaSource);
135
136 GURL gurl(url);
137
138 setVolume(GetClient()->volume());
139 SetNetworkState(WebMediaPlayer::NetworkStateLoading);
140 SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
141 media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
142
143 video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
144 url,
145 base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
146 base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
147
148 RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
149 audio_renderer_ = renderer_factory_->GetAudioRenderer(
150 url,
151 frame->GetRenderView()->GetRoutingID(),
152 frame->GetRoutingID());
153
154 if (video_frame_provider_.get() || audio_renderer_.get()) {
155 if (audio_renderer_.get())
156 audio_renderer_->Start();
157
158 if (video_frame_provider_.get()) {
159 video_frame_provider_->Start();
160 } else {
161 // This is audio-only mode.
162 DCHECK(audio_renderer_.get());
163 SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
164 SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
165 }
166 } else {
167 SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
168 }
169 }
170
play()171 void WebMediaPlayerMS::play() {
172 DVLOG(1) << "WebMediaPlayerMS::play";
173 DCHECK(thread_checker_.CalledOnValidThread());
174
175 if (paused_) {
176 if (video_frame_provider_.get())
177 video_frame_provider_->Play();
178
179 if (audio_renderer_.get())
180 audio_renderer_->Play();
181
182 if (delegate_.get())
183 delegate_->DidPlay(this);
184 }
185
186 paused_ = false;
187
188 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
189 }
190
pause()191 void WebMediaPlayerMS::pause() {
192 DVLOG(1) << "WebMediaPlayerMS::pause";
193 DCHECK(thread_checker_.CalledOnValidThread());
194
195 if (video_frame_provider_.get())
196 video_frame_provider_->Pause();
197
198 if (!paused_) {
199 if (audio_renderer_.get())
200 audio_renderer_->Pause();
201
202 if (delegate_.get())
203 delegate_->DidPause(this);
204 }
205
206 paused_ = true;
207
208 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
209
210 if (!current_frame_)
211 return;
212
213 // Copy the frame so that rendering can show the last received frame.
214 // The original frame must not be referenced when the player is paused since
215 // there might be a finite number of available buffers. E.g, video that
216 // originates from a video camera.
217 scoped_refptr<media::VideoFrame> new_frame = CopyFrameToYV12(current_frame_);
218 base::AutoLock auto_lock(current_frame_lock_);
219 current_frame_ = new_frame;
220 }
221
supportsSave() const222 bool WebMediaPlayerMS::supportsSave() const {
223 DCHECK(thread_checker_.CalledOnValidThread());
224 return false;
225 }
226
seek(double seconds)227 void WebMediaPlayerMS::seek(double seconds) {
228 DCHECK(thread_checker_.CalledOnValidThread());
229 }
230
setRate(double rate)231 void WebMediaPlayerMS::setRate(double rate) {
232 DCHECK(thread_checker_.CalledOnValidThread());
233 }
234
setVolume(double volume)235 void WebMediaPlayerMS::setVolume(double volume) {
236 DCHECK(thread_checker_.CalledOnValidThread());
237 if (!audio_renderer_.get())
238 return;
239 DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
240 audio_renderer_->SetVolume(volume);
241 }
242
setPreload(WebMediaPlayer::Preload preload)243 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
244 DCHECK(thread_checker_.CalledOnValidThread());
245 }
246
hasVideo() const247 bool WebMediaPlayerMS::hasVideo() const {
248 DCHECK(thread_checker_.CalledOnValidThread());
249 return (video_frame_provider_.get() != NULL);
250 }
251
hasAudio() const252 bool WebMediaPlayerMS::hasAudio() const {
253 DCHECK(thread_checker_.CalledOnValidThread());
254 return (audio_renderer_.get() != NULL);
255 }
256
naturalSize() const257 blink::WebSize WebMediaPlayerMS::naturalSize() const {
258 DCHECK(thread_checker_.CalledOnValidThread());
259
260 gfx::Size size;
261 if (current_frame_.get())
262 size = current_frame_->natural_size();
263 DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
264 return blink::WebSize(size);
265 }
266
paused() const267 bool WebMediaPlayerMS::paused() const {
268 DCHECK(thread_checker_.CalledOnValidThread());
269 return paused_;
270 }
271
seeking() const272 bool WebMediaPlayerMS::seeking() const {
273 DCHECK(thread_checker_.CalledOnValidThread());
274 return false;
275 }
276
duration() const277 double WebMediaPlayerMS::duration() const {
278 DCHECK(thread_checker_.CalledOnValidThread());
279 return std::numeric_limits<double>::infinity();
280 }
281
currentTime() const282 double WebMediaPlayerMS::currentTime() const {
283 DCHECK(thread_checker_.CalledOnValidThread());
284 if (current_time_.ToInternalValue() != 0) {
285 return current_time_.InSecondsF();
286 } else if (audio_renderer_.get()) {
287 return audio_renderer_->GetCurrentRenderTime().InSecondsF();
288 }
289 return 0.0;
290 }
291
networkState() const292 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
293 DCHECK(thread_checker_.CalledOnValidThread());
294 DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
295 return network_state_;
296 }
297
readyState() const298 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
299 DCHECK(thread_checker_.CalledOnValidThread());
300 DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
301 return ready_state_;
302 }
303
buffered() const304 blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
305 DCHECK(thread_checker_.CalledOnValidThread());
306 return buffered_;
307 }
308
maxTimeSeekable() const309 double WebMediaPlayerMS::maxTimeSeekable() const {
310 DCHECK(thread_checker_.CalledOnValidThread());
311 return 0.0;
312 }
313
didLoadingProgress()314 bool WebMediaPlayerMS::didLoadingProgress() {
315 DCHECK(thread_checker_.CalledOnValidThread());
316 return true;
317 }
318
paint(WebCanvas * canvas,const WebRect & rect,unsigned char alpha)319 void WebMediaPlayerMS::paint(WebCanvas* canvas,
320 const WebRect& rect,
321 unsigned char alpha) {
322 DVLOG(3) << "WebMediaPlayerMS::paint";
323 DCHECK(thread_checker_.CalledOnValidThread());
324
325 gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
326 video_renderer_.Paint(current_frame_.get(), canvas, dest_rect, alpha);
327
328 {
329 base::AutoLock auto_lock(current_frame_lock_);
330 if (current_frame_.get())
331 current_frame_used_ = true;
332 }
333 }
334
hasSingleSecurityOrigin() const335 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
336 DCHECK(thread_checker_.CalledOnValidThread());
337 return true;
338 }
339
didPassCORSAccessCheck() const340 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
341 DCHECK(thread_checker_.CalledOnValidThread());
342 return true;
343 }
344
mediaTimeForTimeValue(double timeValue) const345 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
346 return ConvertSecondsToTimestamp(timeValue).InSecondsF();
347 }
348
decodedFrameCount() const349 unsigned WebMediaPlayerMS::decodedFrameCount() const {
350 DCHECK(thread_checker_.CalledOnValidThread());
351 DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
352 return total_frame_count_;
353 }
354
droppedFrameCount() const355 unsigned WebMediaPlayerMS::droppedFrameCount() const {
356 DCHECK(thread_checker_.CalledOnValidThread());
357 DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
358 return dropped_frame_count_;
359 }
360
audioDecodedByteCount() const361 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
362 DCHECK(thread_checker_.CalledOnValidThread());
363 NOTIMPLEMENTED();
364 return 0;
365 }
366
videoDecodedByteCount() const367 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
368 DCHECK(thread_checker_.CalledOnValidThread());
369 NOTIMPLEMENTED();
370 return 0;
371 }
372
SetVideoFrameProviderClient(cc::VideoFrameProvider::Client * client)373 void WebMediaPlayerMS::SetVideoFrameProviderClient(
374 cc::VideoFrameProvider::Client* client) {
375 // This is called from both the main renderer thread and the compositor
376 // thread (when the main thread is blocked).
377 if (video_frame_provider_client_)
378 video_frame_provider_client_->StopUsingProvider();
379 video_frame_provider_client_ = client;
380 }
381
GetCurrentFrame()382 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
383 DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
384 base::AutoLock auto_lock(current_frame_lock_);
385 DCHECK(!pending_repaint_);
386 if (!current_frame_.get())
387 return NULL;
388 pending_repaint_ = true;
389 current_frame_used_ = true;
390 return current_frame_;
391 }
392
PutCurrentFrame(const scoped_refptr<media::VideoFrame> & frame)393 void WebMediaPlayerMS::PutCurrentFrame(
394 const scoped_refptr<media::VideoFrame>& frame) {
395 DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
396 DCHECK(pending_repaint_);
397 pending_repaint_ = false;
398 }
399
OnFrameAvailable(const scoped_refptr<media::VideoFrame> & frame)400 void WebMediaPlayerMS::OnFrameAvailable(
401 const scoped_refptr<media::VideoFrame>& frame) {
402 DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
403 DCHECK(thread_checker_.CalledOnValidThread());
404 ++total_frame_count_;
405 if (!received_first_frame_) {
406 received_first_frame_ = true;
407 {
408 base::AutoLock auto_lock(current_frame_lock_);
409 DCHECK(!current_frame_used_);
410 current_frame_ = frame;
411 }
412 SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
413 SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
414 GetClient()->sizeChanged();
415
416 if (video_frame_provider_) {
417 video_weblayer_.reset(new WebLayerImpl(cc::VideoLayer::Create(this)));
418 video_weblayer_->setOpaque(true);
419 GetClient()->setWebLayer(video_weblayer_.get());
420 }
421 }
422
423 // Do not update |current_frame_| when paused.
424 if (paused_)
425 return;
426
427 bool size_changed = !current_frame_.get() ||
428 current_frame_->natural_size() != frame->natural_size();
429
430 {
431 base::AutoLock auto_lock(current_frame_lock_);
432 if (!current_frame_used_ && current_frame_.get())
433 ++dropped_frame_count_;
434 current_frame_ = frame;
435 current_time_ = frame->timestamp();
436 current_frame_used_ = false;
437 }
438
439 if (size_changed)
440 GetClient()->sizeChanged();
441
442 GetClient()->repaint();
443 }
444
RepaintInternal()445 void WebMediaPlayerMS::RepaintInternal() {
446 DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
447 DCHECK(thread_checker_.CalledOnValidThread());
448 GetClient()->repaint();
449 }
450
OnSourceError()451 void WebMediaPlayerMS::OnSourceError() {
452 DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
453 DCHECK(thread_checker_.CalledOnValidThread());
454 SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
455 RepaintInternal();
456 }
457
SetNetworkState(WebMediaPlayer::NetworkState state)458 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
459 DCHECK(thread_checker_.CalledOnValidThread());
460 network_state_ = state;
461 // Always notify to ensure client has the latest value.
462 GetClient()->networkStateChanged();
463 }
464
SetReadyState(WebMediaPlayer::ReadyState state)465 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
466 DCHECK(thread_checker_.CalledOnValidThread());
467 ready_state_ = state;
468 // Always notify to ensure client has the latest value.
469 GetClient()->readyStateChanged();
470 }
471
GetClient()472 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
473 DCHECK(thread_checker_.CalledOnValidThread());
474 DCHECK(client_);
475 return client_;
476 }
477
478 } // namespace content
479