1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/browser/media/capture/content_video_capture_device_core.h"
6
7 #include "base/basictypes.h"
8 #include "base/bind.h"
9 #include "base/callback_forward.h"
10 #include "base/callback_helpers.h"
11 #include "base/debug/trace_event.h"
12 #include "base/logging.h"
13 #include "base/memory/scoped_ptr.h"
14 #include "base/memory/weak_ptr.h"
15 #include "base/message_loop/message_loop_proxy.h"
16 #include "base/metrics/histogram.h"
17 #include "base/sequenced_task_runner.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/strings/stringprintf.h"
20 #include "base/synchronization/lock.h"
21 #include "base/threading/thread.h"
22 #include "base/threading/thread_checker.h"
23 #include "base/time/time.h"
24 #include "content/public/browser/browser_thread.h"
25 #include "media/base/bind_to_current_loop.h"
26 #include "media/base/video_frame.h"
27 #include "media/base/video_util.h"
28 #include "media/video/capture/video_capture_types.h"
29 #include "ui/gfx/rect.h"
30
31 namespace content {
32
33 namespace {
34
DeleteCaptureMachineOnUIThread(scoped_ptr<VideoCaptureMachine> capture_machine)35 void DeleteCaptureMachineOnUIThread(
36 scoped_ptr<VideoCaptureMachine> capture_machine) {
37 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
38
39 capture_machine.reset();
40 }
41
42 } // namespace
43
ThreadSafeCaptureOracle(scoped_ptr<media::VideoCaptureDevice::Client> client,scoped_ptr<VideoCaptureOracle> oracle,const media::VideoCaptureParams & params)44 ThreadSafeCaptureOracle::ThreadSafeCaptureOracle(
45 scoped_ptr<media::VideoCaptureDevice::Client> client,
46 scoped_ptr<VideoCaptureOracle> oracle,
47 const media::VideoCaptureParams& params)
48 : client_(client.Pass()),
49 oracle_(oracle.Pass()),
50 params_(params),
51 capture_size_updated_(false) {
52 switch (params_.requested_format.pixel_format) {
53 case media::PIXEL_FORMAT_I420:
54 video_frame_format_ = media::VideoFrame::I420;
55 break;
56 case media::PIXEL_FORMAT_TEXTURE:
57 video_frame_format_ = media::VideoFrame::NATIVE_TEXTURE;
58 break;
59 default:
60 LOG(FATAL) << "Unexpected pixel_format "
61 << params_.requested_format.pixel_format;
62 }
63 }
64
~ThreadSafeCaptureOracle()65 ThreadSafeCaptureOracle::~ThreadSafeCaptureOracle() {}
66
ObserveEventAndDecideCapture(VideoCaptureOracle::Event event,base::TimeTicks event_time,scoped_refptr<media::VideoFrame> * storage,CaptureFrameCallback * callback)67 bool ThreadSafeCaptureOracle::ObserveEventAndDecideCapture(
68 VideoCaptureOracle::Event event,
69 base::TimeTicks event_time,
70 scoped_refptr<media::VideoFrame>* storage,
71 CaptureFrameCallback* callback) {
72 base::AutoLock guard(lock_);
73
74 if (!client_)
75 return false; // Capture is stopped.
76
77 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer =
78 client_->ReserveOutputBuffer(video_frame_format_,
79 params_.requested_format.frame_size);
80 const bool should_capture =
81 oracle_->ObserveEventAndDecideCapture(event, event_time);
82 const bool content_is_dirty =
83 (event == VideoCaptureOracle::kCompositorUpdate ||
84 event == VideoCaptureOracle::kSoftwarePaint);
85 const char* event_name =
86 (event == VideoCaptureOracle::kTimerPoll ? "poll" :
87 (event == VideoCaptureOracle::kCompositorUpdate ? "gpu" :
88 "paint"));
89
90 // Consider the various reasons not to initiate a capture.
91 if (should_capture && !output_buffer) {
92 TRACE_EVENT_INSTANT1("mirroring",
93 "EncodeLimited",
94 TRACE_EVENT_SCOPE_THREAD,
95 "trigger",
96 event_name);
97 return false;
98 } else if (!should_capture && output_buffer) {
99 if (content_is_dirty) {
100 // This is a normal and acceptable way to drop a frame. We've hit our
101 // capture rate limit: for example, the content is animating at 60fps but
102 // we're capturing at 30fps.
103 TRACE_EVENT_INSTANT1("mirroring", "FpsRateLimited",
104 TRACE_EVENT_SCOPE_THREAD,
105 "trigger", event_name);
106 }
107 return false;
108 } else if (!should_capture && !output_buffer) {
109 // We decided not to capture, but we wouldn't have been able to if we wanted
110 // to because no output buffer was available.
111 TRACE_EVENT_INSTANT1("mirroring", "NearlyEncodeLimited",
112 TRACE_EVENT_SCOPE_THREAD,
113 "trigger", event_name);
114 return false;
115 }
116 int frame_number = oracle_->RecordCapture();
117 TRACE_EVENT_ASYNC_BEGIN2("mirroring", "Capture", output_buffer.get(),
118 "frame_number", frame_number,
119 "trigger", event_name);
120 // NATIVE_TEXTURE frames wrap a texture mailbox, which we don't have at the
121 // moment. We do not construct those frames.
122 if (video_frame_format_ != media::VideoFrame::NATIVE_TEXTURE) {
123 *storage = media::VideoFrame::WrapExternalPackedMemory(
124 video_frame_format_,
125 params_.requested_format.frame_size,
126 gfx::Rect(params_.requested_format.frame_size),
127 params_.requested_format.frame_size,
128 static_cast<uint8*>(output_buffer->data()),
129 output_buffer->size(),
130 base::SharedMemory::NULLHandle(),
131 base::TimeDelta(),
132 base::Closure());
133 }
134 *callback = base::Bind(&ThreadSafeCaptureOracle::DidCaptureFrame,
135 this,
136 frame_number,
137 output_buffer);
138 return true;
139 }
140
GetCaptureSize() const141 gfx::Size ThreadSafeCaptureOracle::GetCaptureSize() const {
142 base::AutoLock guard(lock_);
143 return params_.requested_format.frame_size;
144 }
145
UpdateCaptureSize(const gfx::Size & source_size)146 void ThreadSafeCaptureOracle::UpdateCaptureSize(const gfx::Size& source_size) {
147 base::AutoLock guard(lock_);
148
149 // If this is the first call to UpdateCaptureSize(), or the receiver supports
150 // variable resolution, then determine the capture size by treating the
151 // requested width and height as maxima.
152 if (!capture_size_updated_ || params_.allow_resolution_change) {
153 // The capture resolution should not exceed the source frame size.
154 // In other words it should downscale the image but not upscale it.
155 if (source_size.width() > params_.requested_format.frame_size.width() ||
156 source_size.height() > params_.requested_format.frame_size.height()) {
157 gfx::Rect capture_rect = media::ComputeLetterboxRegion(
158 gfx::Rect(params_.requested_format.frame_size), source_size);
159 params_.requested_format.frame_size.SetSize(
160 MakeEven(capture_rect.width()), MakeEven(capture_rect.height()));
161 } else {
162 params_.requested_format.frame_size.SetSize(
163 MakeEven(source_size.width()), MakeEven(source_size.height()));
164 }
165 capture_size_updated_ = true;
166 }
167 }
168
Stop()169 void ThreadSafeCaptureOracle::Stop() {
170 base::AutoLock guard(lock_);
171 client_.reset();
172 }
173
ReportError(const std::string & reason)174 void ThreadSafeCaptureOracle::ReportError(const std::string& reason) {
175 base::AutoLock guard(lock_);
176 if (client_)
177 client_->OnError(reason);
178 }
179
DidCaptureFrame(int frame_number,const scoped_refptr<media::VideoCaptureDevice::Client::Buffer> & buffer,const scoped_refptr<media::VideoFrame> & frame,base::TimeTicks timestamp,bool success)180 void ThreadSafeCaptureOracle::DidCaptureFrame(
181 int frame_number,
182 const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
183 const scoped_refptr<media::VideoFrame>& frame,
184 base::TimeTicks timestamp,
185 bool success) {
186 base::AutoLock guard(lock_);
187 TRACE_EVENT_ASYNC_END2("mirroring", "Capture", buffer.get(),
188 "success", success,
189 "timestamp", timestamp.ToInternalValue());
190
191 if (!client_)
192 return; // Capture is stopped.
193
194 if (success) {
195 if (oracle_->CompleteCapture(frame_number, timestamp)) {
196 media::VideoCaptureFormat format = params_.requested_format;
197 format.frame_size = frame->coded_size();
198 client_->OnIncomingCapturedVideoFrame(buffer, format, frame, timestamp);
199 }
200 }
201 }
202
AllocateAndStart(const media::VideoCaptureParams & params,scoped_ptr<media::VideoCaptureDevice::Client> client)203 void ContentVideoCaptureDeviceCore::AllocateAndStart(
204 const media::VideoCaptureParams& params,
205 scoped_ptr<media::VideoCaptureDevice::Client> client) {
206 DCHECK(thread_checker_.CalledOnValidThread());
207
208 if (state_ != kIdle) {
209 DVLOG(1) << "Allocate() invoked when not in state Idle.";
210 return;
211 }
212
213 if (params.requested_format.frame_rate <= 0) {
214 std::string error_msg("Invalid frame_rate: ");
215 error_msg += base::DoubleToString(params.requested_format.frame_rate);
216 DVLOG(1) << error_msg;
217 client->OnError(error_msg);
218 return;
219 }
220
221 if (params.requested_format.pixel_format != media::PIXEL_FORMAT_I420 &&
222 params.requested_format.pixel_format != media::PIXEL_FORMAT_TEXTURE) {
223 std::string error_msg = base::StringPrintf(
224 "unsupported format: %d", params.requested_format.pixel_format);
225 DVLOG(1) << error_msg;
226 client->OnError(error_msg);
227 return;
228 }
229
230 if (params.requested_format.frame_size.width() < kMinFrameWidth ||
231 params.requested_format.frame_size.height() < kMinFrameHeight) {
232 std::string error_msg =
233 "invalid frame size: " + params.requested_format.frame_size.ToString();
234 DVLOG(1) << error_msg;
235 client->OnError(error_msg);
236 return;
237 }
238
239 media::VideoCaptureParams new_params = params;
240 // Frame dimensions must each be an even integer since the client wants (or
241 // will convert to) YUV420.
242 new_params.requested_format.frame_size.SetSize(
243 MakeEven(params.requested_format.frame_size.width()),
244 MakeEven(params.requested_format.frame_size.height()));
245
246 base::TimeDelta capture_period = base::TimeDelta::FromMicroseconds(
247 1000000.0 / params.requested_format.frame_rate + 0.5);
248
249 scoped_ptr<VideoCaptureOracle> oracle(
250 new VideoCaptureOracle(capture_period,
251 kAcceleratedSubscriberIsSupported));
252 oracle_proxy_ =
253 new ThreadSafeCaptureOracle(client.Pass(), oracle.Pass(), new_params);
254
255 // Starts the capture machine asynchronously.
256 BrowserThread::PostTaskAndReplyWithResult(
257 BrowserThread::UI,
258 FROM_HERE,
259 base::Bind(&VideoCaptureMachine::Start,
260 base::Unretained(capture_machine_.get()),
261 oracle_proxy_,
262 new_params),
263 base::Bind(&ContentVideoCaptureDeviceCore::CaptureStarted, AsWeakPtr()));
264
265 TransitionStateTo(kCapturing);
266 }
267
StopAndDeAllocate()268 void ContentVideoCaptureDeviceCore::StopAndDeAllocate() {
269 DCHECK(thread_checker_.CalledOnValidThread());
270
271 if (state_ != kCapturing)
272 return;
273
274 oracle_proxy_->Stop();
275 oracle_proxy_ = NULL;
276
277 TransitionStateTo(kIdle);
278
279 // Stops the capture machine asynchronously.
280 BrowserThread::PostTask(
281 BrowserThread::UI, FROM_HERE, base::Bind(
282 &VideoCaptureMachine::Stop,
283 base::Unretained(capture_machine_.get()),
284 base::Bind(&base::DoNothing)));
285 }
286
CaptureStarted(bool success)287 void ContentVideoCaptureDeviceCore::CaptureStarted(bool success) {
288 DCHECK(thread_checker_.CalledOnValidThread());
289 if (!success) {
290 std::string reason("Failed to start capture machine.");
291 DVLOG(1) << reason;
292 Error(reason);
293 }
294 }
295
ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine> capture_machine)296 ContentVideoCaptureDeviceCore::ContentVideoCaptureDeviceCore(
297 scoped_ptr<VideoCaptureMachine> capture_machine)
298 : state_(kIdle),
299 capture_machine_(capture_machine.Pass()) {}
300
~ContentVideoCaptureDeviceCore()301 ContentVideoCaptureDeviceCore::~ContentVideoCaptureDeviceCore() {
302 // If capture_machine is not NULL, then we need to return to the UI thread to
303 // safely stop the capture machine.
304 if (capture_machine_) {
305 VideoCaptureMachine* capture_machine_ptr = capture_machine_.get();
306 BrowserThread::PostTask(
307 BrowserThread::UI, FROM_HERE,
308 base::Bind(&VideoCaptureMachine::Stop,
309 base::Unretained(capture_machine_ptr),
310 base::Bind(&DeleteCaptureMachineOnUIThread,
311 base::Passed(&capture_machine_))));
312 }
313 DVLOG(1) << "ContentVideoCaptureDeviceCore@" << this << " destroying.";
314 }
315
TransitionStateTo(State next_state)316 void ContentVideoCaptureDeviceCore::TransitionStateTo(State next_state) {
317 DCHECK(thread_checker_.CalledOnValidThread());
318
319 #ifndef NDEBUG
320 static const char* kStateNames[] = {
321 "Idle", "Allocated", "Capturing", "Error"
322 };
323 DVLOG(1) << "State change: " << kStateNames[state_]
324 << " --> " << kStateNames[next_state];
325 #endif
326
327 state_ = next_state;
328 }
329
Error(const std::string & reason)330 void ContentVideoCaptureDeviceCore::Error(const std::string& reason) {
331 DCHECK(thread_checker_.CalledOnValidThread());
332
333 if (state_ == kIdle)
334 return;
335
336 if (oracle_proxy_)
337 oracle_proxy_->ReportError(reason);
338
339 StopAndDeAllocate();
340 TransitionStateTo(kError);
341 }
342
343 } // namespace content
344