1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/desktop_capture/desktop_and_cursor_composer.h"
12
13 #include <stdint.h>
14 #include <string.h>
15
16 #include <memory>
17 #include <utility>
18
19 #include "modules/desktop_capture/desktop_capturer.h"
20 #include "modules/desktop_capture/desktop_frame.h"
21 #include "modules/desktop_capture/mouse_cursor.h"
22 #include "modules/desktop_capture/shared_desktop_frame.h"
23 #include "rtc_base/arraysize.h"
24 #include "test/gtest.h"
25
26 namespace webrtc {
27
28 namespace {
29
30 const int kScreenWidth = 100;
31 const int kScreenHeight = 100;
32 const int kCursorWidth = 10;
33 const int kCursorHeight = 10;
34
35 const int kTestCursorSize = 3;
36 const uint32_t kTestCursorData[kTestCursorSize][kTestCursorSize] = {
37 {
38 0xffffffff,
39 0x99990000,
40 0xaa222222,
41 },
42 {
43 0x88008800,
44 0xaa0000aa,
45 0xaa333333,
46 },
47 {
48 0x00000000,
49 0xaa0000aa,
50 0xaa333333,
51 },
52 };
53
GetFakeFramePixelValue(const DesktopVector & p)54 uint32_t GetFakeFramePixelValue(const DesktopVector& p) {
55 uint32_t r = 100 + p.x();
56 uint32_t g = 100 + p.y();
57 uint32_t b = 100 + p.x() + p.y();
58 return b + (g << 8) + (r << 16) + 0xff000000;
59 }
60
GetFramePixel(const DesktopFrame & frame,const DesktopVector & pos)61 uint32_t GetFramePixel(const DesktopFrame& frame, const DesktopVector& pos) {
62 return *reinterpret_cast<uint32_t*>(frame.GetFrameDataAtPos(pos));
63 }
64
65 // Blends two pixel values taking into account alpha.
BlendPixels(uint32_t dest,uint32_t src)66 uint32_t BlendPixels(uint32_t dest, uint32_t src) {
67 uint8_t alpha = 255 - ((src & 0xff000000) >> 24);
68 uint32_t r =
69 ((dest & 0x00ff0000) >> 16) * alpha / 255 + ((src & 0x00ff0000) >> 16);
70 uint32_t g =
71 ((dest & 0x0000ff00) >> 8) * alpha / 255 + ((src & 0x0000ff00) >> 8);
72 uint32_t b = (dest & 0x000000ff) * alpha / 255 + (src & 0x000000ff);
73 return b + (g << 8) + (r << 16) + 0xff000000;
74 }
75
CreateTestFrame()76 DesktopFrame* CreateTestFrame() {
77 DesktopFrame* frame =
78 new BasicDesktopFrame(DesktopSize(kScreenWidth, kScreenHeight));
79 uint32_t* data = reinterpret_cast<uint32_t*>(frame->data());
80 for (int y = 0; y < kScreenHeight; ++y) {
81 for (int x = 0; x < kScreenWidth; ++x) {
82 *(data++) = GetFakeFramePixelValue(DesktopVector(x, y));
83 }
84 }
85 return frame;
86 }
87
CreateTestCursor(DesktopVector hotspot)88 MouseCursor* CreateTestCursor(DesktopVector hotspot) {
89 std::unique_ptr<DesktopFrame> image(
90 new BasicDesktopFrame(DesktopSize(kCursorWidth, kCursorHeight)));
91 uint32_t* data = reinterpret_cast<uint32_t*>(image->data());
92 // Set four pixels near the hotspot and leave all other blank.
93 for (int y = 0; y < kTestCursorSize; ++y) {
94 for (int x = 0; x < kTestCursorSize; ++x) {
95 data[(hotspot.y() + y) * kCursorWidth + (hotspot.x() + x)] =
96 kTestCursorData[y][x];
97 }
98 }
99 return new MouseCursor(image.release(), hotspot);
100 }
101
102 class FakeScreenCapturer : public DesktopCapturer {
103 public:
FakeScreenCapturer()104 FakeScreenCapturer() {}
105
Start(Callback * callback)106 void Start(Callback* callback) override { callback_ = callback; }
107
CaptureFrame()108 void CaptureFrame() override {
109 callback_->OnCaptureResult(
110 next_frame_ ? Result::SUCCESS : Result::ERROR_TEMPORARY,
111 std::move(next_frame_));
112 }
113
SetNextFrame(std::unique_ptr<DesktopFrame> next_frame)114 void SetNextFrame(std::unique_ptr<DesktopFrame> next_frame) {
115 next_frame_ = std::move(next_frame);
116 }
117
IsOccluded(const DesktopVector & pos)118 bool IsOccluded(const DesktopVector& pos) override { return is_occluded_; }
119
set_is_occluded(bool value)120 void set_is_occluded(bool value) { is_occluded_ = value; }
121
122 private:
123 Callback* callback_ = nullptr;
124
125 std::unique_ptr<DesktopFrame> next_frame_;
126 bool is_occluded_ = false;
127 };
128
129 class FakeMouseMonitor : public MouseCursorMonitor {
130 public:
FakeMouseMonitor()131 FakeMouseMonitor() : changed_(true) {}
132
SetState(CursorState state,const DesktopVector & pos)133 void SetState(CursorState state, const DesktopVector& pos) {
134 state_ = state;
135 position_ = pos;
136 }
137
SetHotspot(const DesktopVector & hotspot)138 void SetHotspot(const DesktopVector& hotspot) {
139 if (!hotspot_.equals(hotspot))
140 changed_ = true;
141 hotspot_ = hotspot;
142 }
143
Init(Callback * callback,Mode mode)144 void Init(Callback* callback, Mode mode) override { callback_ = callback; }
145
Capture()146 void Capture() override {
147 if (changed_) {
148 callback_->OnMouseCursor(CreateTestCursor(hotspot_));
149 }
150 callback_->OnMouseCursorPosition(position_);
151 }
152
153 private:
154 Callback* callback_;
155 CursorState state_;
156 DesktopVector position_;
157 DesktopVector hotspot_;
158 bool changed_;
159 };
160
VerifyFrame(const DesktopFrame & frame,MouseCursorMonitor::CursorState state,const DesktopVector & pos)161 void VerifyFrame(const DesktopFrame& frame,
162 MouseCursorMonitor::CursorState state,
163 const DesktopVector& pos) {
164 // Verify that all other pixels are set to their original values.
165 DesktopRect image_rect =
166 DesktopRect::MakeWH(kTestCursorSize, kTestCursorSize);
167 image_rect.Translate(pos);
168
169 for (int y = 0; y < kScreenHeight; ++y) {
170 for (int x = 0; x < kScreenWidth; ++x) {
171 DesktopVector p(x, y);
172 if (state == MouseCursorMonitor::INSIDE && image_rect.Contains(p)) {
173 EXPECT_EQ(BlendPixels(GetFakeFramePixelValue(p),
174 kTestCursorData[y - pos.y()][x - pos.x()]),
175 GetFramePixel(frame, p));
176 } else {
177 EXPECT_EQ(GetFakeFramePixelValue(p), GetFramePixel(frame, p));
178 }
179 }
180 }
181 }
182
183 } // namespace
184
185 class DesktopAndCursorComposerTest : public ::testing::Test,
186 public DesktopCapturer::Callback {
187 public:
DesktopAndCursorComposerTest(bool include_cursor=true)188 DesktopAndCursorComposerTest(bool include_cursor = true)
189 : fake_screen_(new FakeScreenCapturer()),
190 fake_cursor_(include_cursor ? new FakeMouseMonitor() : nullptr),
191 blender_(fake_screen_, fake_cursor_) {
192 blender_.Start(this);
193 }
194
195 // DesktopCapturer::Callback interface
OnCaptureResult(DesktopCapturer::Result result,std::unique_ptr<DesktopFrame> frame)196 void OnCaptureResult(DesktopCapturer::Result result,
197 std::unique_ptr<DesktopFrame> frame) override {
198 frame_ = std::move(frame);
199 }
200
201 protected:
202 // Owned by |blender_|.
203 FakeScreenCapturer* fake_screen_;
204 FakeMouseMonitor* fake_cursor_;
205
206 DesktopAndCursorComposer blender_;
207 std::unique_ptr<DesktopFrame> frame_;
208 };
209
210 class DesktopAndCursorComposerNoCursorMonitorTest
211 : public DesktopAndCursorComposerTest {
212 public:
DesktopAndCursorComposerNoCursorMonitorTest()213 DesktopAndCursorComposerNoCursorMonitorTest()
214 : DesktopAndCursorComposerTest(false) {}
215 };
216
TEST_F(DesktopAndCursorComposerTest,CursorShouldBeIgnoredIfNoFrameCaptured)217 TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfNoFrameCaptured) {
218 struct {
219 int x, y;
220 int hotspot_x, hotspot_y;
221 bool inside;
222 } tests[] = {
223 {0, 0, 0, 0, true}, {50, 50, 0, 0, true}, {100, 50, 0, 0, true},
224 {50, 100, 0, 0, true}, {100, 100, 0, 0, true}, {0, 0, 2, 5, true},
225 {1, 1, 2, 5, true}, {50, 50, 2, 5, true}, {100, 100, 2, 5, true},
226 {0, 0, 5, 2, true}, {50, 50, 5, 2, true}, {100, 100, 5, 2, true},
227 {0, 0, 0, 0, false},
228 };
229
230 for (size_t i = 0; i < arraysize(tests); i++) {
231 SCOPED_TRACE(i);
232
233 DesktopVector hotspot(tests[i].hotspot_x, tests[i].hotspot_y);
234 fake_cursor_->SetHotspot(hotspot);
235
236 MouseCursorMonitor::CursorState state = tests[i].inside
237 ? MouseCursorMonitor::INSIDE
238 : MouseCursorMonitor::OUTSIDE;
239 DesktopVector pos(tests[i].x, tests[i].y);
240 fake_cursor_->SetState(state, pos);
241
242 std::unique_ptr<SharedDesktopFrame> frame(
243 SharedDesktopFrame::Wrap(CreateTestFrame()));
244
245 blender_.CaptureFrame();
246 // If capturer captured nothing, then cursor should be ignored, not matter
247 // its state or position.
248 EXPECT_EQ(frame_, nullptr);
249 }
250 }
251
TEST_F(DesktopAndCursorComposerTest,CursorShouldBeIgnoredIfItIsOutOfDesktopFrame)252 TEST_F(DesktopAndCursorComposerTest,
253 CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) {
254 std::unique_ptr<SharedDesktopFrame> frame(
255 SharedDesktopFrame::Wrap(CreateTestFrame()));
256 frame->set_top_left(DesktopVector(100, 200));
257 // The frame covers (100, 200) - (200, 300).
258
259 struct {
260 int x;
261 int y;
262 } tests[] = {
263 {0, 0}, {50, 50}, {50, 150}, {100, 150}, {50, 200},
264 {99, 200}, {100, 199}, {200, 300}, {200, 299}, {199, 300},
265 {-1, -1}, {-10000, -10000}, {10000, 10000},
266 };
267 for (size_t i = 0; i < arraysize(tests); i++) {
268 SCOPED_TRACE(i);
269
270 fake_screen_->SetNextFrame(frame->Share());
271 // The CursorState is ignored when using absolute cursor position.
272 fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE,
273 DesktopVector(tests[i].x, tests[i].y));
274 blender_.CaptureFrame();
275 VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector(0, 0));
276 }
277 }
278
TEST_F(DesktopAndCursorComposerTest,IsOccludedShouldBeConsidered)279 TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) {
280 std::unique_ptr<SharedDesktopFrame> frame(
281 SharedDesktopFrame::Wrap(CreateTestFrame()));
282 frame->set_top_left(DesktopVector(100, 200));
283 // The frame covers (100, 200) - (200, 300).
284
285 struct {
286 int x;
287 int y;
288 } tests[] = {
289 {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299},
290 };
291 fake_screen_->set_is_occluded(true);
292 for (size_t i = 0; i < arraysize(tests); i++) {
293 SCOPED_TRACE(i);
294
295 fake_screen_->SetNextFrame(frame->Share());
296 // The CursorState is ignored when using absolute cursor position.
297 fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE,
298 DesktopVector(tests[i].x, tests[i].y));
299 blender_.CaptureFrame();
300 VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector());
301 }
302 }
303
TEST_F(DesktopAndCursorComposerTest,CursorIncluded)304 TEST_F(DesktopAndCursorComposerTest, CursorIncluded) {
305 std::unique_ptr<SharedDesktopFrame> frame(
306 SharedDesktopFrame::Wrap(CreateTestFrame()));
307 frame->set_top_left(DesktopVector(100, 200));
308 // The frame covers (100, 200) - (200, 300).
309
310 struct {
311 int x;
312 int y;
313 } tests[] = {
314 {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299},
315 };
316 for (size_t i = 0; i < arraysize(tests); i++) {
317 SCOPED_TRACE(i);
318
319 const DesktopVector abs_pos(tests[i].x, tests[i].y);
320 const DesktopVector rel_pos(abs_pos.subtract(frame->top_left()));
321
322 fake_screen_->SetNextFrame(frame->Share());
323 // The CursorState is ignored when using absolute cursor position.
324 fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, abs_pos);
325 blender_.CaptureFrame();
326 VerifyFrame(*frame_, MouseCursorMonitor::INSIDE, rel_pos);
327
328 // Verify that the cursor is erased before the frame buffer is returned to
329 // the screen capturer.
330 frame_.reset();
331 VerifyFrame(*frame, MouseCursorMonitor::OUTSIDE, DesktopVector());
332 }
333 }
334
TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,UpdatedRegionIncludesOldAndNewCursorRectsIfMoved)335 TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
336 UpdatedRegionIncludesOldAndNewCursorRectsIfMoved) {
337 std::unique_ptr<SharedDesktopFrame> frame(
338 SharedDesktopFrame::Wrap(CreateTestFrame()));
339 DesktopRect first_cursor_rect;
340 {
341 // Block to scope test_cursor, which is invalidated by OnMouseCursor.
342 MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
343 first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
344 blender_.OnMouseCursor(test_cursor);
345 }
346 blender_.OnMouseCursorPosition(DesktopVector(0, 0));
347 fake_screen_->SetNextFrame(frame->Share());
348 blender_.CaptureFrame();
349
350 DesktopVector cursor_move_offset(1, 1);
351 DesktopRect second_cursor_rect = first_cursor_rect;
352 second_cursor_rect.Translate(cursor_move_offset);
353 blender_.OnMouseCursorPosition(cursor_move_offset);
354 fake_screen_->SetNextFrame(frame->Share());
355 blender_.CaptureFrame();
356
357 EXPECT_TRUE(frame->updated_region().is_empty());
358 DesktopRegion expected_region;
359 expected_region.AddRect(first_cursor_rect);
360 expected_region.AddRect(second_cursor_rect);
361 EXPECT_TRUE(frame_->updated_region().Equals(expected_region));
362 }
363
TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,UpdatedRegionIncludesOldAndNewCursorRectsIfShapeChanged)364 TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
365 UpdatedRegionIncludesOldAndNewCursorRectsIfShapeChanged) {
366 std::unique_ptr<SharedDesktopFrame> frame(
367 SharedDesktopFrame::Wrap(CreateTestFrame()));
368 DesktopRect first_cursor_rect;
369 {
370 // Block to scope test_cursor, which is invalidated by OnMouseCursor.
371 MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
372 first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
373 blender_.OnMouseCursor(test_cursor);
374 }
375 blender_.OnMouseCursorPosition(DesktopVector(0, 0));
376 fake_screen_->SetNextFrame(frame->Share());
377 blender_.CaptureFrame();
378
379 // Create a second cursor, the same shape as the first. Since the code doesn't
380 // compare the cursor pixels, this is sufficient, and avoids needing two test
381 // cursor bitmaps.
382 DesktopRect second_cursor_rect;
383 {
384 MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
385 second_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
386 blender_.OnMouseCursor(test_cursor);
387 }
388 fake_screen_->SetNextFrame(frame->Share());
389 blender_.CaptureFrame();
390
391 EXPECT_TRUE(frame->updated_region().is_empty());
392 DesktopRegion expected_region;
393 expected_region.AddRect(first_cursor_rect);
394 expected_region.AddRect(second_cursor_rect);
395 EXPECT_TRUE(frame_->updated_region().Equals(expected_region));
396 }
397
TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,UpdatedRegionUnchangedIfCursorUnchanged)398 TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
399 UpdatedRegionUnchangedIfCursorUnchanged) {
400 std::unique_ptr<SharedDesktopFrame> frame(
401 SharedDesktopFrame::Wrap(CreateTestFrame()));
402 blender_.OnMouseCursor(CreateTestCursor(DesktopVector(0, 0)));
403 blender_.OnMouseCursorPosition(DesktopVector(0, 0));
404 fake_screen_->SetNextFrame(frame->Share());
405 blender_.CaptureFrame();
406 fake_screen_->SetNextFrame(frame->Share());
407 blender_.CaptureFrame();
408
409 EXPECT_TRUE(frame->updated_region().is_empty());
410 EXPECT_TRUE(frame_->updated_region().is_empty());
411 }
412
413 } // namespace webrtc
414