1 /*
2 * Copyright 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <cstdint>
18 #include <memory>
19
20 #include "VirtualCameraDevice.h"
21 #include "VirtualCameraSession.h"
22 #include "aidl/android/companion/virtualcamera/BnVirtualCameraCallback.h"
23 #include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
24 #include "aidl/android/companion/virtualcamera/VirtualCameraConfiguration.h"
25 #include "aidl/android/hardware/camera/common/Status.h"
26 #include "aidl/android/hardware/camera/device/BnCameraDeviceCallback.h"
27 #include "aidl/android/hardware/camera/device/StreamConfiguration.h"
28 #include "aidl/android/hardware/graphics/common/PixelFormat.h"
29 #include "android/binder_auto_utils.h"
30 #include "android/binder_interface_utils.h"
31 #include "gmock/gmock.h"
32 #include "gtest/gtest.h"
33 #include "util/MetadataUtil.h"
34
35 namespace android {
36 namespace companion {
37 namespace virtualcamera {
38 namespace {
39
40 constexpr char kCameraId[] = "42";
41 constexpr int kQvgaWidth = 320;
42 constexpr int kQvgaHeight = 240;
43 constexpr int kVgaWidth = 640;
44 constexpr int kVgaHeight = 480;
45 constexpr int kSvgaWidth = 800;
46 constexpr int kSvgaHeight = 600;
47 constexpr int kMaxFps = 30;
48 constexpr int kStreamId = 0;
49 constexpr int kSecondStreamId = 1;
50 constexpr int kDefaultDeviceId = 0;
51
52 using ::aidl::android::companion::virtualcamera::BnVirtualCameraCallback;
53 using ::aidl::android::companion::virtualcamera::Format;
54 using ::aidl::android::companion::virtualcamera::LensFacing;
55 using ::aidl::android::companion::virtualcamera::SensorOrientation;
56 using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
57 using ::aidl::android::companion::virtualcamera::VirtualCameraConfiguration;
58 using ::aidl::android::hardware::camera::common::Status;
59 using ::aidl::android::hardware::camera::device::BnCameraDeviceCallback;
60 using ::aidl::android::hardware::camera::device::BufferRequest;
61 using ::aidl::android::hardware::camera::device::BufferRequestStatus;
62 using ::aidl::android::hardware::camera::device::CaptureRequest;
63 using ::aidl::android::hardware::camera::device::CaptureResult;
64 using ::aidl::android::hardware::camera::device::HalStream;
65 using ::aidl::android::hardware::camera::device::NotifyMsg;
66 using ::aidl::android::hardware::camera::device::Stream;
67 using ::aidl::android::hardware::camera::device::StreamBuffer;
68 using ::aidl::android::hardware::camera::device::StreamBufferRet;
69 using ::aidl::android::hardware::camera::device::StreamConfiguration;
70 using ::aidl::android::hardware::graphics::common::PixelFormat;
71 using ::aidl::android::view::Surface;
72 using ::testing::_;
73 using ::testing::ElementsAre;
74 using ::testing::Eq;
75 using ::testing::Return;
76 using ::testing::SizeIs;
77
createStream(int streamId,int width,int height,PixelFormat format)78 Stream createStream(int streamId, int width, int height, PixelFormat format) {
79 Stream s;
80 s.id = streamId;
81 s.width = width;
82 s.height = height;
83 s.format = format;
84 return s;
85 }
86
87 class MockCameraDeviceCallback : public BnCameraDeviceCallback {
88 public:
89 MOCK_METHOD(ndk::ScopedAStatus, notify, (const std::vector<NotifyMsg>&),
90 (override));
91 MOCK_METHOD(ndk::ScopedAStatus, processCaptureResult,
92 (const std::vector<CaptureResult>&), (override));
93 MOCK_METHOD(ndk::ScopedAStatus, requestStreamBuffers,
94 (const std::vector<BufferRequest>&, std::vector<StreamBufferRet>*,
95 BufferRequestStatus*),
96 (override));
97 MOCK_METHOD(ndk::ScopedAStatus, returnStreamBuffers,
98 (const std::vector<StreamBuffer>&), (override));
99 };
100
101 class MockVirtualCameraCallback : public BnVirtualCameraCallback {
102 public:
103 MOCK_METHOD(ndk::ScopedAStatus, onStreamConfigured,
104 (int, const Surface&, int32_t, int32_t, Format), (override));
105 MOCK_METHOD(ndk::ScopedAStatus, onProcessCaptureRequest, (int, int),
106 (override));
107 MOCK_METHOD(ndk::ScopedAStatus, onStreamClosed, (int), (override));
108 };
109
110 class VirtualCameraSessionTestBase : public ::testing::Test {
111 public:
SetUp()112 virtual void SetUp() override {
113 mMockCameraDeviceCallback =
114 ndk::SharedRefBase::make<MockCameraDeviceCallback>();
115 mMockVirtualCameraClientCallback =
116 ndk::SharedRefBase::make<MockVirtualCameraCallback>();
117
118 // Explicitly defining default actions below to prevent gmock from
119 // default-constructing ndk::ScopedAStatus, because default-constructed
120 // status wraps nullptr AStatus and causes crash when attempting to print
121 // it in gtest report.
122 ON_CALL(*mMockCameraDeviceCallback, notify)
123 .WillByDefault(ndk::ScopedAStatus::ok);
124 ON_CALL(*mMockCameraDeviceCallback, processCaptureResult)
125 .WillByDefault(ndk::ScopedAStatus::ok);
126 ON_CALL(*mMockCameraDeviceCallback, requestStreamBuffers)
127 .WillByDefault(ndk::ScopedAStatus::ok);
128 ON_CALL(*mMockCameraDeviceCallback, returnStreamBuffers)
129 .WillByDefault(ndk::ScopedAStatus::ok);
130
131 ON_CALL(*mMockVirtualCameraClientCallback, onStreamConfigured)
132 .WillByDefault(ndk::ScopedAStatus::ok);
133 ON_CALL(*mMockVirtualCameraClientCallback, onProcessCaptureRequest)
134 .WillByDefault(ndk::ScopedAStatus::ok);
135 ON_CALL(*mMockVirtualCameraClientCallback, onStreamClosed)
136 .WillByDefault(ndk::ScopedAStatus::ok);
137 }
138
139 protected:
140 std::shared_ptr<MockCameraDeviceCallback> mMockCameraDeviceCallback;
141 std::shared_ptr<MockVirtualCameraCallback> mMockVirtualCameraClientCallback;
142 };
143
144 class VirtualCameraSessionTest : public VirtualCameraSessionTestBase {
145 public:
SetUp()146 void SetUp() override {
147 VirtualCameraSessionTestBase::SetUp();
148
149 mVirtualCameraDevice = ndk::SharedRefBase::make<VirtualCameraDevice>(
150 kCameraId,
151 VirtualCameraConfiguration{
152 .supportedStreamConfigs = {SupportedStreamConfiguration{
153 .width = kVgaWidth,
154 .height = kVgaHeight,
155 .pixelFormat = Format::YUV_420_888,
156 .maxFps = kMaxFps},
157 SupportedStreamConfiguration{
158 .width = kSvgaWidth,
159 .height = kSvgaHeight,
160 .pixelFormat = Format::YUV_420_888,
161 .maxFps = kMaxFps}},
162 .virtualCameraCallback = mMockVirtualCameraClientCallback,
163 .sensorOrientation = SensorOrientation::ORIENTATION_0,
164 .lensFacing = LensFacing::FRONT},
165 kDefaultDeviceId);
166 mVirtualCameraSession = ndk::SharedRefBase::make<VirtualCameraSession>(
167 mVirtualCameraDevice, mMockCameraDeviceCallback,
168 mMockVirtualCameraClientCallback);
169 }
170
171 protected:
172 std::shared_ptr<VirtualCameraDevice> mVirtualCameraDevice;
173 std::shared_ptr<VirtualCameraSession> mVirtualCameraSession;
174 };
175
TEST_F(VirtualCameraSessionTest,ConfigureTriggersClientConfigureCallback)176 TEST_F(VirtualCameraSessionTest, ConfigureTriggersClientConfigureCallback) {
177 PixelFormat format = PixelFormat::YCBCR_420_888;
178 StreamConfiguration streamConfiguration;
179 streamConfiguration.streams = {
180 createStream(kStreamId, kVgaWidth, kVgaHeight, format),
181 createStream(kSecondStreamId, kSvgaWidth, kSvgaHeight, format)};
182 std::vector<HalStream> halStreams;
183
184 // Expect highest resolution to be picked for the client input.
185 EXPECT_CALL(*mMockVirtualCameraClientCallback,
186 onStreamConfigured(kStreamId, _, kSvgaWidth, kSvgaHeight,
187 Format::YUV_420_888));
188
189 ASSERT_TRUE(
190 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
191 .isOk());
192
193 EXPECT_THAT(halStreams, SizeIs(streamConfiguration.streams.size()));
194 EXPECT_THAT(mVirtualCameraSession->getStreamIds(),
195 ElementsAre(kStreamId, kSecondStreamId));
196 }
197
TEST_F(VirtualCameraSessionTest,SecondConfigureDropsUnreferencedStreams)198 TEST_F(VirtualCameraSessionTest, SecondConfigureDropsUnreferencedStreams) {
199 PixelFormat format = PixelFormat::YCBCR_420_888;
200 StreamConfiguration streamConfiguration;
201 std::vector<HalStream> halStreams;
202
203 streamConfiguration.streams = {createStream(0, kVgaWidth, kVgaHeight, format),
204 createStream(1, kVgaWidth, kVgaHeight, format),
205 createStream(2, kVgaWidth, kVgaHeight, format)};
206 ASSERT_TRUE(
207 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
208 .isOk());
209
210 EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0, 1, 2));
211
212 streamConfiguration.streams = {createStream(0, kVgaWidth, kVgaHeight, format),
213 createStream(2, kVgaWidth, kVgaHeight, format),
214 createStream(3, kVgaWidth, kVgaHeight, format)};
215 ASSERT_TRUE(
216 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
217 .isOk());
218
219 EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0, 2, 3));
220 }
221
TEST_F(VirtualCameraSessionTest,CloseTriggersClientTerminateCallback)222 TEST_F(VirtualCameraSessionTest, CloseTriggersClientTerminateCallback) {
223 // First, configure a stream.
224 PixelFormat format = PixelFormat::YCBCR_420_888;
225 StreamConfiguration streamConfiguration;
226 streamConfiguration.streams = {
227 createStream(kStreamId, kVgaWidth, kVgaHeight, format)};
228 std::vector<HalStream> halStreams;
229 ASSERT_TRUE(
230 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
231 .isOk());
232
233 EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamClosed(kStreamId))
234 .WillOnce(Return(ndk::ScopedAStatus::ok()));
235
236 ASSERT_TRUE(mVirtualCameraSession->close().isOk());
237 }
238
TEST_F(VirtualCameraSessionTest,FlushBeforeConfigure)239 TEST_F(VirtualCameraSessionTest, FlushBeforeConfigure) {
240 // Flush request coming before the configure request finished
241 // (so potentially the thread is not yet running) should be
242 // gracefully handled.
243
244 EXPECT_TRUE(mVirtualCameraSession->flush().isOk());
245 }
246
TEST_F(VirtualCameraSessionTest,onProcessCaptureRequestTriggersClientCallback)247 TEST_F(VirtualCameraSessionTest, onProcessCaptureRequestTriggersClientCallback) {
248 StreamConfiguration streamConfiguration;
249 streamConfiguration.streams = {createStream(kStreamId, kVgaWidth, kVgaHeight,
250 PixelFormat::YCBCR_420_888)};
251 std::vector<CaptureRequest> requests(1);
252 requests[0].frameNumber = 42;
253 requests[0].settings = *(
254 MetadataBuilder().setControlAfMode(ANDROID_CONTROL_AF_MODE_AUTO).build());
255
256 std::vector<HalStream> halStreams;
257 ASSERT_TRUE(
258 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
259 .isOk());
260
261 EXPECT_CALL(*mMockVirtualCameraClientCallback,
262 onProcessCaptureRequest(kStreamId, requests[0].frameNumber))
263 .WillOnce(Return(ndk::ScopedAStatus::ok()));
264 int32_t aidlReturn = 0;
265 ASSERT_TRUE(mVirtualCameraSession
266 ->processCaptureRequest(requests, /*in_cachesToRemove=*/{},
267 &aidlReturn)
268 .isOk());
269 EXPECT_THAT(aidlReturn, Eq(requests.size()));
270 }
271
TEST_F(VirtualCameraSessionTest,configureAfterCameraRelease)272 TEST_F(VirtualCameraSessionTest, configureAfterCameraRelease) {
273 StreamConfiguration streamConfiguration;
274 streamConfiguration.streams = {createStream(kStreamId, kVgaWidth, kVgaHeight,
275 PixelFormat::YCBCR_420_888)};
276 std::vector<HalStream> halStreams;
277
278 // Release virtual camera.
279 mVirtualCameraDevice.reset();
280
281 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
282 EXPECT_THAT(
283 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
284 .getServiceSpecificError(),
285 Eq(static_cast<int32_t>(Status::CAMERA_DISCONNECTED)));
286 }
287
TEST_F(VirtualCameraSessionTest,ConfigureWithEmptyStreams)288 TEST_F(VirtualCameraSessionTest, ConfigureWithEmptyStreams) {
289 StreamConfiguration streamConfiguration;
290 std::vector<HalStream> halStreams;
291
292 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
293 EXPECT_THAT(
294 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
295 .getServiceSpecificError(),
296 Eq(static_cast<int32_t>(Status::ILLEGAL_ARGUMENT)));
297 }
298
TEST_F(VirtualCameraSessionTest,ConfigureWithDifferentAspectRatioFails)299 TEST_F(VirtualCameraSessionTest, ConfigureWithDifferentAspectRatioFails) {
300 StreamConfiguration streamConfiguration;
301 streamConfiguration.streams = {
302 createStream(kStreamId, kVgaWidth, kVgaHeight, PixelFormat::YCBCR_420_888),
303 createStream(kSecondStreamId, kVgaHeight, kVgaWidth,
304 PixelFormat::YCBCR_420_888)};
305
306 std::vector<HalStream> halStreams;
307
308 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
309 EXPECT_THAT(
310 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
311 .getServiceSpecificError(),
312 Eq(static_cast<int32_t>(Status::ILLEGAL_ARGUMENT)));
313 }
314
315 class VirtualCameraSessionInputChoiceTest : public VirtualCameraSessionTestBase {
316 public:
createSession(const std::vector<SupportedStreamConfiguration> & supportedInputConfigs)317 std::shared_ptr<VirtualCameraSession> createSession(
318 const std::vector<SupportedStreamConfiguration>& supportedInputConfigs) {
319 mVirtualCameraDevice = ndk::SharedRefBase::make<VirtualCameraDevice>(
320 kCameraId,
321 VirtualCameraConfiguration{
322 .supportedStreamConfigs = supportedInputConfigs,
323 .virtualCameraCallback = mMockVirtualCameraClientCallback,
324 .sensorOrientation = SensorOrientation::ORIENTATION_0,
325 .lensFacing = LensFacing::FRONT},
326 kDefaultDeviceId);
327 return ndk::SharedRefBase::make<VirtualCameraSession>(
328 mVirtualCameraDevice, mMockCameraDeviceCallback,
329 mMockVirtualCameraClientCallback);
330 }
331
332 protected:
333 std::shared_ptr<VirtualCameraDevice> mVirtualCameraDevice;
334 };
335
TEST_F(VirtualCameraSessionInputChoiceTest,configureChoosesCorrectInputStreamForDownsampledOutput)336 TEST_F(VirtualCameraSessionInputChoiceTest,
337 configureChoosesCorrectInputStreamForDownsampledOutput) {
338 // Create camera configured to support SVGA YUV input and RGB QVGA input.
339 auto virtualCameraSession = createSession(
340 {SupportedStreamConfiguration{.width = kSvgaWidth,
341 .height = kSvgaHeight,
342 .pixelFormat = Format::YUV_420_888,
343 .maxFps = kMaxFps},
344 SupportedStreamConfiguration{.width = kQvgaWidth,
345 .height = kQvgaHeight,
346 .pixelFormat = Format::RGBA_8888,
347 .maxFps = kMaxFps}});
348
349 // Configure VGA stream. Expect SVGA input to be chosen to downscale from.
350 StreamConfiguration streamConfiguration;
351 streamConfiguration.streams = {createStream(
352 kStreamId, kVgaWidth, kVgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
353 std::vector<HalStream> halStreams;
354
355 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
356 EXPECT_CALL(*mMockVirtualCameraClientCallback,
357 onStreamConfigured(kStreamId, _, kSvgaWidth, kSvgaHeight,
358 Format::YUV_420_888));
359 EXPECT_TRUE(
360 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
361 .isOk());
362 }
363
TEST_F(VirtualCameraSessionInputChoiceTest,configureChoosesCorrectInputStreamForMatchingResolution)364 TEST_F(VirtualCameraSessionInputChoiceTest,
365 configureChoosesCorrectInputStreamForMatchingResolution) {
366 // Create camera configured to support SVGA YUV input and RGB QVGA input.
367 auto virtualCameraSession = createSession(
368 {SupportedStreamConfiguration{.width = kSvgaWidth,
369 .height = kSvgaHeight,
370 .pixelFormat = Format::YUV_420_888,
371 .maxFps = kMaxFps},
372 SupportedStreamConfiguration{.width = kQvgaWidth,
373 .height = kQvgaHeight,
374 .pixelFormat = Format::RGBA_8888,
375 .maxFps = kMaxFps}});
376
377 // Configure VGA stream. Expect SVGA input to be chosen to downscale from.
378 StreamConfiguration streamConfiguration;
379 streamConfiguration.streams = {createStream(
380 kStreamId, kQvgaWidth, kQvgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
381 std::vector<HalStream> halStreams;
382
383 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
384 EXPECT_CALL(*mMockVirtualCameraClientCallback,
385 onStreamConfigured(kStreamId, _, kQvgaWidth, kQvgaHeight,
386 Format::RGBA_8888));
387 EXPECT_TRUE(
388 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
389 .isOk());
390 }
391
TEST_F(VirtualCameraSessionInputChoiceTest,reconfigureSwitchesInputStream)392 TEST_F(VirtualCameraSessionInputChoiceTest, reconfigureSwitchesInputStream) {
393 // Create camera configured to support SVGA YUV input and RGB QVGA input.
394 auto virtualCameraSession = createSession(
395 {SupportedStreamConfiguration{.width = kSvgaWidth,
396 .height = kSvgaHeight,
397 .pixelFormat = Format::YUV_420_888,
398 .maxFps = kMaxFps},
399 SupportedStreamConfiguration{.width = kQvgaWidth,
400 .height = kQvgaHeight,
401 .pixelFormat = Format::RGBA_8888,
402 .maxFps = kMaxFps}});
403
404 // First configure QVGA stream.
405 StreamConfiguration streamConfiguration;
406 streamConfiguration.streams = {createStream(
407 kStreamId, kQvgaWidth, kQvgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
408 std::vector<HalStream> halStreams;
409
410 // Expect QVGA input configuragion to be chosen.
411 EXPECT_CALL(*mMockVirtualCameraClientCallback,
412 onStreamConfigured(kStreamId, _, kQvgaWidth, kQvgaHeight,
413 Format::RGBA_8888));
414 EXPECT_TRUE(
415 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
416 .isOk());
417
418 // Reconfigure with additional VGA stream.
419 streamConfiguration.streams.push_back(
420 createStream(kStreamId + 1, kVgaWidth, kVgaHeight,
421 PixelFormat::IMPLEMENTATION_DEFINED));
422
423 // Expect original surface to be discarded.
424 EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamClosed(kStreamId));
425
426 // Expect SVGA input configuragion to be chosen.
427 EXPECT_CALL(*mMockVirtualCameraClientCallback,
428 onStreamConfigured(kStreamId + 1, _, kSvgaWidth, kSvgaHeight,
429 Format::YUV_420_888));
430 EXPECT_TRUE(
431 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
432 .isOk());
433 }
434
TEST_F(VirtualCameraSessionInputChoiceTest,reconfigureKeepsInputStreamIfUnchanged)435 TEST_F(VirtualCameraSessionInputChoiceTest,
436 reconfigureKeepsInputStreamIfUnchanged) {
437 // Create camera configured to support SVGA YUV input and RGB QVGA input.
438 auto virtualCameraSession = createSession(
439 {SupportedStreamConfiguration{.width = kSvgaWidth,
440 .height = kSvgaHeight,
441 .pixelFormat = Format::YUV_420_888,
442 .maxFps = kMaxFps},
443 SupportedStreamConfiguration{.width = kQvgaWidth,
444 .height = kQvgaHeight,
445 .pixelFormat = Format::RGBA_8888,
446 .maxFps = kMaxFps}});
447
448 // First configure SVGA stream.
449 StreamConfiguration streamConfiguration;
450 streamConfiguration.streams = {createStream(
451 kStreamId, kSvgaWidth, kSvgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
452 std::vector<HalStream> halStreams;
453
454 // Expect SVGA input configuragion to be chosen.
455 EXPECT_CALL(*mMockVirtualCameraClientCallback,
456 onStreamConfigured(kStreamId, _, kSvgaWidth, kSvgaHeight,
457 Format::YUV_420_888));
458 EXPECT_TRUE(
459 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
460 .isOk());
461
462 // Reconfigure with VGA + QVA stream. Because we only allow downscaling,
463 // this will be matched to SVGA input resolution.
464 streamConfiguration.streams = {
465 createStream(kStreamId + 1, kVgaWidth, kVgaHeight,
466 PixelFormat::IMPLEMENTATION_DEFINED),
467 createStream(kStreamId + 2, kVgaWidth, kVgaHeight,
468 PixelFormat::IMPLEMENTATION_DEFINED)};
469
470 // Expect the onStreamConfigured callback not to be invoked, since the
471 // original Surface is still best fit for current output streams.
472 EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamConfigured).Times(0);
473 EXPECT_TRUE(
474 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
475 .isOk());
476 }
477
478 } // namespace
479 } // namespace virtualcamera
480 } // namespace companion
481 } // namespace android
482