1 /*
2 * Copyright 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <cstdint>
18 #include <memory>
19
20 #include "VirtualCameraDevice.h"
21 #include "VirtualCameraSession.h"
22 #include "aidl/android/companion/virtualcamera/BnVirtualCameraCallback.h"
23 #include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
24 #include "aidl/android/companion/virtualcamera/VirtualCameraConfiguration.h"
25 #include "aidl/android/hardware/camera/common/Status.h"
26 #include "aidl/android/hardware/camera/device/BnCameraDeviceCallback.h"
27 #include "aidl/android/hardware/camera/device/StreamConfiguration.h"
28 #include "aidl/android/hardware/graphics/common/PixelFormat.h"
29 #include "android/binder_auto_utils.h"
30 #include "android/binder_interface_utils.h"
31 #include "gmock/gmock.h"
32 #include "gtest/gtest.h"
33 #include "util/MetadataUtil.h"
34
35 namespace android {
36 namespace companion {
37 namespace virtualcamera {
38 namespace {
39
40 constexpr char kCameraId[] = "42";
41 constexpr int kQvgaWidth = 320;
42 constexpr int kQvgaHeight = 240;
43 constexpr int kVgaWidth = 640;
44 constexpr int kVgaHeight = 480;
45 constexpr int kSvgaWidth = 800;
46 constexpr int kSvgaHeight = 600;
47 constexpr int kMaxFps = 30;
48 constexpr int kStreamId = 0;
49 constexpr int kSecondStreamId = 1;
50 constexpr int kDefaultDeviceId = 0;
51
52 using ::aidl::android::companion::virtualcamera::BnVirtualCameraCallback;
53 using ::aidl::android::companion::virtualcamera::Format;
54 using ::aidl::android::companion::virtualcamera::LensFacing;
55 using ::aidl::android::companion::virtualcamera::SensorOrientation;
56 using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
57 using ::aidl::android::companion::virtualcamera::VirtualCameraConfiguration;
58 using ::aidl::android::hardware::camera::common::Status;
59 using ::aidl::android::hardware::camera::device::BnCameraDeviceCallback;
60 using ::aidl::android::hardware::camera::device::BufferRequest;
61 using ::aidl::android::hardware::camera::device::BufferRequestStatus;
62 using ::aidl::android::hardware::camera::device::CaptureRequest;
63 using ::aidl::android::hardware::camera::device::CaptureResult;
64 using ::aidl::android::hardware::camera::device::HalStream;
65 using ::aidl::android::hardware::camera::device::NotifyMsg;
66 using ::aidl::android::hardware::camera::device::Stream;
67 using ::aidl::android::hardware::camera::device::StreamBuffer;
68 using ::aidl::android::hardware::camera::device::StreamBufferRet;
69 using ::aidl::android::hardware::camera::device::StreamConfiguration;
70 using ::aidl::android::hardware::graphics::common::PixelFormat;
71 using ::aidl::android::view::Surface;
72 using ::testing::_;
73 using ::testing::ElementsAre;
74 using ::testing::Eq;
75 using ::testing::Return;
76 using ::testing::SizeIs;
77
createStream(int streamId,int width,int height,PixelFormat format)78 Stream createStream(int streamId, int width, int height, PixelFormat format) {
79 Stream s;
80 s.id = streamId;
81 s.width = width;
82 s.height = height;
83 s.format = format;
84 return s;
85 }
86
87 class MockCameraDeviceCallback : public BnCameraDeviceCallback {
88 public:
89 MOCK_METHOD(ndk::ScopedAStatus, notify, (const std::vector<NotifyMsg>&),
90 (override));
91 MOCK_METHOD(ndk::ScopedAStatus, processCaptureResult,
92 (const std::vector<CaptureResult>&), (override));
93 MOCK_METHOD(ndk::ScopedAStatus, requestStreamBuffers,
94 (const std::vector<BufferRequest>&, std::vector<StreamBufferRet>*,
95 BufferRequestStatus*),
96 (override));
97 MOCK_METHOD(ndk::ScopedAStatus, returnStreamBuffers,
98 (const std::vector<StreamBuffer>&), (override));
99 };
100
101 class MockVirtualCameraCallback : public BnVirtualCameraCallback {
102 public:
103 MOCK_METHOD(ndk::ScopedAStatus, onStreamConfigured,
104 (int, const Surface&, int32_t, int32_t, Format), (override));
105 MOCK_METHOD(ndk::ScopedAStatus, onProcessCaptureRequest, (int, int),
106 (override));
107 MOCK_METHOD(ndk::ScopedAStatus, onStreamClosed, (int), (override));
108 };
109
110 class VirtualCameraSessionTestBase : public ::testing::Test {
111 public:
SetUp()112 virtual void SetUp() override {
113 mMockCameraDeviceCallback =
114 ndk::SharedRefBase::make<MockCameraDeviceCallback>();
115 mMockVirtualCameraClientCallback =
116 ndk::SharedRefBase::make<MockVirtualCameraCallback>();
117
118 // Explicitly defining default actions below to prevent gmock from
119 // default-constructing ndk::ScopedAStatus, because default-constructed
120 // status wraps nullptr AStatus and causes crash when attempting to print
121 // it in gtest report.
122 ON_CALL(*mMockCameraDeviceCallback, notify)
123 .WillByDefault(ndk::ScopedAStatus::ok);
124 ON_CALL(*mMockCameraDeviceCallback, processCaptureResult)
125 .WillByDefault(ndk::ScopedAStatus::ok);
126 ON_CALL(*mMockCameraDeviceCallback, requestStreamBuffers)
127 .WillByDefault(ndk::ScopedAStatus::ok);
128 ON_CALL(*mMockCameraDeviceCallback, returnStreamBuffers)
129 .WillByDefault(ndk::ScopedAStatus::ok);
130
131 ON_CALL(*mMockVirtualCameraClientCallback, onStreamConfigured)
132 .WillByDefault(ndk::ScopedAStatus::ok);
133 ON_CALL(*mMockVirtualCameraClientCallback, onProcessCaptureRequest)
134 .WillByDefault(ndk::ScopedAStatus::ok);
135 ON_CALL(*mMockVirtualCameraClientCallback, onStreamClosed)
136 .WillByDefault(ndk::ScopedAStatus::ok);
137 }
138
139 protected:
140 std::shared_ptr<MockCameraDeviceCallback> mMockCameraDeviceCallback;
141 std::shared_ptr<MockVirtualCameraCallback> mMockVirtualCameraClientCallback;
142 };
143
144 class VirtualCameraSessionTest : public VirtualCameraSessionTestBase {
145 public:
SetUp()146 void SetUp() override {
147 VirtualCameraSessionTestBase::SetUp();
148
149 mVirtualCameraDevice = ndk::SharedRefBase::make<VirtualCameraDevice>(
150 kCameraId,
151 VirtualCameraConfiguration{
152 .supportedStreamConfigs = {SupportedStreamConfiguration{
153 .width = kVgaWidth,
154 .height = kVgaHeight,
155 .pixelFormat = Format::YUV_420_888,
156 .maxFps = kMaxFps},
157 SupportedStreamConfiguration{
158 .width = kSvgaWidth,
159 .height = kSvgaHeight,
160 .pixelFormat = Format::YUV_420_888,
161 .maxFps = kMaxFps}},
162 .virtualCameraCallback = mMockVirtualCameraClientCallback,
163 .sensorOrientation = SensorOrientation::ORIENTATION_0,
164 .lensFacing = LensFacing::FRONT},
165 kDefaultDeviceId);
166 mVirtualCameraSession = ndk::SharedRefBase::make<VirtualCameraSession>(
167 mVirtualCameraDevice, mMockCameraDeviceCallback,
168 mMockVirtualCameraClientCallback);
169 }
170
171 protected:
172 std::shared_ptr<VirtualCameraDevice> mVirtualCameraDevice;
173 std::shared_ptr<VirtualCameraSession> mVirtualCameraSession;
174 };
175
TEST_F(VirtualCameraSessionTest,ConfigureTriggersClientConfigureCallback)176 TEST_F(VirtualCameraSessionTest, ConfigureTriggersClientConfigureCallback) {
177 PixelFormat format = PixelFormat::YCBCR_420_888;
178 StreamConfiguration streamConfiguration;
179 streamConfiguration.streams = {
180 createStream(kStreamId, kVgaWidth, kVgaHeight, format),
181 createStream(kSecondStreamId, kSvgaWidth, kSvgaHeight, format)};
182 std::vector<HalStream> halStreams;
183
184 // Expect highest resolution to be picked for the client input.
185 EXPECT_CALL(*mMockVirtualCameraClientCallback,
186 onStreamConfigured(kStreamId, _, kSvgaWidth, kSvgaHeight,
187 Format::YUV_420_888));
188
189 ASSERT_TRUE(
190 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
191 .isOk());
192
193 EXPECT_THAT(halStreams, SizeIs(streamConfiguration.streams.size()));
194 EXPECT_THAT(mVirtualCameraSession->getStreamIds(),
195 ElementsAre(kStreamId, kSecondStreamId));
196 }
197
TEST_F(VirtualCameraSessionTest,SecondConfigureDropsUnreferencedStreams)198 TEST_F(VirtualCameraSessionTest, SecondConfigureDropsUnreferencedStreams) {
199 PixelFormat format = PixelFormat::YCBCR_420_888;
200 StreamConfiguration streamConfiguration;
201 std::vector<HalStream> halStreams;
202
203 streamConfiguration.streams = {createStream(0, kVgaWidth, kVgaHeight, format),
204 createStream(1, kVgaWidth, kVgaHeight, format),
205 createStream(2, kVgaWidth, kVgaHeight, format)};
206 ASSERT_TRUE(
207 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
208 .isOk());
209
210 EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0, 1, 2));
211
212 streamConfiguration.streams = {createStream(0, kVgaWidth, kVgaHeight, format),
213 createStream(2, kVgaWidth, kVgaHeight, format),
214 createStream(3, kVgaWidth, kVgaHeight, format)};
215 ASSERT_TRUE(
216 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
217 .isOk());
218
219 EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0, 2, 3));
220 }
221
TEST_F(VirtualCameraSessionTest,CloseTriggersClientTerminateCallback)222 TEST_F(VirtualCameraSessionTest, CloseTriggersClientTerminateCallback) {
223 EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamClosed(kStreamId))
224 .WillOnce(Return(ndk::ScopedAStatus::ok()));
225
226 ASSERT_TRUE(mVirtualCameraSession->close().isOk());
227 }
228
TEST_F(VirtualCameraSessionTest,FlushBeforeConfigure)229 TEST_F(VirtualCameraSessionTest, FlushBeforeConfigure) {
230 // Flush request coming before the configure request finished
231 // (so potentially the thread is not yet running) should be
232 // gracefully handled.
233
234 EXPECT_TRUE(mVirtualCameraSession->flush().isOk());
235 }
236
TEST_F(VirtualCameraSessionTest,onProcessCaptureRequestTriggersClientCallback)237 TEST_F(VirtualCameraSessionTest, onProcessCaptureRequestTriggersClientCallback) {
238 StreamConfiguration streamConfiguration;
239 streamConfiguration.streams = {createStream(kStreamId, kVgaWidth, kVgaHeight,
240 PixelFormat::YCBCR_420_888)};
241 std::vector<CaptureRequest> requests(1);
242 requests[0].frameNumber = 42;
243 requests[0].settings = *(
244 MetadataBuilder().setControlAfMode(ANDROID_CONTROL_AF_MODE_AUTO).build());
245
246 std::vector<HalStream> halStreams;
247 ASSERT_TRUE(
248 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
249 .isOk());
250
251 EXPECT_CALL(*mMockVirtualCameraClientCallback,
252 onProcessCaptureRequest(kStreamId, requests[0].frameNumber))
253 .WillOnce(Return(ndk::ScopedAStatus::ok()));
254 int32_t aidlReturn = 0;
255 ASSERT_TRUE(mVirtualCameraSession
256 ->processCaptureRequest(requests, /*in_cachesToRemove=*/{},
257 &aidlReturn)
258 .isOk());
259 EXPECT_THAT(aidlReturn, Eq(requests.size()));
260 }
261
TEST_F(VirtualCameraSessionTest,configureAfterCameraRelease)262 TEST_F(VirtualCameraSessionTest, configureAfterCameraRelease) {
263 StreamConfiguration streamConfiguration;
264 streamConfiguration.streams = {createStream(kStreamId, kVgaWidth, kVgaHeight,
265 PixelFormat::YCBCR_420_888)};
266 std::vector<HalStream> halStreams;
267
268 // Release virtual camera.
269 mVirtualCameraDevice.reset();
270
271 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
272 EXPECT_THAT(
273 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
274 .getServiceSpecificError(),
275 Eq(static_cast<int32_t>(Status::CAMERA_DISCONNECTED)));
276 }
277
TEST_F(VirtualCameraSessionTest,ConfigureWithEmptyStreams)278 TEST_F(VirtualCameraSessionTest, ConfigureWithEmptyStreams) {
279 StreamConfiguration streamConfiguration;
280 std::vector<HalStream> halStreams;
281
282 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
283 EXPECT_THAT(
284 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
285 .getServiceSpecificError(),
286 Eq(static_cast<int32_t>(Status::ILLEGAL_ARGUMENT)));
287 }
288
TEST_F(VirtualCameraSessionTest,ConfigureWithDifferentAspectRatioFails)289 TEST_F(VirtualCameraSessionTest, ConfigureWithDifferentAspectRatioFails) {
290 StreamConfiguration streamConfiguration;
291 streamConfiguration.streams = {
292 createStream(kStreamId, kVgaWidth, kVgaHeight, PixelFormat::YCBCR_420_888),
293 createStream(kSecondStreamId, kVgaHeight, kVgaWidth,
294 PixelFormat::YCBCR_420_888)};
295
296 std::vector<HalStream> halStreams;
297
298 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
299 EXPECT_THAT(
300 mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
301 .getServiceSpecificError(),
302 Eq(static_cast<int32_t>(Status::ILLEGAL_ARGUMENT)));
303 }
304
305 class VirtualCameraSessionInputChoiceTest : public VirtualCameraSessionTestBase {
306 public:
createSession(const std::vector<SupportedStreamConfiguration> & supportedInputConfigs)307 std::shared_ptr<VirtualCameraSession> createSession(
308 const std::vector<SupportedStreamConfiguration>& supportedInputConfigs) {
309 mVirtualCameraDevice = ndk::SharedRefBase::make<VirtualCameraDevice>(
310 kCameraId,
311 VirtualCameraConfiguration{
312 .supportedStreamConfigs = supportedInputConfigs,
313 .virtualCameraCallback = mMockVirtualCameraClientCallback,
314 .sensorOrientation = SensorOrientation::ORIENTATION_0,
315 .lensFacing = LensFacing::FRONT},
316 kDefaultDeviceId);
317 return ndk::SharedRefBase::make<VirtualCameraSession>(
318 mVirtualCameraDevice, mMockCameraDeviceCallback,
319 mMockVirtualCameraClientCallback);
320 }
321
322 protected:
323 std::shared_ptr<VirtualCameraDevice> mVirtualCameraDevice;
324 };
325
TEST_F(VirtualCameraSessionInputChoiceTest,configureChoosesCorrectInputStreamForDownsampledOutput)326 TEST_F(VirtualCameraSessionInputChoiceTest,
327 configureChoosesCorrectInputStreamForDownsampledOutput) {
328 // Create camera configured to support SVGA YUV input and RGB QVGA input.
329 auto virtualCameraSession = createSession(
330 {SupportedStreamConfiguration{.width = kSvgaWidth,
331 .height = kSvgaHeight,
332 .pixelFormat = Format::YUV_420_888,
333 .maxFps = kMaxFps},
334 SupportedStreamConfiguration{.width = kQvgaWidth,
335 .height = kQvgaHeight,
336 .pixelFormat = Format::RGBA_8888,
337 .maxFps = kMaxFps}});
338
339 // Configure VGA stream. Expect SVGA input to be chosen to downscale from.
340 StreamConfiguration streamConfiguration;
341 streamConfiguration.streams = {createStream(
342 kStreamId, kVgaWidth, kVgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
343 std::vector<HalStream> halStreams;
344
345 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
346 EXPECT_CALL(*mMockVirtualCameraClientCallback,
347 onStreamConfigured(kStreamId, _, kSvgaWidth, kSvgaHeight,
348 Format::YUV_420_888));
349 EXPECT_TRUE(
350 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
351 .isOk());
352 }
353
TEST_F(VirtualCameraSessionInputChoiceTest,configureChoosesCorrectInputStreamForMatchingResolution)354 TEST_F(VirtualCameraSessionInputChoiceTest,
355 configureChoosesCorrectInputStreamForMatchingResolution) {
356 // Create camera configured to support SVGA YUV input and RGB QVGA input.
357 auto virtualCameraSession = createSession(
358 {SupportedStreamConfiguration{.width = kSvgaWidth,
359 .height = kSvgaHeight,
360 .pixelFormat = Format::YUV_420_888,
361 .maxFps = kMaxFps},
362 SupportedStreamConfiguration{.width = kQvgaWidth,
363 .height = kQvgaHeight,
364 .pixelFormat = Format::RGBA_8888,
365 .maxFps = kMaxFps}});
366
367 // Configure VGA stream. Expect SVGA input to be chosen to downscale from.
368 StreamConfiguration streamConfiguration;
369 streamConfiguration.streams = {createStream(
370 kStreamId, kQvgaWidth, kQvgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
371 std::vector<HalStream> halStreams;
372
373 // Expect configuration attempt returns CAMERA_DISCONNECTED service specific code.
374 EXPECT_CALL(*mMockVirtualCameraClientCallback,
375 onStreamConfigured(kStreamId, _, kQvgaWidth, kQvgaHeight,
376 Format::RGBA_8888));
377 EXPECT_TRUE(
378 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
379 .isOk());
380 }
381
TEST_F(VirtualCameraSessionInputChoiceTest,reconfigureSwitchesInputStream)382 TEST_F(VirtualCameraSessionInputChoiceTest, reconfigureSwitchesInputStream) {
383 // Create camera configured to support SVGA YUV input and RGB QVGA input.
384 auto virtualCameraSession = createSession(
385 {SupportedStreamConfiguration{.width = kSvgaWidth,
386 .height = kSvgaHeight,
387 .pixelFormat = Format::YUV_420_888,
388 .maxFps = kMaxFps},
389 SupportedStreamConfiguration{.width = kQvgaWidth,
390 .height = kQvgaHeight,
391 .pixelFormat = Format::RGBA_8888,
392 .maxFps = kMaxFps}});
393
394 // First configure QVGA stream.
395 StreamConfiguration streamConfiguration;
396 streamConfiguration.streams = {createStream(
397 kStreamId, kQvgaWidth, kQvgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
398 std::vector<HalStream> halStreams;
399
400 // Expect QVGA input configuragion to be chosen.
401 EXPECT_CALL(*mMockVirtualCameraClientCallback,
402 onStreamConfigured(kStreamId, _, kQvgaWidth, kQvgaHeight,
403 Format::RGBA_8888));
404 EXPECT_TRUE(
405 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
406 .isOk());
407
408 // Reconfigure with additional VGA stream.
409 streamConfiguration.streams.push_back(
410 createStream(kStreamId + 1, kVgaWidth, kVgaHeight,
411 PixelFormat::IMPLEMENTATION_DEFINED));
412
413 // Expect original surface to be discarded.
414 EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamClosed(kStreamId));
415
416 // Expect SVGA input configuragion to be chosen.
417 EXPECT_CALL(*mMockVirtualCameraClientCallback,
418 onStreamConfigured(kStreamId + 1, _, kSvgaWidth, kSvgaHeight,
419 Format::YUV_420_888));
420 EXPECT_TRUE(
421 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
422 .isOk());
423 }
424
TEST_F(VirtualCameraSessionInputChoiceTest,reconfigureKeepsInputStreamIfUnchanged)425 TEST_F(VirtualCameraSessionInputChoiceTest,
426 reconfigureKeepsInputStreamIfUnchanged) {
427 // Create camera configured to support SVGA YUV input and RGB QVGA input.
428 auto virtualCameraSession = createSession(
429 {SupportedStreamConfiguration{.width = kSvgaWidth,
430 .height = kSvgaHeight,
431 .pixelFormat = Format::YUV_420_888,
432 .maxFps = kMaxFps},
433 SupportedStreamConfiguration{.width = kQvgaWidth,
434 .height = kQvgaHeight,
435 .pixelFormat = Format::RGBA_8888,
436 .maxFps = kMaxFps}});
437
438 // First configure SVGA stream.
439 StreamConfiguration streamConfiguration;
440 streamConfiguration.streams = {createStream(
441 kStreamId, kSvgaWidth, kSvgaHeight, PixelFormat::IMPLEMENTATION_DEFINED)};
442 std::vector<HalStream> halStreams;
443
444 // Expect SVGA input configuragion to be chosen.
445 EXPECT_CALL(*mMockVirtualCameraClientCallback,
446 onStreamConfigured(kStreamId, _, kSvgaWidth, kSvgaHeight,
447 Format::YUV_420_888));
448 EXPECT_TRUE(
449 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
450 .isOk());
451
452 // Reconfigure with VGA + QVA stream. Because we only allow downscaling,
453 // this will be matched to SVGA input resolution.
454 streamConfiguration.streams = {
455 createStream(kStreamId + 1, kVgaWidth, kVgaHeight,
456 PixelFormat::IMPLEMENTATION_DEFINED),
457 createStream(kStreamId + 2, kVgaWidth, kVgaHeight,
458 PixelFormat::IMPLEMENTATION_DEFINED)};
459
460 // Expect the onStreamConfigured callback not to be invoked, since the
461 // original Surface is still best fit for current output streams.
462 EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamConfigured).Times(0);
463 EXPECT_TRUE(
464 virtualCameraSession->configureStreams(streamConfiguration, &halStreams)
465 .isOk());
466 }
467
468 } // namespace
469 } // namespace virtualcamera
470 } // namespace companion
471 } // namespace android
472