1 /*
2 * Copyright (c) 2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "output/metadata_output.h"
17
18 #include <set>
19
20 #include "camera_util.h"
21 #include "camera_log.h"
22 #include "input/camera_input.h"
23 #include "session/capture_session.h"
24
25 namespace OHOS {
26 namespace CameraStandard {
MetadataFaceObject(double timestamp,Rect rect)27 MetadataFaceObject::MetadataFaceObject(double timestamp, Rect rect)
28 : MetadataObject(MetadataObjectType::FACE, timestamp, rect)
29 {}
30
MetadataObject(MetadataObjectType type,double timestamp,Rect rect)31 MetadataObject::MetadataObject(MetadataObjectType type, double timestamp, Rect rect)
32 : type_(type), timestamp_(timestamp), box_(rect)
33 {}
34
GetType()35 MetadataObjectType MetadataObject::GetType()
36 {
37 return type_;
38 }
GetTimestamp()39 double MetadataObject::GetTimestamp()
40 {
41 return timestamp_;
42 }
GetBoundingBox()43 Rect MetadataObject::GetBoundingBox()
44 {
45 return box_;
46 }
47
MetadataOutput(sptr<IConsumerSurface> surface,sptr<IStreamMetadata> & streamMetadata)48 MetadataOutput::MetadataOutput(sptr<IConsumerSurface> surface, sptr<IStreamMetadata> &streamMetadata)
49 : CaptureOutput(CAPTURE_OUTPUT_TYPE_METADATA, StreamType::METADATA, streamMetadata)
50 {
51 surface_ = surface;
52 }
53
~MetadataOutput()54 MetadataOutput::~MetadataOutput()
55 {
56 if (surface_) {
57 SurfaceError ret = surface_->UnregisterConsumerListener();
58 if (ret != SURFACE_ERROR_OK) {
59 MEDIA_ERR_LOG("Failed to unregister surface consumer listener");
60 }
61 surface_ = nullptr;
62 }
63 }
64
GetAppObjectCallback()65 std::shared_ptr<MetadataObjectCallback> MetadataOutput::GetAppObjectCallback()
66 {
67 MEDIA_DEBUG_LOG("CameraDeviceServiceCallback::GetAppObjectCallback");
68 std::lock_guard<std::mutex> lock(outputCallbackMutex_);
69 return appObjectCallback_;
70 }
71
GetAppStateCallback()72 std::shared_ptr<MetadataStateCallback> MetadataOutput::GetAppStateCallback()
73 {
74 MEDIA_DEBUG_LOG("CameraDeviceServiceCallback::GetAppStateCallback");
75 std::lock_guard<std::mutex> lock(outputCallbackMutex_);
76 return appStateCallback_;
77 }
78
GetSupportedMetadataObjectTypes()79 std::vector<MetadataObjectType> MetadataOutput::GetSupportedMetadataObjectTypes()
80 {
81 CaptureSession* captureSession = GetSession();
82 if ((captureSession == nullptr) || (captureSession->inputDevice_ == nullptr)) {
83 return {};
84 }
85 sptr<CameraDevice> cameraObj = captureSession->inputDevice_->GetCameraDeviceInfo();
86 std::shared_ptr<Camera::CameraMetadata> metadata = cameraObj->GetMetadata();
87 camera_metadata_item_t item;
88 int ret = Camera::FindCameraMetadataItem(metadata->get(), OHOS_STATISTICS_FACE_DETECT_MODE, &item);
89 if (ret) {
90 return {};
91 }
92 std::vector<MetadataObjectType> objectTypes;
93 for (size_t index = 0; index < item.count; index++) {
94 if (item.data.u8[index] == OHOS_CAMERA_FACE_DETECT_MODE_SIMPLE) {
95 objectTypes.emplace_back(MetadataObjectType::FACE);
96 }
97 }
98 return objectTypes;
99 }
100
SetCapturingMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)101 void MetadataOutput::SetCapturingMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)
102 {
103 CaptureSession* captureSession = GetSession();
104 if ((captureSession == nullptr) || (captureSession->inputDevice_ == nullptr)) {
105 return;
106 }
107 std::set<camera_face_detect_mode_t> objectTypes;
108 for (const auto &type : metadataObjectTypes) {
109 if (type == MetadataObjectType::FACE) {
110 objectTypes.insert(OHOS_CAMERA_FACE_DETECT_MODE_SIMPLE);
111 }
112 }
113 if (objectTypes.empty()) {
114 objectTypes.insert(OHOS_CAMERA_FACE_DETECT_MODE_OFF);
115 }
116
117 captureSession->SetCaptureMetadataObjectTypes(objectTypes);
118 }
119
SetCallback(std::shared_ptr<MetadataObjectCallback> metadataObjectCallback)120 void MetadataOutput::SetCallback(std::shared_ptr<MetadataObjectCallback> metadataObjectCallback)
121 {
122 std::lock_guard<std::mutex> lock(outputCallbackMutex_);
123 appObjectCallback_ = metadataObjectCallback;
124 }
125
SetCallback(std::shared_ptr<MetadataStateCallback> metadataStateCallback)126 void MetadataOutput::SetCallback(std::shared_ptr<MetadataStateCallback> metadataStateCallback)
127 {
128 std::lock_guard<std::mutex> lock(outputCallbackMutex_);
129 appStateCallback_ = metadataStateCallback;
130 }
131
Start()132 int32_t MetadataOutput::Start()
133 {
134 CaptureSession* captureSession = GetSession();
135 if (captureSession == nullptr || !captureSession->IsSessionCommited()) {
136 MEDIA_ERR_LOG("MetadataOutput Failed to Start!, session not config");
137 return CameraErrorCode::SESSION_NOT_CONFIG;
138 }
139 if (GetStream() == nullptr) {
140 MEDIA_ERR_LOG("MetadataOutput Failed to Start!, GetStream is nullptr");
141 return CameraErrorCode::SERVICE_FATL_ERROR;
142 }
143 int32_t errCode = static_cast<IStreamMetadata *>(GetStream().GetRefPtr())->Start();
144 if (errCode != CAMERA_OK) {
145 MEDIA_ERR_LOG("Failed to Start MetadataOutput!, errCode: %{public}d", errCode);
146 }
147 return ServiceToCameraError(errCode);
148 }
149
Stop()150 int32_t MetadataOutput::Stop()
151 {
152 if (GetStream() == nullptr) {
153 MEDIA_ERR_LOG("MetadataOutput Failed to Stop!, GetStream is nullptr");
154 return CameraErrorCode::SERVICE_FATL_ERROR;
155 }
156 int32_t errCode = static_cast<IStreamMetadata *>(GetStream().GetRefPtr())->Stop();
157 if (errCode != CAMERA_OK) {
158 MEDIA_ERR_LOG("Failed to Stop MetadataOutput!, errCode: %{public}d", errCode);
159 }
160 return ServiceToCameraError(errCode);
161 }
162
Release()163 int32_t MetadataOutput::Release()
164 {
165 {
166 std::lock_guard<std::mutex> lock(outputCallbackMutex_);
167 appObjectCallback_ = nullptr;
168 appStateCallback_ = nullptr;
169 }
170 if (GetStream() == nullptr) {
171 MEDIA_ERR_LOG("MetadataOutput Failed to Release!, GetStream is nullptr");
172 return CameraErrorCode::SERVICE_FATL_ERROR;
173 }
174 int32_t errCode = static_cast<IStreamMetadata *>(GetStream().GetRefPtr())->Release();
175 if (errCode != CAMERA_OK) {
176 MEDIA_ERR_LOG("Failed to release MetadataOutput!, errCode: %{public}d", errCode);
177 }
178 if (surface_) {
179 SurfaceError ret = surface_->UnregisterConsumerListener();
180 if (ret != SURFACE_ERROR_OK) {
181 MEDIA_ERR_LOG("Failed to unregister surface consumer listener");
182 }
183 surface_ = nullptr;
184 }
185 CaptureOutput::Release();
186 return ServiceToCameraError(errCode);
187 }
188
MetadataObjectListener(sptr<MetadataOutput> metadata)189 MetadataObjectListener::MetadataObjectListener(sptr<MetadataOutput> metadata) : metadata_(metadata)
190 {}
191
ProcessFaceRectangles(int64_t timestamp,const camera_metadata_item_t & metadataItem,std::vector<sptr<MetadataObject>> & metaObjects)192 int32_t MetadataObjectListener::ProcessFaceRectangles(int64_t timestamp, const camera_metadata_item_t &metadataItem,
193 std::vector<sptr<MetadataObject>> &metaObjects)
194 {
195 constexpr int32_t rectangleUnitLen = 4;
196
197 if (metadataItem.count % rectangleUnitLen) {
198 MEDIA_ERR_LOG("Metadata item: %{public}d count: %{public}d is invalid", metadataItem.item, metadataItem.count);
199 return ERROR_UNKNOWN;
200 }
201 metaObjects.reserve(metadataItem.count / rectangleUnitLen);
202 float* start = metadataItem.data.f;
203 float* end = metadataItem.data.f + metadataItem.count;
204 for (; start < end; start += rectangleUnitLen) {
205 sptr<MetadataObject> metadataObject = new(std::nothrow) MetadataFaceObject(timestamp,
206 (Rect) {start[0], start[1], start[2], start[3]});
207 if (!metadataObject) {
208 MEDIA_ERR_LOG("Failed to allocate MetadataFaceObject");
209 return ERROR_INSUFFICIENT_RESOURCES;
210 }
211 metaObjects.emplace_back(metadataObject);
212 }
213 return CAMERA_OK;
214 }
215
ProcessMetadataBuffer(void * buffer,int64_t timestamp)216 int32_t MetadataObjectListener::ProcessMetadataBuffer(void* buffer, int64_t timestamp)
217 {
218 if (!buffer) {
219 MEDIA_ERR_LOG("Buffer is null");
220 return ERROR_UNKNOWN;
221 }
222 common_metadata_header_t* metadata = (common_metadata_header_t *)buffer;
223 uint32_t itemCount = Camera::GetCameraMetadataItemCount(metadata);
224 camera_metadata_item_t metadataItem;
225 std::vector<sptr<MetadataObject>> metaObjects;
226 for (uint32_t i = 0; i < itemCount; i++) {
227 int32_t ret = Camera::GetCameraMetadataItem(metadata, i, &metadataItem);
228 if (ret) {
229 MEDIA_ERR_LOG("Failed to get metadata item at index: %{public}d, with return code: %{public}d", i, ret);
230 return ERROR_UNKNOWN;
231 }
232 if (metadataItem.item == OHOS_STATISTICS_FACE_RECTANGLES) {
233 ret = ProcessFaceRectangles(timestamp, metadataItem, metaObjects);
234 if (ret) {
235 MEDIA_ERR_LOG("Failed to process face rectangles");
236 return ret;
237 }
238 }
239 }
240 std::shared_ptr<MetadataObjectCallback> appObjectCallback = metadata_->GetAppObjectCallback();
241 if (!metaObjects.empty() && appObjectCallback) {
242 appObjectCallback->OnMetadataObjectsAvailable(metaObjects);
243 }
244 return CAMERA_OK;
245 }
246
OnBufferAvailable()247 void MetadataObjectListener::OnBufferAvailable()
248 {
249 if (!metadata_) {
250 MEDIA_ERR_LOG("Metadata is null");
251 return;
252 }
253 sptr<IConsumerSurface> surface = metadata_->surface_;
254 if (!surface) {
255 MEDIA_ERR_LOG("Metadata surface is null");
256 return;
257 }
258 int32_t fence = -1;
259 int64_t timestamp;
260 OHOS::Rect damage;
261 sptr<SurfaceBuffer> buffer = nullptr;
262 SurfaceError surfaceRet = surface->AcquireBuffer(buffer, fence, timestamp, damage);
263 if (surfaceRet != SURFACE_ERROR_OK) {
264 MEDIA_ERR_LOG("Failed to acquire surface buffer");
265 return;
266 }
267 int32_t ret = ProcessMetadataBuffer(buffer->GetVirAddr(), timestamp);
268 if (ret) {
269 std::shared_ptr<MetadataStateCallback> appStateCallback = metadata_->GetAppStateCallback();
270 if (appStateCallback) {
271 appStateCallback->OnError(ret);
272 }
273 }
274
275 surface->ReleaseBuffer(buffer, -1);
276 }
277 } // CameraStandard
278 } // OHOS
279