1 /*
2 * Copyright (c) 2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <cmath>
17 #include <memory>
18 #include <valarray>
19 #include "mc_camera_tracking_controller.h"
20 #include "mechbody_controller_service.h"
21 #include "mc_controller_ipc_death_listener.h"
22 #include "os_account_manager.h"
23 #include "capture_scene_const.h"
24 #include "mechbody_controller_utils.h"
25
26 namespace OHOS {
27 namespace MechBodyController {
28 namespace {
29 const std::string TAG = "McCameraTrackingController";
30
31 constexpr float DOUBLE = 2.0f;
32 constexpr int32_t NUM_1 = 1;
33 constexpr int32_t NUM_2 = 2;
34
35 constexpr int32_t CONFIDENCE_LEVEL_LOST_END = 100;
36 constexpr int32_t CONFIDENCE_LEVEL_LOW_END = 400;
37 constexpr int32_t CONFIDENCE_LEVEL_MIDDEL_END = 700;
38
39 constexpr int VALID_INCLINATION_ANGLE_THRESHOLD_COEFFICIENT = 3;
40 constexpr int32_t POSTURE_INTERVAL = 100000000; // 100ms
41 constexpr int32_t MOBILE_ROTATION_CHECK_UP_BEGIN = 330;
42 constexpr int32_t MOBILE_ROTATION_CHECK_UP_END = 30;
43 constexpr int32_t MOBILE_ROTATION_CHECK_RIGHT_BEGIN = 60;
44 constexpr int32_t MOBILE_ROTATION_CHECK_RIGHT_END = 120;
45 constexpr int32_t MOBILE_ROTATION_CHECK_DOWN_BEGIN = 150;
46 constexpr int32_t MOBILE_ROTATION_CHECK_DOWN_END = 210;
47 constexpr int32_t MOBILE_ROTATION_CHECK_LEFT_BEGIN = 240;
48 constexpr int32_t MOBILE_ROTATION_CHECK_LEFT_END = 300;
49
50 constexpr int32_t INVALID_USER_ID = -1;
51
52 constexpr int32_t REAR_MOUNTED_LENS = 1;
53
54 constexpr float LAYOUT_LEFT = 0.3f;
55 constexpr float LAYOUT_MIDDLE = 0.5f;
56 constexpr float LAYOUT_RIGHT = 0.7f;
57
58 constexpr int32_t DEGREE_CONSTANT_90 = 90;
59 constexpr int32_t DEGREE_CONSTANT_180 = 180;
60 constexpr int32_t DEGREE_CONSTANT_360 = 360;
61
62 constexpr float TRACKING_LOST_CHECK = 0.0001f;
63 static const std::set<int32_t> FOCUS_MODE_WHITELIST = { 1 };
64 static const std::set<int32_t> SESSION_MODE_WHITELIST = {
65 static_cast<int32_t>(CameraStandard::SceneMode::NORMAL), // 0
66 static_cast<int32_t>(CameraStandard::SceneMode::CAPTURE), // 1
67 static_cast<int32_t>(CameraStandard::SceneMode::VIDEO), // 2
68 static_cast<int32_t>(CameraStandard::SceneMode::PORTRAIT), // 3
69 };
70
71 const std::string SEND_CAMERA_INFO_TASK_NAME = "sendCameraInfoTaskName";
72 constexpr int32_t SEND_CAMERA_INFO_TASK_DELAY = 500;
73
74 constexpr float SHORT_CUT = 9.0f / 12.0f;
75 constexpr float OHOS_CAMERA_VIDEO_STABILIZATION_HIGH_CUT = 0.6f;
76 constexpr float OHOS_CAMERA_VIDEO_STABILIZATION_AUTO_CUT = 0.8f;
77 static const std::set<int32_t> VIDEO_STABILIZATION_WHITELIST = {
78 static_cast<int32_t>(CameraStandard::SceneMode::VIDEO), // 2
79 };
80 }
81
GetInstance()82 McCameraTrackingController& McCameraTrackingController::GetInstance()
83 {
84 static auto instance = new McCameraTrackingController();
85 return *instance;
86 }
87
McCameraTrackingController()88 McCameraTrackingController::McCameraTrackingController()
89 {
90 auto runner = AppExecFwk::EventRunner::Create("McCameraTrackingController");
91 eventHandler_ = std::make_shared<AppExecFwk::EventHandler>(runner);
92 }
93
Init()94 void McCameraTrackingController::Init()
95 {
96 HILOGI("start");
97 std::lock_guard<std::mutex> lock(cameraTrackingControllerInitMutex_);
98 RegisterTrackingListener();
99 RegisterSensorListener();
100 HILOGI("end");
101 }
102
UnInit()103 void McCameraTrackingController::UnInit()
104 {
105 HILOGI("start");
106 std::lock_guard<std::mutex> lock(cameraTrackingControllerInitMutex_);
107 UnRegisterTrackingListener();
108 UnRegisterSensorListener();
109 HILOGI("end");
110 }
111
OnCaptureSessionConfiged(const CameraStandard::CaptureSessionInfo & captureSessionInfo)112 int32_t McCameraTrackingController::OnCaptureSessionConfiged(
113 const CameraStandard::CaptureSessionInfo& captureSessionInfo)
114 {
115 HILOGI("start");
116 if (currentCameraInfo_ == nullptr) {
117 currentCameraInfo_ = std::make_shared<CameraInfo>();
118 }
119 currentCameraInfo_->tokenId = captureSessionInfo.callerTokenId;
120 HILOGI("The app that is using the camera changes, app token id: %{public}s; ",
121 GetAnonymUint32(currentCameraInfo_->tokenId).c_str());
122 for (const auto &item: captureSessionInfo.outputInfos) {
123 if (item.type == CameraStandard::OutputType::VIDEO ||
124 item.type == CameraStandard::OutputType::PREVIEW) {
125 currentCameraInfo_->width = item.width;
126 currentCameraInfo_->height = item.height;
127 break;
128 }
129 }
130 currentCameraInfo_->sessionMode = captureSessionInfo.sessionMode;
131 currentCameraInfo_->isRecording =
132 captureSessionInfo.sessionMode == static_cast<int32_t>(CameraStandard::SceneMode::VIDEO);
133 currentCameraInfo_->cameraType =
134 captureSessionInfo.position == REAR_MOUNTED_LENS ? CameraType::BACK : CameraType::FRONT;
135 currentCameraInfo_->zoomFactor = captureSessionInfo.zoomInfo.zoomValue;
136 currentCameraInfo_->equivalentFocus = captureSessionInfo.zoomInfo.equivalentFocus;
137 currentCameraInfo_->videoStabilizationMode = captureSessionInfo.zoomInfo.videoStabilizationMode;
138 ComputeFov();
139 if (currentCameraInfo_->tokenId != 0) {
140 if (!appSettings.empty() && appSettings.find(currentCameraInfo_->tokenId) != appSettings.end()) {
141 HILOGI("The application settings trackingEnable is not empty.");
142 SetTrackingEnabled(currentCameraInfo_->tokenId,
143 appSettings[currentCameraInfo_->tokenId]->isTrackingEnabled);
144 }
145 return UpdateMotionManagers();
146 }
147 HILOGI("end");
148 return ERR_OK;
149 }
150
OnZoomInfoChange(int32_t sessionid,const CameraStandard::ZoomInfo & zoomInfo)151 int32_t McCameraTrackingController::OnZoomInfoChange(int32_t sessionid, const CameraStandard::ZoomInfo& zoomInfo)
152 {
153 HILOGI("start");
154 if (currentCameraInfo_ == nullptr) {
155 HILOGW("currentCameraInfo_ is nullptr; ");
156 return CAMERA_INFO_IS_EMPTY;
157 }
158 currentCameraInfo_->zoomFactor = zoomInfo.zoomValue;
159 currentCameraInfo_->equivalentFocus = zoomInfo.equivalentFocus;
160 currentCameraInfo_->videoStabilizationMode = zoomInfo.videoStabilizationMode;
161 if (zoomInfo.focusMode >= 0) {
162 currentCameraInfo_->focusMode = zoomInfo.focusMode;
163 }
164 if (currentCameraInfo_->tokenId != 0 && ComputeFov() == ERR_OK) {
165 return UpdateMotionManagers();
166 }
167 HILOGI("end");
168 return ERR_OK;
169 }
170
ComputeFov()171 int32_t McCameraTrackingController::ComputeFov()
172 {
173 if (currentCameraInfo_ == nullptr) {
174 HILOGW("currentCameraInfo_ is nullptr; ");
175 return CAMERA_INFO_IS_EMPTY;
176 }
177 float sensorWidthCut = 0.0f;
178 float sensorHighCut = 0.0f;
179 float sensorRatio = currentCameraInfo_->sensorHeight != 0 ?
180 currentCameraInfo_->sensorWidth / currentCameraInfo_->sensorHeight : 0;
181 float viewRatio = currentCameraInfo_->height > 0 ?
182 static_cast<float>(currentCameraInfo_->width) / static_cast<float>(currentCameraInfo_->height) : 0;
183
184 if (sensorRatio > viewRatio) {
185 sensorHighCut = currentCameraInfo_->sensorHeight;
186 sensorWidthCut = currentCameraInfo_->sensorHeight * viewRatio;
187 } else {
188 sensorWidthCut = currentCameraInfo_->sensorWidth;
189 sensorHighCut = currentCameraInfo_->sensorWidth / (viewRatio > 0 ? viewRatio : 1);
190 }
191
192 auto focus = static_cast<float>(currentCameraInfo_->equivalentFocus);
193 float fovA = DOUBLE *
194 std::round(RadToDegree(std::atan(sensorWidthCut / (DOUBLE * focus * currentCameraInfo_->zoomFactor))));
195 float fovB = DOUBLE *
196 std::round(RadToDegree(std::atan(sensorHighCut / (DOUBLE * focus * currentCameraInfo_->zoomFactor))));
197 float shortSide = (fovA < fovB) ? fovA : fovB;
198 float longSide = (fovA < fovB) ? fovB : fovA;
199 HILOGI("fov before cut, short side: %{public}f; long side: %{public}f; videoStabilizationMode: %{public}d;"
200 " sessionMode: %{public}d",
201 shortSide, longSide, currentCameraInfo_->videoStabilizationMode, currentCameraInfo_->sessionMode);
202 if (VIDEO_STABILIZATION_WHITELIST.find(currentCameraInfo_->sessionMode) != VIDEO_STABILIZATION_WHITELIST.end()) {
203 if (currentCameraInfo_->videoStabilizationMode ==
204 static_cast<int32_t>(CameraVideoStabilizationMode::OHOS_CAMERA_VIDEO_STABILIZATION_AUTO)) {
205 shortSide *= SHORT_CUT * OHOS_CAMERA_VIDEO_STABILIZATION_AUTO_CUT;
206 longSide *= OHOS_CAMERA_VIDEO_STABILIZATION_AUTO_CUT;
207 }
208 if (currentCameraInfo_->videoStabilizationMode ==
209 static_cast<int32_t>(CameraVideoStabilizationMode::OHOS_CAMERA_VIDEO_STABILIZATION_HIGH)) {
210 shortSide *= SHORT_CUT * OHOS_CAMERA_VIDEO_STABILIZATION_HIGH_CUT;
211 longSide *= OHOS_CAMERA_VIDEO_STABILIZATION_HIGH_CUT;
212 }
213 }
214
215 HILOGI("fov after cut, short side: %{public}f; long side: %{public}f;", shortSide, longSide);
216 if (sensorRotation_ == MobileRotation::UP || sensorRotation_ == MobileRotation::DOWN) {
217 currentCameraInfo_->fovH = static_cast<uint8_t>(shortSide);
218 currentCameraInfo_->fovV = static_cast<uint8_t>(longSide);
219 } else {
220 currentCameraInfo_->fovH = static_cast<uint8_t>(longSide);
221 currentCameraInfo_->fovV = static_cast<uint8_t>(shortSide);
222 }
223 return ERR_OK;
224 }
225
OnSessionStatusChange(int32_t sessionid,bool status)226 int32_t McCameraTrackingController::OnSessionStatusChange(int32_t sessionid, bool status)
227 {
228 HILOGI("start, status: %{public}s", status ? "true" : "false");
229 if (currentCameraInfo_ == nullptr) {
230 HILOGW("currentCameraInfo_ is nullptr; ");
231 return CAMERA_INFO_IS_EMPTY;
232 }
233 currentCameraInfo_->isCameraOn = status;
234 HILOGI("end");
235 return ERR_OK;
236 }
237
IsCurrentFocus()238 bool McCameraTrackingController::IsCurrentFocus()
239 {
240 HILOGI("Current focus is : %{public}d.", currentCameraInfo_->focusMode);
241 return FOCUS_MODE_WHITELIST.find(currentCameraInfo_->focusMode) == FOCUS_MODE_WHITELIST.end();
242 }
243
UpdateMotionManagers()244 int32_t McCameraTrackingController::UpdateMotionManagers()
245 {
246 HILOGI("start");
247 if (eventHandler_ != nullptr) {
248 eventHandler_->RemoveTask(SEND_CAMERA_INFO_TASK_NAME);
249 eventHandler_->PostTask([]() {
250 McCameraTrackingController::GetInstance().UpdateMotionManagers();
251 }, SEND_CAMERA_INFO_TASK_NAME, SEND_CAMERA_INFO_TASK_DELAY);
252 }
253 const auto& motionManagers = MechBodyControllerService::GetInstance().motionManagers_;
254 CameraInfoParams cameraInfoParams;
255 cameraInfoParams.fovH = currentCameraInfo_->fovH;
256 cameraInfoParams.fovV = currentCameraInfo_->fovV;
257 cameraInfoParams.zoomFactor = currentCameraInfo_->zoomFactor;
258 cameraInfoParams.isRecording = currentCameraInfo_->isRecording;
259 cameraInfoParams.cameraType = currentCameraInfo_->cameraType;
260
261 std::lock_guard<std::mutex> lock(MechBodyControllerService::GetInstance().motionManagersMutex);
262 for (const auto &item : motionManagers) {
263 int32_t mechId = item.first;
264 std::shared_ptr<MotionManager> motionManager = item.second;
265 if (!motionManager) {
266 HILOGE("MotionManager not exits; mechId: %{public}d", mechId);
267 continue;
268 }
269
270 int32_t result = motionManager->SetMechCameraInfo(cameraInfoParams);
271 HILOGI("mech id: %{public}d result code: %{public}d", mechId, result);
272 if (result != ERR_OK) {
273 return result;
274 }
275 }
276 HILOGI("end");
277 return ERR_OK;
278 }
279
OnFocusTracking(CameraStandard::FocusTrackingMetaInfo & info)280 int32_t McCameraTrackingController::OnFocusTracking(CameraStandard::FocusTrackingMetaInfo &info)
281 {
282 HILOGI("OnFocusTracking.");
283 bool isEnableTrackingMode =
284 SESSION_MODE_WHITELIST.find(currentCameraInfo_->sessionMode) != SESSION_MODE_WHITELIST.end();
285 if (!currentCameraInfo_->currentTrackingEnable || IsCurrentFocus() || !isEnableTrackingMode) {
286 HILOGW("current tracking is not enabled");
287 return ERR_OK;
288 }
289
290 HILOGI("Tracking mode: %{public}d; "
291 "Tracking region: [x: %{public}f, y: %{public}f, width: %{public}f, height: %{public}f]; "
292 "Tracking object ID: %{public}d",
293 info.GetTrackingMode(),
294 info.GetTrackingRegion().topLeftX, info.GetTrackingRegion().topLeftY,
295 info.GetTrackingRegion().width, info.GetTrackingRegion().height,
296 info.GetTrackingObjectId());
297
298 std::ostringstream oss;
299 for (const auto& obj : info.GetDetectedObjects()) {
300 oss << "Detected object ID: " << obj->GetObjectId()
301 << ", Type: " << static_cast<int>(obj->GetType())
302 << ", BBox: ["
303 << obj->GetBoundingBox().topLeftX << ", "
304 << obj->GetBoundingBox().topLeftY << ", "
305 << obj->GetBoundingBox().width << ", "
306 << obj->GetBoundingBox().height << "]"
307 << ", Confidence: " << obj->GetConfidence();
308 }
309 HILOGI("tracking object info: %{public}s", oss.str().c_str());
310
311 ROI lastRoi = lastTrackingFrame_->roi;
312 std::shared_ptr<TrackingFrameParams> trackingParams = BuildTrackingParams(info);
313 if (!trackingParams) {
314 HILOGE("Failed to build tracking params");
315 return ERR_OK;
316 }
317 HILOGI("tracking param: %{public}s", trackingParams->ToString().c_str());
318 if (trackingParams->roi.width <= TRACKING_LOST_CHECK ||
319 trackingParams->roi.height <= TRACKING_LOST_CHECK) {
320 HILOGW("tracking rect lost, width or height is zero.");
321 return ERR_OK;
322 }
323 ROI roi = trackingParams->roi;
324 if (std::abs(lastRoi.x - roi.x) < TRACKING_LOST_CHECK &&
325 std::abs(lastRoi.y - roi.y) < TRACKING_LOST_CHECK &&
326 std::abs(lastRoi.width - roi.width) < TRACKING_LOST_CHECK &&
327 std::abs(lastRoi.height - roi.height) < TRACKING_LOST_CHECK) {
328 HILOGI("tracking param is same as last time. send ignore.");
329 return ERR_OK;
330 }
331 return UpdateMotionsWithTrackingData(trackingParams, info.GetTrackingObjectId());
332 }
333
BuildTrackingParams(CameraStandard::FocusTrackingMetaInfo & info)334 std::shared_ptr<TrackingFrameParams> McCameraTrackingController::BuildTrackingParams(
335 CameraStandard::FocusTrackingMetaInfo &info)
336 {
337 CameraStandard::Rect trackingRegion = info.GetTrackingRegion();
338
339 std::shared_ptr<TrackingFrameParams> trackingFrameParams = std::make_shared<TrackingFrameParams>();
340 trackingFrameParams->confidence = ConfidenceLevel::HIGH;
341 trackingFrameParams->targetId = info.GetTrackingObjectId();
342 auto now = std::chrono::system_clock::now();
343 trackingFrameParams->timeStamp = static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
344 now.time_since_epoch()).count());
345 UpdateROI(trackingFrameParams, trackingRegion);
346 trackingFrameParams->fovV = currentCameraInfo_->fovV;
347 trackingFrameParams->fovH = currentCameraInfo_->fovH;
348 trackingFrameParams->isRecording = currentCameraInfo_->isRecording;
349 trackingFrameParams->timeDelay = 0;
350 trackingFrameParams->objectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_FACE);
351
352 sptr<CameraStandard::MetadataObject> targetObject;
353 std::vector<sptr<CameraStandard::MetadataObject>> detectedObjects = info.GetDetectedObjects();
354 if (GetTrackingTarget(trackingRegion, detectedObjects, info.GetTrackingObjectId(), targetObject) ==
355 ERR_OK) {
356 int32_t confidence = targetObject->GetConfidence();
357 if (confidence > CONFIDENCE_LEVEL_MIDDEL_END) {
358 trackingFrameParams->confidence = ConfidenceLevel::HIGH;
359 } else if (confidence > CONFIDENCE_LEVEL_LOW_END) {
360 trackingFrameParams->confidence = ConfidenceLevel::MEDIUM;
361 } else if (confidence > CONFIDENCE_LEVEL_LOST_END) {
362 trackingFrameParams->confidence = ConfidenceLevel::LOW;
363 } else {
364 trackingFrameParams->confidence = ConfidenceLevel::LOST;
365 }
366
367 trackingFrameParams->timeDelay = 0;
368 trackingFrameParams->timeStamp = static_cast<uint64_t>(targetObject->GetTimestamp());
369 trackingFrameParams->targetId = targetObject->GetObjectId();
370 CameraStandard::MetadataObjectType objectType = targetObject->GetType();
371 ConvertObjectType(objectType, trackingFrameParams->objectType);
372 CameraStandard::Rect targetRect = targetObject->GetBoundingBox();
373 UpdateROI(trackingFrameParams, targetRect);
374 }
375 lastTrackingFrame_ = trackingFrameParams;
376 return trackingFrameParams;
377 }
378
GetTrackingTarget(CameraStandard::Rect & trackingRegion,std::vector<sptr<CameraStandard::MetadataObject>> & detectedObjects,int32_t trackingObjectId,sptr<CameraStandard::MetadataObject> & targetObject)379 int32_t McCameraTrackingController::GetTrackingTarget(CameraStandard::Rect &trackingRegion,
380 std::vector<sptr<CameraStandard::MetadataObject>> &detectedObjects, int32_t trackingObjectId,
381 sptr<CameraStandard::MetadataObject> &targetObject)
382 {
383 if (detectedObjects.empty()) {
384 HILOGW("detectedObjects is empty!!!");
385 return DETECTED_OBJECT_IS_EMPTY;
386 }
387 if (trackingObjectId >= 0) {
388 for (const auto &item: detectedObjects) {
389 if (item->GetObjectId() == trackingObjectId) {
390 HILOGI("got detected object for id: %{public}d", trackingObjectId);
391 targetObject = item;
392 return ERR_OK;
393 }
394 }
395 }
396
397 if (lastTrackingFrame_ != nullptr && lastTrackingFrame_->targetId >= 0) {
398 for (const auto &item: detectedObjects) {
399 if (item->GetObjectId() == lastTrackingFrame_->targetId) {
400 HILOGI("got detected object for id: %{public}d", lastTrackingFrame_->targetId);
401 targetObject = item;
402 return ERR_OK;
403 }
404 }
405 }
406
407 for (const auto &item: detectedObjects) {
408 CameraStandard::Rect itemRect = item->GetBoundingBox();
409 if (std::abs(itemRect.topLeftX - trackingRegion.topLeftX) <= TRACKING_LOST_CHECK &&
410 std::abs(itemRect.topLeftY - trackingRegion.topLeftY) <= TRACKING_LOST_CHECK) {
411 HILOGI("got detected object which is same as trackingRegion");
412 targetObject = item;
413 return ERR_OK;
414 }
415 }
416
417 HILOGW("use first object as target object.");
418 targetObject = detectedObjects[0];
419 return ERR_OK;
420 }
421
FilterDetectedObject(sptr<CameraStandard::MetadataObject> & detectedObject)422 bool McCameraTrackingController::FilterDetectedObject(sptr<CameraStandard::MetadataObject> &detectedObject)
423 {
424 CameraStandard::MetadataObjectType cameraObjectType = detectedObject->GetType();
425 switch (cameraObjectType) {
426 case CameraStandard::MetadataObjectType::INVALID :
427 return false;
428 case CameraStandard::MetadataObjectType::FACE :
429 return true;
430 case CameraStandard::MetadataObjectType::HUMAN_BODY :
431 return true;
432 case CameraStandard::MetadataObjectType::CAT_FACE :
433 return false;
434 case CameraStandard::MetadataObjectType::CAT_BODY :
435 return false;
436 case CameraStandard::MetadataObjectType::DOG_FACE :
437 return false;
438 case CameraStandard::MetadataObjectType::DOG_BODY :
439 return false;
440 case CameraStandard::MetadataObjectType::SALIENT_DETECTION :
441 return false;
442 case CameraStandard::MetadataObjectType::BAR_CODE_DETECTION :
443 return false;
444 case CameraStandard::MetadataObjectType::BASE_FACE_DETECTION :
445 return false;
446 default:
447 return false;
448 }
449 }
450
UpdateROI(std::shared_ptr<TrackingFrameParams> & trackingFrameParams,CameraStandard::Rect & rect)451 void McCameraTrackingController::UpdateROI(std::shared_ptr<TrackingFrameParams> &trackingFrameParams,
452 CameraStandard::Rect &rect)
453 {
454 HILOGI("rect info, topx: %{public}f; topy: %{public}f;width: %{public}f; height: %{public}f", rect.topLeftX,
455 rect.topLeftY, rect.width, rect.height);
456 if (trackingFrameParams == nullptr) {
457 HILOGW("trackingFrameParams is nullptr.");
458 return;
459 }
460 if ((rect.topLeftX < TRACKING_LOST_CHECK && rect.topLeftY < TRACKING_LOST_CHECK) ||
461 rect.width < TRACKING_LOST_CHECK || rect.height < TRACKING_LOST_CHECK) {
462 trackingFrameParams->roi = {0.0f, 0.0f, 0.0f, 0.0f};
463 return;
464 }
465 trackingFrameParams->roi.width = rect.width - rect.topLeftX;
466 trackingFrameParams->roi.height = rect.height - rect.topLeftY;
467 if (currentCameraInfo_->cameraType == CameraType::FRONT) {
468 if (sensorRotation_ == MobileRotation::UP || sensorRotation_ == MobileRotation::LEFT) {
469 trackingFrameParams->roi.x = NUM_1 - rect.topLeftX - trackingFrameParams->roi.width / NUM_2;
470 trackingFrameParams->roi.y = NUM_1 - rect.topLeftY - trackingFrameParams->roi.height / NUM_2;
471 } else if (sensorRotation_ == MobileRotation::RIGHT || sensorRotation_ == MobileRotation::DOWN) {
472 trackingFrameParams->roi.x = rect.topLeftX + trackingFrameParams->roi.width / NUM_2;
473 trackingFrameParams->roi.y = rect.topLeftY + trackingFrameParams->roi.height / NUM_2;
474 }
475 } else if (currentCameraInfo_->cameraType == CameraType::BACK) {
476 if (sensorRotation_ == MobileRotation::UP || sensorRotation_ == MobileRotation::LEFT) {
477 trackingFrameParams->roi.x = rect.topLeftX + trackingFrameParams->roi.width / NUM_2;
478 trackingFrameParams->roi.y = rect.topLeftY + trackingFrameParams->roi.height / NUM_2;
479 } else if (sensorRotation_ == MobileRotation::RIGHT || sensorRotation_ == MobileRotation::DOWN) {
480 trackingFrameParams->roi.x = NUM_1 - rect.topLeftX - trackingFrameParams->roi.width / NUM_2;
481 trackingFrameParams->roi.y = NUM_1 - rect.topLeftY - trackingFrameParams->roi.height / NUM_2;
482 }
483 }
484 }
485
UpdateMotionsWithTrackingData(const std::shared_ptr<TrackingFrameParams> & params,int32_t trackingObjectId)486 int32_t McCameraTrackingController::UpdateMotionsWithTrackingData(
487 const std::shared_ptr<TrackingFrameParams> ¶ms, int32_t trackingObjectId)
488 {
489 if (params == nullptr) {
490 HILOGW("Params is null for tracking ID: %{public}d", trackingObjectId);
491 return ERR_OK;
492 }
493
494 HILOGD("tracking frame info: %{public}s", params->ToString().c_str());
495
496 std::lock_guard<std::mutex> lock(MechBodyControllerService::GetInstance().motionManagersMutex);
497 for (const auto &item : MechBodyControllerService::GetInstance().motionManagers_) {
498 int32_t mechId = item.first;
499 std::shared_ptr<MotionManager> motionManager = item.second;
500 if (motionManager == nullptr) {
501 HILOGE("MotionManager not exists; mechId: %{public}d", mechId);
502 continue;
503 }
504
505 int32_t setResult = motionManager->SetMechCameraTrackingFrame(params);
506 HILOGI("Set tracking info for mech id: %{public}d; result: %{public}s; code: %{public}d",
507 mechId, setResult == ERR_OK ? "success" : "failed", setResult);
508 }
509 return ERR_OK;
510 }
511
SetTrackingEnabled(const uint32_t & tokenId,bool & isEnabled)512 int32_t McCameraTrackingController::SetTrackingEnabled(const uint32_t &tokenId, bool &isEnabled)
513 {
514 std::lock_guard<std::mutex> lock(appSettingsMutex_);
515 if (appSettings.find(tokenId) == appSettings.end()) {
516 std::shared_ptr<AppSetting> appSetting = std::make_shared<AppSetting>();
517 appSettings[tokenId] = appSetting;
518 }
519 HILOGI("update tracking enable");
520 appSettings[tokenId]->isTrackingEnabled = isEnabled;
521 currentCameraInfo_->currentTrackingEnable = isEnabled;
522 return ERR_OK;
523 }
524
GetTrackingEnabled(const uint32_t & tokenId,bool & isEnabled)525 int32_t McCameraTrackingController::GetTrackingEnabled(const uint32_t &tokenId, bool &isEnabled)
526 {
527 std::lock_guard<std::mutex> lock(appSettingsMutex_);
528 if (appSettings.find(tokenId) == appSettings.end()) {
529 HILOGE("No configuration information is saved. app token id: %{public}s;", GetAnonymUint32(tokenId).c_str());
530 isEnabled = true;
531 return ERR_OK;
532 }
533 std::shared_ptr<AppSetting> setting = appSettings[tokenId];
534 isEnabled = setting->isTrackingEnabled;
535 return ERR_OK;
536 }
537
RegisterTrackingEventCallback(const uint32_t & tokenId,sptr<IRemoteObject> callback)538 int32_t McCameraTrackingController::RegisterTrackingEventCallback(const uint32_t &tokenId,
539 sptr <IRemoteObject> callback)
540 {
541 HILOGI("start, tokenid: %{public}s;", GetAnonymUint32(tokenId).c_str());
542 sptr <MechControllerIpcDeathListener> diedListener = new MechControllerIpcDeathListener();
543 diedListener->tokenId_ = tokenId;
544 diedListener->objectType_ = RemoteObjectType::TRACKING_EVENT_CALLBACK;
545 CHECK_POINTER_RETURN_VALUE(callback, INVALID_PARAMETERS_ERR, "callback");
546 callback->AddDeathRecipient(diedListener);
547 {
548 std::lock_guard<std::mutex> lock(trackingEventCallbackMutex_);
549 trackingEventCallback_[tokenId] = callback;
550 }
551 HILOGI("callback add success, tokenid: %{public}s;", GetAnonymUint32(tokenId).c_str());
552 return ERR_OK;
553 }
554
UnRegisterTrackingEventCallback(const uint32_t & tokenId)555 int32_t McCameraTrackingController::UnRegisterTrackingEventCallback(const uint32_t &tokenId)
556 {
557 HILOGI("start, tokenId: %{public}s;", GetAnonymUint32(tokenId).c_str());
558 std::lock_guard<std::mutex> lock(trackingEventCallbackMutex_);
559 trackingEventCallback_.erase(tokenId);
560 HILOGI("callback remove success, tokenId: %{public}s;", GetAnonymUint32(tokenId).c_str());
561 return ERR_OK;
562 }
563
OnTrackingEvent(const int32_t & mechId,const TrackingEvent & event)564 int32_t McCameraTrackingController::OnTrackingEvent(const int32_t &mechId, const TrackingEvent &event)
565 {
566 HILOGI("start notify tracking event, mechId: %{public}d; event: %{public}d", mechId,
567 static_cast<int32_t>(event));
568 std::lock_guard<std::mutex> lock(trackingEventCallbackMutex_);
569 for (const auto &item: trackingEventCallback_) {
570 uint32_t tokenId = item.first;
571 HILOGI("notify tracking event to tokenId: %{public}s;", GetAnonymUint32(tokenId).c_str());
572 sptr <IRemoteObject> callback = item.second;
573 MessageParcel data;
574 if (!data.WriteInterfaceToken(MECH_SERVICE_IPC_TOKEN)) {
575 HILOGE("Write interface token failed.");
576 continue;
577 }
578 if (!data.WriteInt32(mechId)) {
579 HILOGE("Write mechId failed.");
580 continue;
581 }
582
583 if (!data.WriteInt32(static_cast<int32_t>(event))) {
584 HILOGE("Write event failed.");
585 continue;
586 }
587 MessageParcel reply;
588 MessageOption option;
589 CHECK_POINTER_RETURN_VALUE(callback, INVALID_PARAMETERS_ERR, "callback");
590 int32_t error = callback->SendRequest(
591 static_cast<uint32_t>(IMechBodyControllerCode::TRACKING_EVENT_CALLBACK), data, reply, option);
592 HILOGI("notify tracking event to tokenId: %{public}s; result: %{public}s", GetAnonymUint32(tokenId).c_str(),
593 error == ERR_NONE ? "success" : "failed");
594 }
595 return ERR_OK;
596 }
597
SetTrackingLayout(CameraTrackingLayout & cameraTrackingLayout)598 int32_t McCameraTrackingController::SetTrackingLayout(CameraTrackingLayout &cameraTrackingLayout)
599 {
600 std::shared_ptr<LayoutParams> layoutParam = std::make_shared<LayoutParams>();
601 layoutParam->isDefault = cameraTrackingLayout == CameraTrackingLayout::DEFAULT;
602 layoutParam->offsetX = LAYOUT_MIDDLE;
603 layoutParam->offsetY = LAYOUT_MIDDLE;
604 if (sensorRotation_ == MobileRotation::UP) {
605 layoutParam->offsetY = ParseLayout(cameraTrackingLayout);
606 }
607 if (sensorRotation_ == MobileRotation::DOWN) {
608 layoutParam->offsetY = ParseReverseLayout(cameraTrackingLayout);
609 }
610 if (sensorRotation_ == MobileRotation::LEFT) {
611 layoutParam->offsetX = ParseReverseLayout(cameraTrackingLayout);
612 }
613 if (sensorRotation_ == MobileRotation::RIGHT) {
614 layoutParam->offsetX = ParseLayout(cameraTrackingLayout);
615 }
616 bool hasSuccess = false;
617 {
618 std::lock_guard<std::mutex> lock(MechBodyControllerService::GetInstance().motionManagersMutex);
619 std::map<int32_t, std::shared_ptr<MotionManager>> motionManagers =
620 MechBodyControllerService::GetInstance().motionManagers_;
621 for (const auto &item: motionManagers) {
622 int32_t mechId = item.first;
623 std::shared_ptr<MotionManager> motionManager = item.second;
624 if (motionManager == nullptr) {
625 HILOGE("motionManager not exist. mechId: %{public}d;", mechId);
626 continue;
627 }
628 int32_t result = motionManager->SetMechCameraTrackingLayout(layoutParam);
629 hasSuccess = hasSuccess || (result == ERR_OK);
630 HILOGI("mech id: %{public}d result code: %{public}d, layout: %{public}s",
631 mechId, result, layoutParam->ToString().c_str());
632 }
633 }
634 if (hasSuccess) {
635 currentCameraInfo_->currentCameraTrackingLayout = cameraTrackingLayout;
636 }
637 return hasSuccess ? ERR_OK : INVALID_TRACKING_LAYOUT;
638 }
639
SetTrackingLayout(const uint32_t & tokenId,CameraTrackingLayout & cameraTrackingLayout)640 int32_t McCameraTrackingController::SetTrackingLayout(const uint32_t &tokenId,
641 CameraTrackingLayout &cameraTrackingLayout)
642 {
643 HILOGI("tokenId: %{public}s;", GetAnonymUint32(tokenId).c_str());
644 int32_t result = SetTrackingLayout(cameraTrackingLayout);
645 {
646 std::lock_guard<std::mutex> lock(appSettingsMutex_);
647 if (appSettings.find(tokenId) == appSettings.end()) {
648 std::shared_ptr<AppSetting> appSetting = std::make_shared<AppSetting>();
649 appSettings[tokenId] = appSetting;
650 }
651 if (result == ERR_OK) {
652 appSettings[tokenId]->cameraTrackingLayout = cameraTrackingLayout;
653 }
654 }
655 return ERR_OK;
656 }
657
GetTrackingLayout(const uint32_t & tokenId,CameraTrackingLayout & cameraTrackingLayout)658 int32_t McCameraTrackingController::GetTrackingLayout(const uint32_t &tokenId,
659 CameraTrackingLayout &cameraTrackingLayout)
660 {
661 HILOGI("tokenId: %{public}s;", GetAnonymUint32(tokenId).c_str());
662 if (appSettings.find(tokenId) == appSettings.end()) {
663 HILOGE("No configuration information is saved. app token id: %{public}s;", GetAnonymUint32(tokenId).c_str());
664 cameraTrackingLayout = CameraTrackingLayout::DEFAULT;
665 return ERR_OK;
666 }
667
668 std::shared_ptr<AppSetting> setting = appSettings[tokenId];
669 if (setting == nullptr) {
670 HILOGE("setting is nullptr. app token id: %{public}s;", GetAnonymUint32(tokenId).c_str());
671 cameraTrackingLayout = CameraTrackingLayout::DEFAULT;
672 return ERR_OK;
673 }
674 cameraTrackingLayout = setting->cameraTrackingLayout;
675 return ERR_OK;
676 }
677
GetCurrentCameraInfo() const678 std::shared_ptr<CameraInfo> McCameraTrackingController::GetCurrentCameraInfo() const
679 {
680 return currentCameraInfo_;
681 }
682
SensorCallback(SensorEvent * event)683 void McCameraTrackingController::SensorCallback(SensorEvent* event)
684 {
685 if (event == nullptr) {
686 return;
687 }
688 GravityData* gravityData = reinterpret_cast<GravityData*>(event->data);
689 MobileRotation sensorRotation = CalculateSensorRotation(gravityData);
690 if (sensorRotation != MobileRotation::INVALID &&
691 McCameraTrackingController::GetInstance().sensorRotation_ != sensorRotation) {
692 HILOGI("update device position to: %{public}d;", static_cast<int32_t>(sensorRotation));
693 McCameraTrackingController::GetInstance().sensorRotation_ = sensorRotation;
694 std::shared_ptr<CameraInfo> currentCameraInfo =
695 McCameraTrackingController::GetInstance().GetCurrentCameraInfo();
696 if (currentCameraInfo == nullptr) {
697 HILOGE("currentCameraInfo is nullptr;");
698 return;
699 }
700 McCameraTrackingController::GetInstance().SetTrackingLayout(currentCameraInfo->currentCameraTrackingLayout);
701 if (currentCameraInfo->tokenId != 0 &&
702 McCameraTrackingController::GetInstance().ComputeFov() == ERR_OK) {
703 McCameraTrackingController::GetInstance().UpdateMotionManagers();
704 }
705 }
706 }
707
CalculateSensorRotation(GravityData * gravityData)708 MobileRotation McCameraTrackingController::CalculateSensorRotation(GravityData* gravityData)
709 {
710 if (gravityData == nullptr) {
711 return MobileRotation::UP;
712 }
713 float x = gravityData->x;
714 float y = gravityData->y;
715 float z = gravityData->z;
716
717 if ((x * x + y * y) * VALID_INCLINATION_ANGLE_THRESHOLD_COEFFICIENT < z * z) {
718 return MobileRotation::INVALID;
719 }
720
721 int degree = DEGREE_CONSTANT_90 - static_cast<int>(round(std::atan2(y, -x) / M_PI * DEGREE_CONSTANT_180));
722 int32_t sensorDegree = degree >= 0 ?
723 degree % DEGREE_CONSTANT_360 : degree % DEGREE_CONSTANT_360 + DEGREE_CONSTANT_360;
724
725 if (sensorDegree >= 0 &&
726 (sensorDegree <= MOBILE_ROTATION_CHECK_UP_END || sensorDegree >= MOBILE_ROTATION_CHECK_UP_BEGIN)) {
727 return MobileRotation::UP;
728 } else if (sensorDegree >= MOBILE_ROTATION_CHECK_RIGHT_BEGIN && sensorDegree <= MOBILE_ROTATION_CHECK_RIGHT_END) {
729 return MobileRotation::RIGHT;
730 } else if (sensorDegree >= MOBILE_ROTATION_CHECK_DOWN_BEGIN && sensorDegree <= MOBILE_ROTATION_CHECK_DOWN_END) {
731 return MobileRotation::DOWN;
732 } else if (sensorDegree >= MOBILE_ROTATION_CHECK_LEFT_BEGIN && sensorDegree <= MOBILE_ROTATION_CHECK_LEFT_END) {
733 return MobileRotation::LEFT;
734 } else {
735 return MobileRotation::UP;
736 }
737 }
738
ConvertObjectType(CameraStandard::MetadataObjectType & cameraObjectType,uint8_t & mechObjectType)739 void McCameraTrackingController::ConvertObjectType(CameraStandard::MetadataObjectType &cameraObjectType,
740 uint8_t &mechObjectType)
741 {
742 switch (cameraObjectType) {
743 case CameraStandard::MetadataObjectType::INVALID :
744 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
745 break;
746 case CameraStandard::MetadataObjectType::FACE :
747 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_FACE);
748 break;
749 case CameraStandard::MetadataObjectType::HUMAN_BODY :
750 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_BODY);
751 break;
752 case CameraStandard::MetadataObjectType::CAT_FACE :
753 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
754 break;
755 case CameraStandard::MetadataObjectType::CAT_BODY :
756 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
757 break;
758 case CameraStandard::MetadataObjectType::DOG_FACE :
759 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
760 break;
761 case CameraStandard::MetadataObjectType::DOG_BODY :
762 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
763 break;
764 case CameraStandard::MetadataObjectType::SALIENT_DETECTION :
765 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
766 break;
767 case CameraStandard::MetadataObjectType::BAR_CODE_DETECTION :
768 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
769 break;
770 case CameraStandard::MetadataObjectType::BASE_FACE_DETECTION :
771 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
772 break;
773 default:
774 mechObjectType = static_cast<uint8_t>(TrackingObjectType::MSG_OBJ_OTHER);
775 }
776 }
777
RegisterTrackingListener()778 void McCameraTrackingController::RegisterTrackingListener()
779 {
780 int userId = INVALID_USER_ID;
781 ErrCode err = AccountSA::OsAccountManager::GetForegroundOsAccountLocalId(userId);
782 if (err != ERR_OK) {
783 HILOGE("GetLocalAccountId passing param invalid or return error!, err : %{public}d", err);
784 }
785 HILOGI("Create MechSession. user id: %{public}s", GetAnonymInt32(userId).c_str());
786 sptr<CameraStandard::CameraManager> cameraManager = CameraStandard::CameraManager::GetInstance();
787 if (cameraManager == nullptr) {
788 HILOGE("CameraManager is nullptr.");
789 return;
790 }
791 int createSessionResult = CameraStandard::CameraManager::GetInstance()->CreateMechSession(userId, &pMechSession);
792 HILOGI("Camera session create result: %{public}s; code: %{public}d",
793 createSessionResult == CameraStandard::CameraErrorCode::SUCCESS ? "success" : "failed",
794 createSessionResult);
795 if (createSessionResult == CameraStandard::CameraErrorCode::SUCCESS && pMechSession != nullptr) {
796 HILOGI("Set Session Callback.");
797 pMechSession->EnableMechDelivery(true);
798 std::shared_ptr<MechSessionCallbackImpl> mechSessionCallback = std::make_shared<MechSessionCallbackImpl>();
799 pMechSession->SetCallback(mechSessionCallback);
800 return;
801 }
802 }
803
UnRegisterTrackingListener()804 void McCameraTrackingController::UnRegisterTrackingListener()
805 {
806 if (pMechSession == nullptr) {
807 return;
808 }
809 pMechSession->EnableMechDelivery(false);
810 pMechSession->Release();
811 pMechSession = nullptr;
812 }
813
RegisterSensorListener()814 void McCameraTrackingController::RegisterSensorListener()
815 {
816 user.callback = McCameraTrackingController::SensorCallback;
817 int32_t subscribeRet = SubscribeSensor(SENSOR_TYPE_ID_GRAVITY, &user);
818 HILOGI("RegisterSensorCallback, subscribeRet: %{public}d", subscribeRet);
819 int32_t setBatchRet = SetBatch(SENSOR_TYPE_ID_GRAVITY, &user, POSTURE_INTERVAL, 0);
820 HILOGI("RegisterSensorCallback, setBatchRet: %{public}d", setBatchRet);
821 int32_t activateRet = ActivateSensor(SENSOR_TYPE_ID_GRAVITY, &user);
822 HILOGI("RegisterSensorCallback, activateRet: %{public}d", activateRet);
823 HILOGI("success");
824 }
825
UnRegisterSensorListener()826 void McCameraTrackingController::UnRegisterSensorListener()
827 {
828 if (user.callback == nullptr) {
829 return;
830 }
831 int32_t activateRet = DeactivateSensor(SENSOR_TYPE_ID_GRAVITY, &user);
832 HILOGI("UnRegisterSensorCallback, activateRet: %{public}d", activateRet);
833 int32_t subscribeRet = SubscribeSensor(SENSOR_TYPE_ID_GRAVITY, &user);
834 HILOGI("UnRegisterSensorCallback, subscribeRet: %{public}d", subscribeRet);
835 user.callback = nullptr;
836 HILOGI("success");
837 }
838
ParseLayout(CameraTrackingLayout & cameraTrackingLayout)839 float McCameraTrackingController::ParseLayout(CameraTrackingLayout &cameraTrackingLayout)
840 {
841 if (cameraTrackingLayout == CameraTrackingLayout::LEFT) {
842 return LAYOUT_LEFT;
843 }
844 if (cameraTrackingLayout == CameraTrackingLayout::MIDDLE) {
845 return LAYOUT_MIDDLE;
846 }
847 if (cameraTrackingLayout == CameraTrackingLayout::RIGHT) {
848 return LAYOUT_RIGHT;
849 }
850 return LAYOUT_MIDDLE;
851 }
852
ParseReverseLayout(CameraTrackingLayout & cameraTrackingLayout)853 float McCameraTrackingController::ParseReverseLayout(CameraTrackingLayout &cameraTrackingLayout)
854 {
855 if (cameraTrackingLayout == CameraTrackingLayout::LEFT) {
856 return LAYOUT_RIGHT;
857 }
858 if (cameraTrackingLayout == CameraTrackingLayout::MIDDLE) {
859 return LAYOUT_MIDDLE;
860 }
861 if (cameraTrackingLayout == CameraTrackingLayout::RIGHT) {
862 return LAYOUT_LEFT;
863 }
864 return LAYOUT_MIDDLE;
865 }
866
OnFocusTrackingInfo(CameraStandard::FocusTrackingMetaInfo info)867 void MechSessionCallbackImpl::OnFocusTrackingInfo(CameraStandard::FocusTrackingMetaInfo info)
868 {
869 McCameraTrackingController::GetInstance().OnFocusTracking(info);
870 }
871
OnCaptureSessionConfiged(CameraStandard::CaptureSessionInfo captureSessionInfo)872 void MechSessionCallbackImpl::OnCaptureSessionConfiged(CameraStandard::CaptureSessionInfo captureSessionInfo)
873 {
874 McCameraTrackingController::GetInstance().OnCaptureSessionConfiged(captureSessionInfo);
875 }
876
OnZoomInfoChange(int sessionid,CameraStandard::ZoomInfo zoomInfo)877 void MechSessionCallbackImpl::OnZoomInfoChange(int sessionid, CameraStandard::ZoomInfo zoomInfo)
878 {
879 McCameraTrackingController::GetInstance().OnZoomInfoChange(sessionid, zoomInfo);
880 }
881
OnSessionStatusChange(int32_t sessionid,bool status)882 void MechSessionCallbackImpl::OnSessionStatusChange(int32_t sessionid, bool status)
883 {
884 McCameraTrackingController::GetInstance().OnSessionStatusChange(sessionid, status);
885 }
886
887 } // namespace MechBodyController
888 } // namespace OHOS
889