• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2022 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef OHOS_CAMERA_METADATA_OUTPUT_H
17 #define OHOS_CAMERA_METADATA_OUTPUT_H
18 
19 #include <cstdint>
20 #include <iostream>
21 #include <mutex>
22 #include <sys/types.h>
23 
24 #include "camera_metadata_info.h"
25 #include "camera_metadata_operator.h"
26 #include "capture_output.h"
27 #include "session/capture_session.h"
28 #include "stream_metadata_callback_stub.h"
29 #include "iconsumer_surface.h"
30 #include "istream_metadata.h"
31 
32 #include "nocopyable.h"
33 #include "singleton.h"
34 #include "surface.h"
35 
36 namespace OHOS {
37 namespace CameraStandard {
38 static const std::map<MetadataObjectType, int32_t> mapLengthOfType = {
39     {MetadataObjectType::FACE, 23},
40     {MetadataObjectType::HUMAN_BODY, 9},
41     {MetadataObjectType::CAT_FACE, 18},
42     {MetadataObjectType::CAT_BODY, 9},
43     {MetadataObjectType::DOG_FACE, 18},
44     {MetadataObjectType::DOG_BODY, 9},
45     {MetadataObjectType::SALIENT_DETECTION, 9},
46     {MetadataObjectType::BAR_CODE_DETECTION, 9},
47     {MetadataObjectType::BASE_FACE_DETECTION, 9},
48 };
49 
50 enum MetadataOutputErrorCode : int32_t {
51     ERROR_UNKNOWN = 1,
52     ERROR_INSUFFICIENT_RESOURCES,
53 };
54 
55 enum Emotion : int32_t { NEUTRAL = 0, SADNESS, SMILE, SUPRISE };
56 
57 struct Rect {
58     double topLeftX;
59     double topLeftY;
60     double width;
61     double height;
62 };
63 
64 struct MetaObjectParms {
65     MetadataObjectType type;
66     int32_t timestamp;
67     Rect box;
68     int32_t objectId;
69     int32_t confidence;
70 };
71 
72 
73 class MetadataObject : public RefBase {
74 public:
75     MetadataObject(const MetadataObjectType type, const int32_t timestamp, const Rect rect, const int32_t objectId,
76                    const int32_t confidence);
77     MetadataObject(const MetaObjectParms& parms);
78     virtual ~MetadataObject() = default;
GetType()79     inline MetadataObjectType GetType()
80     {
81         return type_;
82     };
GetTimestamp()83     inline int32_t GetTimestamp()
84     {
85         return timestamp_;
86     };
GetBoundingBox()87     inline Rect GetBoundingBox()
88     {
89         return box_;
90     };
GetObjectId()91     inline int32_t GetObjectId()
92     {
93         return objectId_;
94     };
GetConfidence()95     inline int32_t GetConfidence()
96     {
97         return confidence_;
98     };
99 
100 private:
101     MetadataObjectType type_;
102     int32_t timestamp_;
103     Rect box_;
104     int32_t objectId_;
105     int32_t confidence_;
106 };
107 
108 class MetadataFaceObject : public MetadataObject {
109 public:
110     MetadataFaceObject(const MetaObjectParms& parms, const Rect leftEyeBoundingBox, const Rect rightEyeBoundingBox,
111                        const Emotion emotion, const int32_t emotionConfidence, const int32_t pitchAngle,
112                        const int32_t yawAngle, const int32_t rollAngle);
113     ~MetadataFaceObject() = default;
GetLeftEyeBoundingBox()114     inline Rect GetLeftEyeBoundingBox()
115     {
116         return leftEyeBoundingBox_;
117     };
GetRightEyeBoundingBox()118     inline Rect GetRightEyeBoundingBox()
119     {
120         return rightEyeBoundingBox_;
121     };
GetEmotion()122     inline Emotion GetEmotion()
123     {
124         return emotion_;
125     };
GetEmotionConfidence()126     inline int32_t GetEmotionConfidence()
127     {
128         return emotionConfidence_;
129     };
GetPitchAngle()130     inline int32_t GetPitchAngle()
131     {
132         return pitchAngle_;
133     };
GetYawAngle()134     inline int32_t GetYawAngle()
135     {
136         return yawAngle_;
137     };
GetRollAngle()138     inline int32_t GetRollAngle()
139     {
140         return rollAngle_;
141     };
142 
143 private:
144     Rect leftEyeBoundingBox_;
145     Rect rightEyeBoundingBox_;
146     Emotion emotion_;
147     int32_t emotionConfidence_;
148     int32_t pitchAngle_;
149     int32_t yawAngle_;
150     int32_t rollAngle_;
151 };
152 
153 class MetadataHumanBodyObject : public MetadataObject {
154 public:
155     MetadataHumanBodyObject(const MetaObjectParms& parms);
156     ~MetadataHumanBodyObject() = default;
157 };
158 
159 class MetadataCatFaceObject : public MetadataObject {
160 public:
161     MetadataCatFaceObject(const MetaObjectParms& parms, const Rect leftEyeBoundingBox, const Rect rightEyeBoundingBox);
162     ~MetadataCatFaceObject() = default;
GetLeftEyeBoundingBox()163     inline Rect GetLeftEyeBoundingBox()
164     {
165         return leftEyeBoundingBox_;
166     };
GetRightEyeBoundingBox()167     inline Rect GetRightEyeBoundingBox()
168     {
169         return rightEyeBoundingBox_;
170     };
171 
172 private:
173     Rect leftEyeBoundingBox_;
174     Rect rightEyeBoundingBox_;
175 };
176 
177 class MetadataCatBodyObject : public MetadataObject {
178 public:
179     MetadataCatBodyObject(const MetaObjectParms& parms);
180     ~MetadataCatBodyObject() = default;
181 };
182 
183 class MetadataDogFaceObject : public MetadataObject {
184 public:
185     MetadataDogFaceObject(const MetaObjectParms& parms, const Rect leftEyeBoundingBox, const Rect rightEyeBoundingBox);
186     ~MetadataDogFaceObject() = default;
GetLeftEyeBoundingBox()187     inline Rect GetLeftEyeBoundingBox()
188     {
189         return leftEyeBoundingBox_;
190     };
GetRightEyeBoundingBox()191     inline Rect GetRightEyeBoundingBox()
192     {
193         return rightEyeBoundingBox_;
194     };
195 
196 private:
197     Rect leftEyeBoundingBox_;
198     Rect rightEyeBoundingBox_;
199 };
200 
201 class MetadataDogBodyObject : public MetadataObject {
202 public:
203     MetadataDogBodyObject(const MetaObjectParms& parms);
204     ~MetadataDogBodyObject() = default;
205 };
206 
207 class MetadataSalientDetectionObject : public MetadataObject {
208 public:
209     MetadataSalientDetectionObject(const MetaObjectParms& parms);
210     ~MetadataSalientDetectionObject() = default;
211 };
212 
213 class MetadataBarCodeDetectionObject : public MetadataObject {
214 public:
215     MetadataBarCodeDetectionObject(const MetaObjectParms& parms);
216     ~MetadataBarCodeDetectionObject() = default;
217 };
218 
219 class MetadataObjectFactory : public RefBase {
220 public:
221     virtual ~MetadataObjectFactory() = default;
222 
223     static sptr<MetadataObjectFactory> &GetInstance();
SetType(MetadataObjectType type)224     inline sptr<MetadataObjectFactory> SetType(MetadataObjectType type)
225     {
226         type_ = type;
227         return this;
228     }
SetTimestamp(int32_t timestamp)229     inline sptr<MetadataObjectFactory> SetTimestamp(int32_t timestamp)
230     {
231         timestamp_ = timestamp;
232         return this;
233     }
SetBox(Rect box)234     inline sptr<MetadataObjectFactory> SetBox(Rect box)
235     {
236         box_ = box;
237         return this;
238     }
SetObjectId(int32_t objectId)239     inline sptr<MetadataObjectFactory> SetObjectId(int32_t objectId)
240     {
241         objectId_ = objectId;
242         return this;
243     }
SetConfidence(int32_t confidence)244     inline sptr<MetadataObjectFactory> SetConfidence(int32_t confidence)
245     {
246         confidence_ = confidence;
247         return this;
248     }
SetLeftEyeBoundingBox(Rect leftEyeBoundingBox)249     inline sptr<MetadataObjectFactory> SetLeftEyeBoundingBox(Rect leftEyeBoundingBox)
250     {
251         leftEyeBoundingBox_ = leftEyeBoundingBox;
252         return this;
253     }
SetRightEyeBoundingBoxd(Rect rightEyeBoundingBox)254     inline sptr<MetadataObjectFactory> SetRightEyeBoundingBoxd(Rect rightEyeBoundingBox)
255     {
256         rightEyeBoundingBox_ = rightEyeBoundingBox;
257         return this;
258     }
SetEmotion(Emotion emotion)259     inline sptr<MetadataObjectFactory> SetEmotion(Emotion emotion)
260     {
261         emotion_ = emotion;
262         return this;
263     }
SetEmotionConfidence(int32_t emotionConfidence)264     inline sptr<MetadataObjectFactory> SetEmotionConfidence(int32_t emotionConfidence)
265     {
266         emotionConfidence_ = emotionConfidence;
267         return this;
268     }
SetPitchAngle(int32_t pitchAngle)269     inline sptr<MetadataObjectFactory> SetPitchAngle(int32_t pitchAngle)
270     {
271         pitchAngle_ = pitchAngle;
272         return this;
273     }
SetYawAngle(int32_t yawAngle)274     inline sptr<MetadataObjectFactory> SetYawAngle(int32_t yawAngle)
275     {
276         yawAngle_ = yawAngle;
277         return this;
278     }
SetRollAngle(int32_t rollAngle)279     inline sptr<MetadataObjectFactory> SetRollAngle(int32_t rollAngle)
280     {
281         rollAngle_ = rollAngle;
282         return this;
283     }
284 
285     sptr<MetadataObject> createMetadataObject(MetadataObjectType type);
286 
287 private:
288     MetadataObjectFactory();
289     static sptr<MetadataObjectFactory> metaFactoryInstance_;
290     static std::mutex instanceMutex_;
291     // Parameters of metadataObject
292     MetadataObjectType type_ = MetadataObjectType::INVALID;
293     int32_t timestamp_ = 0;
294     Rect box_ = {0.0, 0.0, 0.0, 0.0};
295     int32_t objectId_ = 0;
296     int32_t confidence_ = 0;
297     // Parameters of All face metadata
298     Rect leftEyeBoundingBox_ = {0.0, 0.0, 0.0, 0.0};
299     Rect rightEyeBoundingBox_ = {0.0, 0.0, 0.0, 0.0};
300     // Parameters of human face metadata
301     Emotion emotion_ = NEUTRAL;
302     int32_t emotionConfidence_ = 0;
303     int32_t pitchAngle_ = 0;
304     int32_t yawAngle_ = 0;
305     int32_t rollAngle_ = 0;
306 
307     void ResetParameters();
308 };
309 
310 class MetadataObjectCallback {
311 public:
312     MetadataObjectCallback() = default;
313     virtual ~MetadataObjectCallback() = default;
314     virtual void OnMetadataObjectsAvailable(std::vector<sptr<MetadataObject>> metaObjects) const = 0;
315 };
316 
317 class MetadataStateCallback {
318 public:
319     MetadataStateCallback() = default;
320     virtual ~MetadataStateCallback() = default;
321     virtual void OnError(int32_t errorCode) const = 0;
322 };
323 
324 class FocusTrackingMetaInfo {
325 public:
GetTrackingMode()326     inline FocusTrackingMode GetTrackingMode()
327     {
328         return trackingMode_;
329     }
330 
GetTrackingRegion()331     inline Rect GetTrackingRegion()
332     {
333         return trackingRegion_;
334     }
335 
GetTrackingObjectId()336     inline int32_t GetTrackingObjectId()
337     {
338         return trackingObjectId_;
339     }
340 
GetDetectedObjects()341     inline std::vector<sptr<MetadataObject>> GetDetectedObjects()
342     {
343         return detectedObjects_;
344     }
345 
SetTrackingRegion(Rect & region)346     inline void SetTrackingRegion(Rect& region)
347     {
348         trackingRegion_ = region;
349     }
350 
SetTrackingMode(FocusTrackingMode mode)351     inline void SetTrackingMode(FocusTrackingMode mode)
352     {
353         trackingMode_ = mode;
354     }
355 
SetTrackingObjectId(int32_t trackingObjectId)356     inline void SetTrackingObjectId(int32_t trackingObjectId)
357     {
358         trackingObjectId_ = trackingObjectId;
359     }
360 
SetDetectedObjects(std::vector<sptr<MetadataObject>> detectedObjects)361     inline void SetDetectedObjects(std::vector<sptr<MetadataObject>> detectedObjects)
362     {
363         detectedObjects_ = detectedObjects;
364     }
365 
366 private:
367     FocusTrackingMode trackingMode_{FOCUS_TRACKING_MODE_AUTO};
368     Rect trackingRegion_{0.0, 0.0, 0.0, 0.0};
369     int32_t trackingObjectId_{-1};
370     std::vector<sptr<MetadataObject>> detectedObjects_{};
371 };
372 
373 class MetadataOutput : public CaptureOutput {
374 public:
375     MetadataOutput(sptr<IConsumerSurface> surface, sptr<IStreamMetadata> &streamMetadata);
376     ~MetadataOutput();
377 
378     /**
379      * @brief Get the supported metadata object types.
380      *
381      * @return Returns vector of MetadataObjectType.
382      */
383     std::vector<MetadataObjectType> GetSupportedMetadataObjectTypes();
384 
385     /**
386      * @brief Set the metadata object types
387      *
388      * @param Vector of MetadataObjectType
389      */
390     void SetCapturingMetadataObjectTypes(std::vector<MetadataObjectType> objectTypes);
391 
392     /**
393      * @brief Add the metadata object types
394      *
395      * @param Vector of MetadataObjectType
396      */
397     int32_t AddMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes);
398 
399     /**
400      * @brief Remove the metadata object types
401      *
402      * @param Vector of MetadataObjectType
403      */
404     int32_t RemoveMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes);
405 
406     /**
407      * @brief Set the metadata object callback for the metadata output.
408      *
409      * @param MetadataObjectCallback pointer to be triggered.
410      */
411     void SetCallback(std::shared_ptr<MetadataObjectCallback> metadataObjectCallback);
412 
413     /**
414      * @brief Set the metadata state callback for the metadata output.
415      *
416      * @param MetadataStateCallback pointer to be triggered.
417      */
418     void SetCallback(std::shared_ptr<MetadataStateCallback> metadataStateCallback);
419 
420     int32_t CreateStream() override;
421 
422     /**
423      * @brief Start the metadata capture.
424      */
425     int32_t Start();
426 
427     /**
428      * @brief Stop the metadata capture.
429      */
430     int32_t Stop();
431 
432     /**
433      * @brief Releases a instance of the MetadataOutput.
434      */
435     int32_t Release() override;
436     bool reportFaceResults_ = false;
437     bool reportLastFaceResults_ = false;
438     void ProcessMetadata(const int32_t streamId, const std::shared_ptr<OHOS::Camera::CameraMetadata>& result,
439                                std::vector<sptr<MetadataObject>>& metaObjects, bool isNeedMirror, bool isNeedFlip);
440     std::shared_ptr<MetadataObjectCallback> GetAppObjectCallback();
441     std::shared_ptr<MetadataStateCallback> GetAppStateCallback();
442 
443     friend class MetadataObjectListener;
444 
445 private:
446     void CameraServerDied(pid_t pid) override;
447     void ReleaseSurface();
448     sptr<IConsumerSurface> GetSurface();
449     bool checkValidType(const std::vector<MetadataObjectType>& typeAdded,
450                         const std::vector<MetadataObjectType>& supportedType);
451     std::vector<int32_t> convert(const std::vector<MetadataObjectType>& typesOfMetadata);
452 
453     std::mutex surfaceMutex_;
454     sptr<IConsumerSurface> surface_;
455     std::shared_ptr<MetadataObjectCallback> appObjectCallback_;
456     std::shared_ptr<MetadataStateCallback> appStateCallback_;
457     sptr<IStreamMetadataCallback> cameraMetadataCallback_;
458 };
459 
460 class MetadataObjectListener : public IBufferConsumerListener {
461 public:
462     MetadataObjectListener(sptr<MetadataOutput> metadata);
463     void OnBufferAvailable() override;
464 
465 private:
466     int32_t ProcessMetadataBuffer(void *buffer, int64_t timestamp);
467     wptr<MetadataOutput> metadata_;
468 };
469 
470 class HStreamMetadataCallbackImpl : public StreamMetadataCallbackStub {
471 public:
HStreamMetadataCallbackImpl(MetadataOutput * metaDataOutput)472     explicit HStreamMetadataCallbackImpl(MetadataOutput *metaDataOutput) : innerMetadataOutput(metaDataOutput) {}
473 
474     ~HStreamMetadataCallbackImpl() = default;
475 
476     int32_t OnMetadataResult(const int32_t streamId,
477                              const std::shared_ptr<OHOS::Camera::CameraMetadata> &result) override;
478 
GetMetadataOutput()479     inline sptr<MetadataOutput> GetMetadataOutput()
480     {
481         if (innerMetadataOutput == nullptr) {
482             return nullptr;
483         }
484         return innerMetadataOutput.promote();
485     }
486 
487 private:
488     wptr<MetadataOutput> innerMetadataOutput = nullptr;
489 };
490 }  // namespace CameraStandard
491 }  // namespace OHOS
492 #endif  // OHOS_CAMERA_METADATA_OUTPUT_H
493