1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef OHOS_CAMERA_H_STREAM_OPERATOR_H 17 #define OHOS_CAMERA_H_STREAM_OPERATOR_H 18 #define EXPORT_API __attribute__((visibility("default"))) 19 20 #include <atomic> 21 #include <cstdint> 22 #include <functional> 23 #include <iostream> 24 #include <list> 25 #include <memory> 26 #include <mutex> 27 #include <refbase.h> 28 #include <unordered_map> 29 #include <unordered_set> 30 #include "camera_util.h" 31 #include "fixed_size_list.h" 32 #include "hcamera_device.h" 33 #include "hcapture_session_stub.h" 34 #include "hstream_metadata.h" 35 #include "hstream_repeat.h" 36 #include "icapture_session.h" 37 #include "istream_common.h" 38 #include "camera_photo_proxy.h" 39 #include "moving_photo/moving_photo_surface_wrapper.h" 40 #include "surface.h" 41 #include "v1_0/istream_operator.h" 42 #include "v1_1/istream_operator.h" 43 #include "v1_2/istream_operator.h" 44 #include "v1_3/istream_operator_callback.h" 45 #include "hcamera_restore_param.h" 46 #include "iconsumer_surface.h" 47 #include "blocking_queue.h" 48 #include "audio_capturer.h" 49 #include "audio_info.h" 50 #include "avcodec_task_manager.h" 51 #include "moving_photo_video_cache.h" 52 #include "drain_manager.h" 53 #include "audio_capturer_session.h" 54 #include "safe_map.h" 55 #ifdef CAMERA_USE_SENSOR 56 #include "sensor_agent.h" 57 #include "sensor_agent_type.h" 58 #endif 59 namespace OHOS { 60 namespace CameraStandard { 61 using OHOS::HDI::Camera::V1_0::CaptureEndedInfo; 62 using OHOS::HDI::Camera::V1_0::CaptureErrorInfo; 63 using namespace AudioStandard; 64 using namespace std::chrono; 65 using namespace DeferredProcessing; 66 using namespace Media; 67 constexpr uint32_t OPERATOR_DEFAULT_ENCODER_THREAD_NUMBER = 1; 68 class PermissionStatusChangeCb; 69 class CameraUseStateChangeCb; 70 class DisplayRotationListener; 71 class CameraServerPhotoProxy; 72 73 class StreamContainer { 74 public: StreamContainer()75 StreamContainer() {}; 76 virtual ~StreamContainer() = default; 77 78 bool AddStream(sptr<HStreamCommon> stream); 79 bool RemoveStream(sptr<HStreamCommon> stream); 80 sptr<HStreamCommon> GetStream(int32_t streamId); 81 sptr<HStreamCommon> GetHdiStream(int32_t streamId); 82 void Clear(); 83 size_t Size(); 84 85 std::list<sptr<HStreamCommon>> GetStreams(const StreamType streamType); 86 std::list<sptr<HStreamCommon>> GetAllStreams(); 87 88 private: 89 std::mutex streamsLock_; 90 std::map<const StreamType, std::list<sptr<HStreamCommon>>> streams_; 91 }; 92 93 class SessionDrainImageCallback; 94 using MetaElementType = std::pair<int64_t, sptr<SurfaceBuffer>>; 95 class MovingPhotoListener : public MovingPhotoSurfaceWrapper::SurfaceBufferListener { 96 public: 97 explicit MovingPhotoListener(sptr<MovingPhotoSurfaceWrapper> surfaceWrapper, sptr<Surface> metaSurface, 98 shared_ptr<FixedSizeList<MetaElementType>> metaCache, uint32_t preCacheFrameCount, 99 uint32_t postCacheFrameCount); 100 ~MovingPhotoListener() override; 101 void OnBufferArrival(sptr<SurfaceBuffer> buffer, int64_t timestamp, GraphicTransformType transform) override; 102 void DrainOutImage(sptr<SessionDrainImageCallback> drainImageCallback); 103 void RemoveDrainImageManager(sptr<SessionDrainImageCallback> drainImageCallback); 104 void StopDrainOut(); 105 void ClearCache(uint64_t timestamp); 106 void SetClearFlag(); 107 108 private: 109 sptr<MovingPhotoSurfaceWrapper> movingPhotoSurfaceWrapper_; 110 sptr<Surface> metaSurface_; 111 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 112 BlockingQueue<sptr<FrameRecord>> recorderBufferQueue_; 113 SafeMap<sptr<SessionDrainImageCallback>, sptr<DrainImageManager>> callbackMap_; 114 std::atomic<bool> isNeededClear_ { false }; 115 std::atomic<bool> isNeededPop_ { false }; 116 int64_t shutterTime_; 117 uint64_t postCacheFrameCount_; 118 shared_ptr<TaskManager> bufferTaskManager_ = nullptr; 119 }; 120 121 class MovingPhotoMetaListener : public IBufferConsumerListener { 122 public: 123 explicit MovingPhotoMetaListener(sptr<Surface> surface, shared_ptr<FixedSizeList<MetaElementType>> metaCache); 124 ~MovingPhotoMetaListener(); 125 void OnBufferAvailable() override; 126 private: 127 sptr<Surface> surface_; 128 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 129 }; 130 131 class SessionDrainImageCallback : public DrainImageCallback { 132 public: 133 explicit SessionDrainImageCallback(std::vector<sptr<FrameRecord>>& frameCacheList, 134 wptr<MovingPhotoListener> listener, 135 wptr<MovingPhotoVideoCache> cache, 136 uint64_t timestamp, 137 int32_t rotation, 138 int32_t captureId); 139 ~SessionDrainImageCallback(); 140 void OnDrainImage(sptr<FrameRecord> frame) override; 141 void OnDrainImageFinish(bool isFinished) override; 142 143 private: 144 std::mutex mutex_; 145 std::vector<sptr<FrameRecord>> frameCacheList_; 146 wptr<MovingPhotoListener> listener_; 147 wptr<MovingPhotoVideoCache> videoCache_; 148 uint64_t timestamp_; 149 int32_t rotation_; 150 int32_t captureId_; 151 }; 152 153 class CameraInfoDumper; 154 155 class EXPORT_API HStreamOperator : public OHOS::HDI::Camera::V1_3::IStreamOperatorCallback { 156 public: 157 static sptr<HStreamOperator> NewInstance(const uint32_t callerToken, int32_t opMode); 158 HStreamOperator(); 159 explicit HStreamOperator(const uint32_t callingTokenId, int32_t opMode); 160 virtual ~HStreamOperator(); 161 int32_t AddOutput(StreamType streamType, sptr<IStreamCommon> stream); 162 int32_t Stop(); 163 int32_t Release(); 164 int32_t EnableMovingPhoto(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 165 bool isEnable, int32_t sensorOritation); 166 int32_t GetCurrentStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 167 std::list<sptr<HStreamCommon>> GetAllStreams(); 168 int32_t EnableMovingPhotoMirror(bool isMirror, bool isConfig); 169 int32_t CreateMediaLibrary(sptr<CameraPhotoProxy>& photoProxy, std::string& uri, int32_t& cameraShotType, 170 std::string& burstKey, int64_t timestamp); 171 int32_t CreateMediaLibrary(std::shared_ptr<PictureIntf> picture, sptr<CameraPhotoProxy> &photoProxy, 172 std::string &uri, int32_t &cameraShotType, std::string& burstKey, int64_t timestamp); 173 void SetCameraPhotoProxyInfo(sptr<CameraServerPhotoProxy> cameraPhotoProxy, int32_t &cameraShotType, 174 bool &isBursting, std::string &burstKey); 175 int32_t LinkInputAndOutputs(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, int32_t opMode); 176 const sptr<HStreamCommon> GetStreamByStreamID(int32_t streamId); 177 const sptr<HStreamCommon> GetHdiStreamByStreamID(int32_t streamId); 178 void SetSensorRotation(int32_t rotationValue, int32_t sensorOrientation, int32_t cameraPosition); 179 void StartMovingPhotoEncode(int32_t rotation, uint64_t timestamp, int32_t format, int32_t captureId); 180 void StartRecord(uint64_t timestamp, int32_t rotation, int32_t captureId); 181 void GetOutputStatus(int32_t &status); 182 int32_t SetPreviewRotation(std::string &deviceClass); 183 void ReleaseStreams(); 184 void StopMovingPhoto(); 185 int32_t GetActiveColorSpace(ColorSpace& colorSpace); 186 int32_t SetColorSpace(ColorSpace colorSpace, bool isNeedUpdate); 187 void SetColorSpaceForStreams(); 188 int32_t CheckIfColorSpaceMatchesFormat(ColorSpace colorSpace); 189 int32_t StartPreviewStream(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 190 camera_position_enum_t cameraPosition); 191 192 int32_t CreateStreams(std::vector<HDI::Camera::V1_1::StreamInfo_V1_1>& streamInfos); 193 int32_t CommitStreams(const std::shared_ptr<OHOS::Camera::CameraMetadata>& deviceSettings, int32_t operationMode); 194 int32_t ReleaseStreams(std::vector<int32_t>& releaseStreamIds); 195 int32_t GetStreamsSize(); 196 int32_t CreateAndCommitStreams(std::vector<HDI::Camera::V1_1::StreamInfo_V1_1>& streamInfos, 197 const std::shared_ptr<OHOS::Camera::CameraMetadata>& deviceSettings, int32_t operationMode); 198 int32_t UpdateStreams(std::vector<StreamInfo_V1_1>& streamInfos); 199 int32_t UpdateStreamInfos(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings); 200 int32_t RemoveOutput(StreamType streamType, sptr<IStreamCommon> stream); 201 int32_t RemoveOutputStream(sptr<HStreamCommon> stream); 202 203 int32_t OnCaptureStarted(int32_t captureId, const std::vector<int32_t>& streamIds) override; 204 int32_t OnCaptureStarted_V1_2( 205 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_2::CaptureStartedInfo>& infos) override; 206 int32_t OnCaptureEnded(int32_t captureId, const std::vector<CaptureEndedInfo>& infos) override; 207 int32_t OnCaptureEndedExt( 208 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_3::CaptureEndedInfoExt>& infos) override; 209 int32_t OnCaptureError(int32_t captureId, const std::vector<CaptureErrorInfo>& infos) override; 210 int32_t OnFrameShutter(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 211 int32_t OnFrameShutterEnd(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 212 int32_t OnCaptureReady(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 213 int32_t OnResult(int32_t streamId, const std::vector<uint8_t>& result) override; 214 int32_t UnlinkInputAndOutputs(); 215 void ClearSketchRepeatStream(); 216 void ExpandSketchRepeatStream(); 217 void ExpandMovingPhotoRepeatStream(); 218 void ClearMovingPhotoRepeatStream(); 219 void GetStreamOperator(); 220 bool IsOfflineCapture(); GetPid()221 inline int32_t GetPid() 222 { 223 return static_cast<int32_t>(pid_); 224 } ResetHdiStreamId()225 inline void ResetHdiStreamId() 226 { 227 hdiStreamIdGenerator_ = HDI_STREAM_ID_INIT; 228 } SetCameraDevice(sptr<HCameraDevice> device)229 inline void SetCameraDevice(sptr<HCameraDevice> device) 230 { 231 std::lock_guard<std::mutex> lock(cameraDeviceLock_); 232 cameraDevice_ = device; 233 if (device == nullptr && !IsOfflineCapture()) { 234 ResetHDIStreamOperator(); 235 } 236 } 237 SetStreamOperatorId(int32_t & streamOperatorId)238 inline void SetStreamOperatorId(int32_t& streamOperatorId) 239 { 240 streamOperatorId_ = streamOperatorId; 241 } 242 GetHDIStreamOperator()243 inline sptr<OHOS::HDI::Camera::V1_0::IStreamOperator> GetHDIStreamOperator() 244 { 245 std::lock_guard<std::mutex> lock(streamOperatorLock_); 246 return streamOperator_; 247 } 248 ResetHDIStreamOperator()249 inline void ResetHDIStreamOperator() 250 { 251 std::lock_guard<std::mutex> lock(streamOperatorLock_); 252 streamOperator_ = nullptr; 253 } 254 255 void StartMovingPhotoStream(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings); 256 int32_t GetOfflineOutptSize(); 257 int32_t GetAllOutptSize(); 258 259 uint32_t preCacheFrameCount_ = CACHE_FRAME_COUNT; 260 uint32_t postCacheFrameCount_ = CACHE_FRAME_COUNT; 261 sptr<AvcodecTaskManager> taskManager_; 262 std::vector<int32_t> GetFrameRateRange(); 263 264 private: 265 int32_t Initialize(const uint32_t callerToken, int32_t opMode); 266 void RegisterDisplayListener(sptr<HStreamRepeat> repeat); 267 void UnRegisterDisplayListener(sptr<HStreamRepeat> repeat); 268 string lastDisplayName_ = ""; 269 string lastBurstPrefix_ = ""; 270 int32_t saveIndex = 0; 271 int32_t streamOperatorId_ = -1; 272 volatile bool isMovingPhotoMirror_ = false; 273 volatile bool isSetMotionPhoto_ = false; 274 std::mutex livePhotoStreamLock_; // Guard livePhotoStreamRepeat_ 275 std::mutex opMutex_; // Lock the operations updateSettings_, streamOperator_, and hdiCameraDevice 276 sptr<HStreamRepeat> livePhotoStreamRepeat_; 277 std::atomic<int32_t> hdiStreamIdGenerator_ = HDI_STREAM_ID_INIT; 278 int32_t deviceSensorOritation_ = 0; 279 GenerateHdiStreamId()280 inline int32_t GenerateHdiStreamId() 281 { 282 return hdiStreamIdGenerator_.fetch_add(1); 283 } 284 285 string CreateDisplayName(); 286 string CreateBurstDisplayName(int32_t imageSeqId, int32_t seqId); 287 int32_t AddOutputStream(sptr<HStreamCommon> stream); 288 289 int32_t CreateMovingPhotoStreamRepeat(int32_t format, int32_t width, int32_t height, 290 sptr<OHOS::IBufferProducer> producer); 291 void CancelStreamsAndGetStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 292 void RestartStreams(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings); 293 void StartMovingPhoto(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 294 sptr<HStreamRepeat>& curStreamRepeat); 295 void ProcessMetaZoomArray(std::vector<uint32_t>& zoomAndTimeArray, sptr<HCameraDevice>& cameraDevice); 296 bool InitAudioCapture(); 297 bool StartAudioCapture(); 298 void ProcessAudioBuffer(); 299 void StartOnceRecord(uint64_t timestamp, int32_t rotation, int32_t captureId); 300 void UpdateMuteSetting(bool muteMode, std::shared_ptr<OHOS::Camera::CameraMetadata> &settings); 301 int32_t GetMovingPhotoBufferDuration(); 302 void GetMovingPhotoStartAndEndTime(); 303 void ConfigPayload(uint32_t pid, uint32_t tid, const char *bundleName, int32_t qosLevel, 304 std::unordered_map<std::string, std::string> &mapPayload); 305 std::shared_ptr<PhotoAssetIntf> ProcessPhotoProxy(int32_t captureId, std::shared_ptr<PictureIntf> picturePtr, 306 bool isBursting, sptr<CameraServerPhotoProxy> cameraPhotoProxy, std::string& uri); 307 void InitDefaultColortSpace(SceneMode opMode); 308 309 #ifdef CAMERA_USE_SENSOR 310 std::mutex sensorLock_; 311 bool isRegisterSensorSuccess_ = false; 312 void RegisterSensorCallback(); 313 void UnRegisterSensorCallback(); 314 static void GravityDataCallbackImpl(SensorEvent* event); 315 static int32_t CalcSensorRotation(int32_t sensorDegree); 316 static int32_t CalcRotationDegree(GravityData data); 317 #endif 318 // Make sure device thread safe,set device by {SetCameraDevice}, get device by {GetCameraDevice} 319 std::mutex cameraDeviceLock_; 320 std::mutex cbMutex_; 321 sptr<HCameraDevice> cameraDevice_; 322 StreamContainer streamContainer_; 323 StreamContainer streamContainerOffline_; 324 #ifdef CAMERA_USE_SENSOR 325 SensorUser user = { "", nullptr, nullptr }; 326 #endif 327 pid_t pid_; 328 uid_t uid_; 329 uint32_t callerToken_; 330 int32_t opMode_; 331 ColorSpace currColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; 332 bool isSessionStarted_ = false; 333 bool enableStreamRotate_ = false; 334 bool isDynamicConfiged_ = false; 335 std::string deviceClass_ = "phone"; 336 std::mutex movingPhotoStatusLock_; // Guard movingPhotoStatus 337 std::mutex releaseOperatorLock_; 338 std::mutex streamOperatorLock_; 339 sptr<MovingPhotoListener> livephotoListener_; 340 sptr<MovingPhotoMetaListener> livephotoMetaListener_; 341 sptr<AudioCapturerSession> audioCapturerSession_; 342 sptr<Surface> metaSurface_ = nullptr; 343 sptr<MovingPhotoVideoCache> videoCache_; 344 std::mutex displayListenerLock_; 345 sptr<DisplayRotationListener> displayListener_; 346 sptr<OHOS::HDI::Camera::V1_0::IStreamOperator> streamOperator_ = nullptr; 347 bool isOfflineStreamOperator_ = false; 348 int32_t mlastCaptureId = 0; 349 int32_t sensorRotation_ = 0; 350 }; 351 } // namespace CameraStandard 352 } // namespace OHOS 353 #endif // OHOS_CAMERA_H_STREAM_OPERATOR_H 354