1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef OHOS_CAMERA_H_STREAM_OPERATOR_H 17 #define OHOS_CAMERA_H_STREAM_OPERATOR_H 18 #define EXPORT_API __attribute__((visibility("default"))) 19 20 #include <atomic> 21 #include <cstdint> 22 #include <functional> 23 #include <iostream> 24 #include <list> 25 #include <memory> 26 #include <mutex> 27 #include <refbase.h> 28 #include <unordered_map> 29 #include <unordered_set> 30 #include "camera_util.h" 31 #include "fixed_size_list.h" 32 #include "hcamera_device.h" 33 #include "capture_session_stub.h" 34 #include "session/capture_scene_const.h" 35 #include "hstream_metadata.h" 36 #include "hstream_repeat.h" 37 #include "icapture_session.h" 38 #include "istream_common.h" 39 #include "camera_photo_proxy.h" 40 #include "moving_photo_surface_wrapper.h" 41 #include "surface.h" 42 #include "v1_0/istream_operator.h" 43 #include "v1_1/istream_operator.h" 44 #include "v1_2/istream_operator.h" 45 #include "v1_3/istream_operator_callback.h" 46 #include "hcamera_restore_param.h" 47 #include "iconsumer_surface.h" 48 #include "blocking_queue.h" 49 #include "drain_manager.h" 50 #include "moving_photo_proxy.h" 51 #include "safe_map.h" 52 #include "display_manager.h" 53 #ifdef CAMERA_USE_SENSOR 54 #include "sensor_agent.h" 55 #include "sensor_agent_type.h" 56 #endif 57 namespace OHOS { 58 namespace CameraStandard { 59 using OHOS::HDI::Camera::V1_0::CaptureEndedInfo; 60 using OHOS::HDI::Camera::V1_0::CaptureErrorInfo; 61 using namespace std::chrono; 62 using namespace DeferredProcessing; 63 using namespace Media; 64 constexpr uint32_t OPERATOR_DEFAULT_ENCODER_THREAD_NUMBER = 1; 65 class PermissionStatusChangeCb; 66 class CameraUseStateChangeCb; 67 class CameraServerPhotoProxy; 68 class AvcodecTaskManagerIntf; 69 class AudioCapturerSessionIntf; 70 class MovingPhotoVideoCacheIntf; 71 72 class StreamContainer { 73 public: StreamContainer()74 StreamContainer() {}; 75 virtual ~StreamContainer() = default; 76 77 bool AddStream(sptr<HStreamCommon> stream); 78 bool RemoveStream(sptr<HStreamCommon> stream); 79 sptr<HStreamCommon> GetStream(int32_t streamId); 80 sptr<HStreamCommon> GetHdiStream(int32_t streamId); 81 void Clear(); 82 size_t Size(); 83 84 std::list<sptr<HStreamCommon>> GetStreams(const StreamType streamType); 85 std::list<sptr<HStreamCommon>> GetAllStreams(); 86 87 private: 88 std::mutex streamsLock_; 89 std::map<const StreamType, std::list<sptr<HStreamCommon>>> streams_; 90 }; 91 92 class SessionDrainImageCallback; 93 using MetaElementType = std::pair<int64_t, sptr<SurfaceBuffer>>; 94 class MovingPhotoListener : public MovingPhotoSurfaceWrapper::SurfaceBufferListener { 95 public: 96 explicit MovingPhotoListener(sptr<MovingPhotoSurfaceWrapper> surfaceWrapper, wptr<Surface> metaSurface, 97 shared_ptr<FixedSizeList<MetaElementType>> metaCache, uint32_t preCacheFrameCount, 98 uint32_t postCacheFrameCount); 99 ~MovingPhotoListener() override; 100 void OnBufferArrival(sptr<SurfaceBuffer> buffer, int64_t timestamp, GraphicTransformType transform) override; 101 void DrainOutImage(sptr<SessionDrainImageCallback> drainImageCallback); 102 void RemoveDrainImageManager(sptr<SessionDrainImageCallback> drainImageCallback); 103 void StopDrainOut(); 104 void ClearCache(uint64_t timestamp); 105 void SetClearFlag(); 106 107 private: 108 sptr<MovingPhotoSurfaceWrapper> movingPhotoSurfaceWrapper_; 109 wptr<Surface> metaSurface_; 110 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 111 BlockingQueue<sptr<FrameRecord>> recorderBufferQueue_; 112 SafeMap<sptr<SessionDrainImageCallback>, sptr<DrainImageManager>> callbackMap_; 113 std::atomic<bool> isNeededClear_ { false }; 114 std::atomic<bool> isNeededPop_ { false }; 115 int64_t shutterTime_; 116 uint64_t postCacheFrameCount_; 117 }; 118 119 class MovingPhotoMetaListener : public IBufferConsumerListener { 120 public: 121 explicit MovingPhotoMetaListener(wptr<Surface> surface, shared_ptr<FixedSizeList<MetaElementType>> metaCache); 122 ~MovingPhotoMetaListener(); 123 void OnBufferAvailable() override; 124 private: 125 wptr<Surface> surface_; 126 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 127 }; 128 129 class SessionDrainImageCallback : public DrainImageCallback { 130 public: 131 explicit SessionDrainImageCallback(std::vector<sptr<FrameRecord>>& frameCacheList, 132 wptr<MovingPhotoListener> listener, 133 wptr<MovingPhotoVideoCacheIntf> movingPhotoVideoCacheIntf, 134 uint64_t timestamp, 135 int32_t rotation, 136 int32_t captureId); 137 ~SessionDrainImageCallback(); 138 void OnDrainImage(sptr<FrameRecord> frame) override; 139 void OnDrainImageFinish(bool isFinished) override; 140 141 private: 142 std::mutex mutex_; 143 std::vector<sptr<FrameRecord>> frameCacheList_; 144 wptr<MovingPhotoListener> listener_; 145 wptr<MovingPhotoVideoCacheIntf> movingPhotoVideoCacheIntf_; 146 uint64_t timestamp_; 147 int32_t rotation_; 148 int32_t captureId_; 149 }; 150 151 class CameraInfoDumper; 152 153 class EXPORT_API HStreamOperator : public OHOS::HDI::Camera::V1_3::IStreamOperatorCallback { 154 public: 155 static sptr<HStreamOperator> NewInstance(const uint32_t callerToken, int32_t opMode); 156 HStreamOperator(); 157 explicit HStreamOperator(const uint32_t callingTokenId, int32_t opMode); 158 virtual ~HStreamOperator(); 159 int32_t AddOutput(StreamType streamType, sptr<IStreamCommon> stream); 160 int32_t Stop(); 161 int32_t Release(); 162 int32_t EnableMovingPhoto(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 163 bool isEnable, int32_t sensorOritation); 164 int32_t GetCurrentStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 165 std::list<sptr<HStreamCommon>> GetAllStreams(); 166 int32_t EnableMovingPhotoMirror(bool isMirror, bool isConfig); 167 int32_t CreateMediaLibrary(const sptr<CameraServerPhotoProxy> &photoProxy, std::string &uri, 168 int32_t &cameraShotType, std::string &burstKey, int64_t timestamp); 169 int32_t CreateMediaLibrary(std::shared_ptr<PictureIntf> picture, const sptr<CameraServerPhotoProxy> &photoProxy, 170 std::string &uri, int32_t &cameraShotType, std::string& burstKey, int64_t timestamp); 171 void SetCameraPhotoProxyInfo(sptr<CameraServerPhotoProxy> cameraPhotoProxy, int32_t &cameraShotType, 172 bool &isBursting, std::string &burstKey); 173 int32_t LinkInputAndOutputs(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, int32_t opMode); 174 const sptr<HStreamCommon> GetStreamByStreamID(int32_t streamId); 175 const sptr<HStreamCommon> GetHdiStreamByStreamID(int32_t streamId); 176 void SetSensorRotation(int32_t rotationValue, int32_t sensorOrientation, int32_t cameraPosition); 177 void StartMovingPhotoEncode(int32_t rotation, uint64_t timestamp, int32_t format, int32_t captureId); 178 void StartRecord(uint64_t timestamp, int32_t rotation, int32_t captureId); 179 void GetOutputStatus(int32_t &status); 180 int32_t SetPreviewRotation(const std::string &deviceClass); 181 void ReleaseStreams(); 182 void StopMovingPhoto(); 183 int32_t GetActiveColorSpace(ColorSpace& colorSpace); 184 int32_t SetColorSpace(ColorSpace colorSpace, bool isNeedUpdate); 185 void SetColorSpaceForStreams(); 186 int32_t CheckIfColorSpaceMatchesFormat(ColorSpace colorSpace); 187 int32_t StartPreviewStream(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 188 camera_position_enum_t cameraPosition); 189 int32_t UpdateSettingForFocusTrackingMech(bool isEnableMech); 190 191 int32_t CreateStreams(std::vector<HDI::Camera::V1_1::StreamInfo_V1_1>& streamInfos); 192 sptr<OHOS::HDI::Camera::V1_1::IStreamOperator> GetStreamOperatorV1_1( 193 sptr<OHOS::HDI::Camera::V1_0::IStreamOperator>& streamOperator); 194 int32_t CommitStreams(const std::shared_ptr<OHOS::Camera::CameraMetadata>& deviceSettings, int32_t operationMode); 195 int32_t ReleaseStreams(std::vector<int32_t>& releaseStreamIds); 196 int32_t GetStreamsSize(); 197 int32_t CreateAndCommitStreams(std::vector<HDI::Camera::V1_1::StreamInfo_V1_1>& streamInfos, 198 const std::shared_ptr<OHOS::Camera::CameraMetadata>& deviceSettings, int32_t operationMode); 199 int32_t UpdateStreams(std::vector<StreamInfo_V1_1>& streamInfos); 200 int32_t UpdateStreamInfos(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings); 201 int32_t RemoveOutput(StreamType streamType, sptr<IStreamCommon> stream); 202 int32_t RemoveOutputStream(sptr<HStreamCommon> stream); 203 204 int32_t OnCaptureStarted(int32_t captureId, const std::vector<int32_t>& streamIds) override; 205 int32_t OnCaptureStarted_V1_2( 206 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_2::CaptureStartedInfo>& infos) override; 207 int32_t OnCaptureEnded(int32_t captureId, const std::vector<CaptureEndedInfo>& infos) override; 208 int32_t OnCaptureEndedExt( 209 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_3::CaptureEndedInfoExt>& infos) override; 210 int32_t OnCaptureError(int32_t captureId, const std::vector<CaptureErrorInfo>& infos) override; 211 int32_t OnFrameShutter(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 212 int32_t OnFrameShutterEnd(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 213 int32_t OnCaptureReady(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 214 int32_t OnResult(int32_t streamId, const std::vector<uint8_t>& result) override; 215 int32_t UnlinkInputAndOutputs(); 216 void ClearSketchRepeatStream(); 217 void ExpandSketchRepeatStream(); 218 void ExpandMovingPhotoRepeatStream(); 219 void ClearMovingPhotoRepeatStream(); 220 void GetStreamOperator(); 221 bool IsOfflineCapture(); 222 bool GetDeviceAbilityByMeta(uint32_t item, camera_metadata_item_t* metadataItem); GetPid()223 inline int32_t GetPid() 224 { 225 return static_cast<int32_t>(pid_); 226 } ResetHdiStreamId()227 inline void ResetHdiStreamId() 228 { 229 hdiStreamIdGenerator_ = HDI_STREAM_ID_INIT; 230 } SetCameraDevice(sptr<HCameraDevice> device)231 inline void SetCameraDevice(sptr<HCameraDevice> device) 232 { 233 std::lock_guard<std::mutex> lock(cameraDeviceLock_); 234 cameraDevice_ = device; 235 if (device == nullptr && !IsOfflineCapture()) { 236 ResetHDIStreamOperator(); 237 } 238 } 239 SetStreamOperatorId(int32_t & streamOperatorId)240 inline void SetStreamOperatorId(int32_t& streamOperatorId) 241 { 242 streamOperatorId_ = streamOperatorId; 243 } 244 GetHDIStreamOperator()245 inline sptr<OHOS::HDI::Camera::V1_0::IStreamOperator> GetHDIStreamOperator() 246 { 247 std::lock_guard<std::mutex> lock(streamOperatorLock_); 248 return streamOperator_; 249 } 250 ResetHDIStreamOperator()251 inline void ResetHDIStreamOperator() 252 { 253 std::lock_guard<std::mutex> lock(streamOperatorLock_); 254 streamOperator_ = nullptr; 255 } 256 257 void StartMovingPhotoStream(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings); 258 int32_t GetOfflineOutptSize(); 259 int32_t GetAllOutptSize(); 260 261 uint32_t preCacheFrameCount_ = CACHE_FRAME_COUNT; 262 uint32_t postCacheFrameCount_ = CACHE_FRAME_COUNT; 263 sptr<AvcodecTaskManagerIntf> avcodecTaskManagerProxy_; 264 sptr<AudioCapturerSessionIntf> audioCapturerSessionProxy_; 265 sptr<MovingPhotoVideoCacheIntf> movingPhotoVideoCacheProxy_; 266 std::vector<int32_t> GetFrameRateRange(); 267 268 class DisplayRotationListener : public OHOS::Rosen::DisplayManager::IDisplayListener { 269 public: DisplayRotationListener()270 explicit DisplayRotationListener() {}; 271 virtual ~DisplayRotationListener() = default; OnCreate(Rosen::DisplayId)272 void OnCreate(Rosen::DisplayId) override {} OnDestroy(Rosen::DisplayId)273 void OnDestroy(Rosen::DisplayId) override {} 274 void OnChange(Rosen::DisplayId displayId) override; 275 276 void AddHstreamRepeatForListener(sptr<HStreamRepeat> repeatStream); 277 278 void RemoveHstreamRepeatForListener(sptr<HStreamRepeat> repeatStream); 279 280 public: 281 std::list<sptr<HStreamRepeat>> repeatStreamList_; 282 std::mutex mStreamManagerLock_; 283 }; 284 285 void UpdateOrientationBaseGravity(int32_t rotationValue, int32_t sensorOrientation, 286 int32_t cameraPosition, int32_t& rotation); 287 288 void SetMechCallback(std::function<void(int32_t, 289 const std::shared_ptr<OHOS::Camera::CameraMetadata>&)> callback); 290 std::mutex mechCallbackLock_; 291 std::function<void(int32_t, const std::shared_ptr<OHOS::Camera::CameraMetadata>&)> mechCallback_; 292 293 private: 294 int32_t Initialize(const uint32_t callerToken, int32_t opMode); 295 void RegisterDisplayListener(sptr<HStreamRepeat> repeat); 296 void UnRegisterDisplayListener(sptr<HStreamRepeat> repeat); 297 void SetBasicInfo(sptr<HStreamCommon> stream); 298 string lastDisplayName_ = ""; 299 string lastBurstPrefix_ = ""; 300 int32_t saveIndex = 0; 301 int32_t streamOperatorId_ = -1; 302 volatile bool isMovingPhotoMirror_ = false; 303 volatile bool isSetMotionPhoto_ = false; 304 std::mutex livePhotoStreamLock_; // Guard livePhotoStreamRepeat_ 305 std::mutex opMutex_; // Lock the operations updateSettings_, streamOperator_, and hdiCameraDevice 306 sptr<HStreamRepeat> livePhotoStreamRepeat_; 307 std::atomic<int32_t> hdiStreamIdGenerator_ = HDI_STREAM_ID_INIT; 308 int32_t deviceSensorOritation_ = 0; 309 GenerateHdiStreamId()310 inline int32_t GenerateHdiStreamId() 311 { 312 return hdiStreamIdGenerator_.fetch_add(1); 313 } 314 315 string CreateDisplayName(const std::string& suffix); 316 string CreateBurstDisplayName(int32_t imageSeqId, int32_t seqId); 317 int32_t AddOutputStream(sptr<HStreamCommon> stream); 318 319 int32_t CreateMovingPhotoStreamRepeat(int32_t format, int32_t width, int32_t height, 320 sptr<OHOS::IBufferProducer> producer); 321 void CancelStreamsAndGetStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 322 void RestartStreams(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings); 323 void StartMovingPhoto(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 324 sptr<HStreamRepeat>& curStreamRepeat); 325 void ProcessMetaZoomArray(std::vector<uint32_t>& zoomAndTimeArray, sptr<HCameraDevice>& cameraDevice); 326 bool InitAudioCapture(); 327 bool StartAudioCapture(); 328 void ProcessAudioBuffer(); 329 void StartOnceRecord(uint64_t timestamp, int32_t rotation, int32_t captureId); 330 void UpdateMuteSetting(bool muteMode, std::shared_ptr<OHOS::Camera::CameraMetadata> &settings); 331 int32_t GetMovingPhotoBufferDuration(); 332 void GetMovingPhotoStartAndEndTime(); 333 bool IsIpsRotateSupported(); 334 void ConfigPayload(uint32_t pid, uint32_t tid, const char *bundleName, int32_t qosLevel, 335 std::unordered_map<std::string, std::string> &mapPayload); 336 std::shared_ptr<PhotoAssetIntf> ProcessPhotoProxy(int32_t captureId, std::shared_ptr<PictureIntf> picturePtr, 337 bool isBursting, sptr<CameraServerPhotoProxy> cameraPhotoProxy, std::string& uri); 338 void InitDefaultColortSpace(SceneMode opMode); 339 340 #ifdef CAMERA_USE_SENSOR 341 std::mutex sensorLock_; 342 bool isRegisterSensorSuccess_ = false; 343 void RegisterSensorCallback(); 344 void UnRegisterSensorCallback(); 345 static void GravityDataCallbackImpl(SensorEvent* event); 346 static int32_t CalcSensorRotation(int32_t sensorDegree); 347 static int32_t CalcRotationDegree(GravityData data); 348 #endif 349 // Make sure device thread safe,set device by {SetCameraDevice}, get device by {GetCameraDevice} 350 std::mutex cameraDeviceLock_; 351 std::mutex cbMutex_; 352 sptr<HCameraDevice> cameraDevice_; 353 StreamContainer streamContainer_; 354 StreamContainer streamContainerOffline_; 355 #ifdef CAMERA_USE_SENSOR 356 SensorUser user = { "", nullptr, nullptr }; 357 #endif 358 pid_t pid_; 359 uid_t uid_; 360 uint32_t callerToken_; 361 int32_t opMode_; 362 ColorSpace currColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; 363 bool isSessionStarted_ = false; 364 bool enableStreamRotate_ = false; 365 bool isDynamicConfiged_ = false; 366 std::string deviceClass_ = "phone"; 367 std::mutex movingPhotoStatusLock_; // Guard movingPhotoStatus 368 std::mutex releaseOperatorLock_; 369 std::mutex streamOperatorLock_; 370 sptr<MovingPhotoListener> livephotoListener_; 371 sptr<MovingPhotoMetaListener> livephotoMetaListener_; 372 sptr<Surface> metaSurface_ = nullptr; 373 std::mutex displayListenerLock_; 374 sptr<DisplayRotationListener> displayListener_; 375 sptr<OHOS::HDI::Camera::V1_0::IStreamOperator> streamOperator_ = nullptr; 376 bool isOfflineStreamOperator_ = false; 377 int32_t mlastCaptureId = 0; 378 int32_t sensorRotation_ = 0; 379 std::map<int32_t, bool> curMotionPhotoStatus_; 380 std::mutex motionPhotoStatusLock_; 381 }; 382 } // namespace CameraStandard 383 } // namespace OHOS 384 #endif // OHOS_CAMERA_H_STREAM_OPERATOR_H 385