1 /* 2 * Copyright (C) 2024 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef IMAGE_EFFECT_IMAGE_EFFECT_H 17 #define IMAGE_EFFECT_IMAGE_EFFECT_H 18 19 #include <vector> 20 #include <mutex> 21 #include <unordered_set> 22 #include <queue> 23 #include <optional> 24 #include <condition_variable> 25 #include <utility> 26 27 #include "any.h" 28 #include "effect.h" 29 #include "external_window.h" 30 #include "image_type.h" 31 #include "surface.h" 32 #include "pixel_map.h" 33 #include "image_effect_marco_define.h" 34 #include "render_thread.h" 35 #include "picture.h" 36 37 #define TIME_FOR_WAITING_BUFFER 2500 38 39 namespace OHOS { 40 namespace Media { 41 namespace Effect { 42 struct SurfaceBufferInfo { 43 SurfaceBuffer *surfaceBuffer_ = nullptr; 44 int64_t timestamp_ = 0; 45 }; 46 47 struct TextureInfo { 48 int32_t textureId_ = 0; 49 int32_t colorSpace_ = 0; 50 }; 51 52 struct DataInfo { 53 DataType dataType_ = DataType::UNKNOWN; 54 PixelMap *pixelMap_ = nullptr; 55 SurfaceBufferInfo surfaceBufferInfo_; 56 TextureInfo textureInfo_; 57 std::string uri_; 58 std::string path_; 59 Picture *picture_ = nullptr; 60 }; 61 62 struct BufferProcessInfo { 63 sptr<SurfaceBuffer> inBuffer_; 64 sptr<SurfaceBuffer> outBuffer_; 65 sptr<SyncFence> inBufferSyncFence_; 66 sptr<SyncFence> outBufferSyncFence_; 67 bool isSrcHebcData_ = false; 68 }; 69 70 struct BufferEntry { 71 uint32_t seqNum_; 72 sptr<SurfaceBuffer> buffer_; 73 sptr<SyncFence> syncFence_; 74 int64_t timestamp_; 75 }; 76 77 template <typename T> 78 class ThreadSafeBufferQueue { 79 public: 80 explicit ThreadSafeBufferQueue(size_t max_capacity = std::numeric_limits<size_t>::max()) 81 : max_capacity_(std::max<size_t>(1, max_capacity)) {} 82 83 ThreadSafeBufferQueue(const ThreadSafeBufferQueue&) = delete; 84 ThreadSafeBufferQueue& operator=(const ThreadSafeBufferQueue&) = delete; 85 86 bool TryPush(T&& element, bool wait = true, 87 std::chrono::milliseconds timeout = std::chrono::milliseconds(TIME_FOR_WAITING_BUFFER)) 88 { 89 std::unique_lock lock(mutex_); 90 91 if (!wait) { 92 return queue_.size() < max_capacity_ ? CommitPush(std::forward<T>(element), lock) : false; 93 } 94 95 if (!WaitForSpace(lock, timeout)) { 96 return false; 97 } 98 99 return CommitPush(std::forward<T>(element), lock); 100 } 101 102 std::optional<T> TryPop(bool wait = true, std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) 103 { 104 std::unique_lock lock(mutex_); 105 if (!wait) { 106 return !queue_.empty() ? CommitPop(lock) : std::nullopt; 107 } 108 109 if (!WaitForData(lock, timeout)) { 110 return std::nullopt; 111 } 112 113 return CommitPop(lock); 114 } 115 Size()116 size_t Size() const 117 { 118 std::lock_guard lock(mutex_); 119 return queue_.size(); 120 } 121 122 private: 123 template<typename Rep = int, typename Period = std::milli> WaitForSpace(std::unique_lock<std::mutex> & lock,const std::chrono::duration<Rep,Period> & timeout)124 bool WaitForSpace(std::unique_lock<std::mutex>& lock, const std::chrono::duration<Rep, Period>& timeout) 125 { 126 if (timeout > std::chrono::duration<Rep, Period>::zero()) { 127 return not_full_cv_.wait_for(lock, timeout, [this] { 128 return queue_.size() < max_capacity_; 129 }); 130 } 131 not_full_cv_.wait(lock, [this] { 132 return queue_.size() < max_capacity_; 133 }); 134 return true; 135 } 136 137 template<typename Rep = int, typename Period = std::milli> WaitForData(std::unique_lock<std::mutex> & lock,const std::chrono::duration<Rep,Period> & timeout)138 bool WaitForData(std::unique_lock<std::mutex>& lock, const std::chrono::duration<Rep, Period>& timeout) 139 { 140 if (timeout > std::chrono::duration<Rep, Period>::zero()) { 141 return not_empty_cv_.wait_for(lock, timeout, [this] { 142 return !queue_.empty(); 143 }); 144 } 145 not_empty_cv_.wait(lock, [this] { 146 return !queue_.empty(); 147 }); 148 return true; 149 } 150 CommitPush(T && element,std::unique_lock<std::mutex> & lock)151 bool CommitPush(T&& element, std::unique_lock<std::mutex>& lock) 152 { 153 queue_.emplace(std::forward<T>(element)); 154 lock.unlock(); 155 not_empty_cv_.notify_one(); 156 return true; 157 } 158 CommitPop(std::unique_lock<std::mutex> & lock)159 std::optional<T> CommitPop(std::unique_lock<std::mutex>& lock) 160 { 161 T element = std::move(queue_.front()); 162 queue_.pop(); 163 lock.unlock(); 164 not_full_cv_.notify_one(); 165 return element; 166 } 167 168 mutable std::mutex mutex_; 169 std::queue<T> queue_; 170 std::condition_variable not_full_cv_; 171 std::condition_variable not_empty_cv_; 172 const size_t max_capacity_; 173 }; 174 175 class ImageEffect : public Effect { 176 public: 177 IMAGE_EFFECT_EXPORT ImageEffect(const char *name = nullptr); 178 IMAGE_EFFECT_EXPORT ~ImageEffect(); 179 180 IMAGE_EFFECT_EXPORT void AddEFilter(const std::shared_ptr<EFilter> &effect) override; 181 182 IMAGE_EFFECT_EXPORT ErrorCode InsertEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index) override; 183 184 IMAGE_EFFECT_EXPORT void RemoveEFilter(const std::shared_ptr<EFilter> &efilter) override; 185 IMAGE_EFFECT_EXPORT ErrorCode RemoveEFilter(uint32_t index) override; 186 187 IMAGE_EFFECT_EXPORT ErrorCode ReplaceEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index) override; 188 189 IMAGE_EFFECT_EXPORT virtual ErrorCode SetInputPixelMap(PixelMap *pixelMap); 190 191 IMAGE_EFFECT_EXPORT ErrorCode Start() override; 192 193 IMAGE_EFFECT_EXPORT ErrorCode Save(EffectJsonPtr &res) override; 194 195 IMAGE_EFFECT_EXPORT static std::shared_ptr<ImageEffect> Restore(std::string &info); 196 197 IMAGE_EFFECT_EXPORT virtual ErrorCode SetOutputPixelMap(PixelMap *pixelMap); 198 199 IMAGE_EFFECT_EXPORT virtual ErrorCode SetOutputSurface(sptr<Surface> &surface); 200 201 IMAGE_EFFECT_EXPORT virtual ErrorCode SetOutNativeWindow(OHNativeWindow *nativeWindow); 202 IMAGE_EFFECT_EXPORT sptr<Surface> GetInputSurface(); 203 204 IMAGE_EFFECT_EXPORT virtual ErrorCode Configure(const std::string &key, const Any &value); 205 206 IMAGE_EFFECT_EXPORT void Stop(); 207 208 IMAGE_EFFECT_EXPORT ErrorCode SetInputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer); 209 210 IMAGE_EFFECT_EXPORT ErrorCode SetOutputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer); 211 212 IMAGE_EFFECT_EXPORT ErrorCode SetInputUri(const std::string &uri); 213 214 IMAGE_EFFECT_EXPORT ErrorCode SetOutputUri(const std::string &uri); 215 216 IMAGE_EFFECT_EXPORT ErrorCode SetInputPath(const std::string &path); 217 218 IMAGE_EFFECT_EXPORT ErrorCode SetOutputPath(const std::string &path); 219 220 IMAGE_EFFECT_EXPORT ErrorCode SetExtraInfo(EffectJsonPtr res); 221 222 IMAGE_EFFECT_EXPORT ErrorCode SetInputPicture(Picture *picture); 223 224 IMAGE_EFFECT_EXPORT ErrorCode SetOutputPicture(Picture *picture); 225 226 IMAGE_EFFECT_EXPORT ErrorCode SetInputTexture(int32_t textureId, int32_t colorSpace); 227 228 IMAGE_EFFECT_EXPORT ErrorCode SetOutputTexture(int32_t textureId); 229 230 protected: 231 IMAGE_EFFECT_EXPORT virtual ErrorCode Render(); 232 233 IMAGE_EFFECT_EXPORT static void ClearDataInfo(DataInfo &dataInfo); 234 235 IMAGE_EFFECT_EXPORT static ErrorCode ParseDataInfo(DataInfo &dataInfo, std::shared_ptr<EffectBuffer> &effectBuffer, 236 bool isOutputData, IEffectFormat format, LOG_STRATEGY strategy = LOG_STRATEGY::NORMAL); 237 238 DataInfo inDateInfo_; 239 DataInfo outDateInfo_; 240 241 private: 242 ErrorCode LockAll(std::shared_ptr<EffectBuffer> &srcEffectBuffer, std::shared_ptr<EffectBuffer> &dstEffectBuffer, 243 IEffectFormat format); 244 245 void RemoveGainMapIfNeed() const; 246 247 static void UnLockData(DataInfo &dataInfo); 248 static BufferRequestConfig GetBufferRequestConfig(const sptr<SurfaceBuffer>& buffer); 249 250 void UnLockAll(); 251 252 void InitEGLEnv(); 253 254 void DestroyEGLEnv(); 255 256 IMAGE_EFFECT_EXPORT 257 void ConsumerBufferAvailable(); 258 void UpdateProducerSurfaceInfo(); 259 260 void ExtInitModule(); 261 void ExtDeinitModule(); 262 263 unsigned long int RequestTaskId(); 264 265 void ConsumerBufferWithGPU(sptr<SurfaceBuffer>& buffer); 266 void OnBufferAvailableWithCPU(); 267 bool SubmitRenderTask(BufferEntry&& entry); 268 void RenderBuffer(); 269 GSError FlushBuffer(sptr<SurfaceBuffer>& buffer, sptr<SyncFence>& fence, bool isNeedAttach, bool sendFence, 270 int64_t& timestamp); 271 GSError ReleaseBuffer(sptr<SurfaceBuffer>& buffer, sptr<SyncFence>& fence); 272 void ProcessRender(BufferProcessInfo& bufferProcessInfo, bool& isNeedSwap, int64_t& timestamp); 273 void ProcessSwapBuffers(BufferProcessInfo& bufferProcessInfo, int64_t& timestamp); 274 275 ErrorCode GetImageInfo(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 276 std::shared_ptr<ExifMetadata> &exifMetadata); 277 ErrorCode ConfigureFilters(std::shared_ptr<EffectBuffer> srcEffectBuffer, 278 std::shared_ptr<EffectBuffer> dstEffectBuffer); 279 ErrorCode GetImageInfoFromPixelMap(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 280 std::shared_ptr<ExifMetadata> &exifMetadata) const; 281 ErrorCode GetImageInfoFromSurface(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat) const; 282 ErrorCode GetImageInfoFromPath(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 283 std::shared_ptr<ExifMetadata> &exifMetadata) const; 284 ErrorCode GetImageInfoFromPicture(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 285 std::shared_ptr<ExifMetadata> &exifMetadata) const; 286 ErrorCode GetImageInfoFromTexInfo(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat) const; 287 288 void UpdateConsumerBuffersNumber(); 289 void UpdateCycleBuffersNumber(); 290 291 void SetPathToSink(); 292 293 ErrorCode InitEffectBuffer(std::shared_ptr<EffectBuffer> &srcEffectBuffer, 294 std::shared_ptr<EffectBuffer> &dstEffectBuffer, IEffectFormat format); 295 296 sptr<Surface> toProducerSurface_; // from ImageEffect to XComponent 297 sptr<Surface> fromProducerSurface_; // to camera hal 298 volatile int32_t imageEffectFlag_ = 0; 299 bool setCycleBuffersNumber_ = false; 300 bool setConsumerBufferSize_ = false; 301 302 GraphicTransformType toProducerTransform_ = GRAPHIC_ROTATE_BUTT; 303 304 // envSupportIpType 305 std::vector<IPType> ipType_ = { 306 { IPType::CPU, IPType::GPU }, 307 }; 308 309 std::map<ConfigType, Any> config_ = { { ConfigType::IPTYPE, ipType_ } }; 310 311 EffectJsonPtr extraInfo_ = nullptr; 312 313 std::string name_; 314 315 class Impl; 316 std::shared_ptr<Impl> impl_; 317 std::mutex innerEffectMutex_; 318 std::mutex consumerListenerMutex_; 319 RenderThread<> *m_renderThread{ nullptr }; 320 std::atomic_ullong m_currentTaskId{0}; 321 bool needPreFlush_ = false; 322 uint32_t failureCount_ = 0; 323 std::shared_ptr<ThreadSafeBufferQueue<BufferEntry>> bufferPool_; 324 }; 325 } // namespace Effect 326 } // namespace Media 327 } // namespace OHOS 328 #endif // IMAGE_EFFECT_IMAGE_EFFECT_H 329