1 /* 2 * Copyright (C) 2024 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef IMAGE_EFFECT_IMAGE_EFFECT_H 17 #define IMAGE_EFFECT_IMAGE_EFFECT_H 18 19 #include <vector> 20 #include <mutex> 21 #include <unordered_set> 22 23 #include "any.h" 24 #include "effect.h" 25 #include "external_window.h" 26 #include "image_type.h" 27 #include "surface.h" 28 #include "pixel_map.h" 29 #include "image_effect_marco_define.h" 30 #include "render_thread.h" 31 #include "picture.h" 32 33 namespace OHOS { 34 namespace Media { 35 namespace Effect { 36 struct SurfaceBufferInfo { 37 SurfaceBuffer *surfaceBuffer_ = nullptr; 38 int64_t timestamp_ = 0; 39 }; 40 41 struct DataInfo { 42 DataType dataType_ = DataType::UNKNOWN; 43 PixelMap *pixelMap_ = nullptr; 44 SurfaceBufferInfo surfaceBufferInfo_; 45 std::string uri_; 46 std::string path_; 47 Picture *picture_ = nullptr; 48 }; 49 50 struct BufferProcessInfo { 51 sptr<SurfaceBuffer> inBuffer_; 52 sptr<SurfaceBuffer> outBuffer_; 53 sptr<SyncFence> inBufferSyncFence_; 54 sptr<SyncFence> outBufferSyncFence_; 55 bool isSrcHebcData_ = false; 56 }; 57 58 class ImageEffect : public Effect { 59 public: 60 IMAGE_EFFECT_EXPORT ImageEffect(const char *name = nullptr); 61 IMAGE_EFFECT_EXPORT ~ImageEffect(); 62 63 IMAGE_EFFECT_EXPORT void AddEFilter(const std::shared_ptr<EFilter> &effect) override; 64 65 IMAGE_EFFECT_EXPORT ErrorCode InsertEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index) override; 66 67 IMAGE_EFFECT_EXPORT void RemoveEFilter(const std::shared_ptr<EFilter> &efilter) override; 68 IMAGE_EFFECT_EXPORT ErrorCode RemoveEFilter(uint32_t index) override; 69 70 IMAGE_EFFECT_EXPORT ErrorCode ReplaceEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index) override; 71 72 IMAGE_EFFECT_EXPORT virtual ErrorCode SetInputPixelMap(PixelMap *pixelMap); 73 74 IMAGE_EFFECT_EXPORT ErrorCode Start() override; 75 76 IMAGE_EFFECT_EXPORT ErrorCode Save(EffectJsonPtr &res) override; 77 78 IMAGE_EFFECT_EXPORT static std::shared_ptr<ImageEffect> Restore(std::string &info); 79 80 IMAGE_EFFECT_EXPORT virtual ErrorCode SetOutputPixelMap(PixelMap *pixelMap); 81 82 IMAGE_EFFECT_EXPORT virtual ErrorCode SetOutputSurface(sptr<Surface> &surface); 83 84 IMAGE_EFFECT_EXPORT virtual ErrorCode SetOutNativeWindow(OHNativeWindow *nativeWindow); 85 IMAGE_EFFECT_EXPORT sptr<Surface> GetInputSurface(); 86 87 IMAGE_EFFECT_EXPORT virtual ErrorCode Configure(const std::string &key, const Plugin::Any &value); 88 89 IMAGE_EFFECT_EXPORT void Stop(); 90 91 IMAGE_EFFECT_EXPORT ErrorCode SetInputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer); 92 93 IMAGE_EFFECT_EXPORT ErrorCode SetOutputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer); 94 95 IMAGE_EFFECT_EXPORT ErrorCode SetInputUri(const std::string &uri); 96 97 IMAGE_EFFECT_EXPORT ErrorCode SetOutputUri(const std::string &uri); 98 99 IMAGE_EFFECT_EXPORT ErrorCode SetInputPath(const std::string &path); 100 101 IMAGE_EFFECT_EXPORT ErrorCode SetOutputPath(const std::string &path); 102 103 IMAGE_EFFECT_EXPORT ErrorCode SetExtraInfo(EffectJsonPtr res); 104 105 IMAGE_EFFECT_EXPORT ErrorCode SetInputPicture(Picture *picture); 106 107 IMAGE_EFFECT_EXPORT ErrorCode SetOutputPicture(Picture *picture); 108 109 protected: 110 IMAGE_EFFECT_EXPORT virtual ErrorCode Render(); 111 112 IMAGE_EFFECT_EXPORT static void ClearDataInfo(DataInfo &dataInfo); 113 114 IMAGE_EFFECT_EXPORT static ErrorCode ParseDataInfo(DataInfo &dataInfo, std::shared_ptr<EffectBuffer> &effectBuffer, 115 bool isOutputData, IEffectFormat format); 116 117 DataInfo inDateInfo_; 118 DataInfo outDateInfo_; 119 120 private: 121 ErrorCode LockAll(std::shared_ptr<EffectBuffer> &srcEffectBuffer, std::shared_ptr<EffectBuffer> &dstEffectBuffer, 122 IEffectFormat format); 123 124 static void UnLockData(DataInfo &dataInfo); 125 IMAGE_EFFECT_EXPORT static BufferRequestConfig GetBufferRequestConfig(const sptr<SurfaceBuffer>& buffer); 126 127 void UnLockAll(); 128 129 void InitEGLEnv(); 130 131 void DestroyEGLEnv(); 132 133 IMAGE_EFFECT_EXPORT 134 void ConsumerBufferAvailable(); 135 IMAGE_EFFECT_EXPORT void UpdateProducerSurfaceInfo(); 136 137 void ExtInitModule(); 138 void ExtDeinitModule(); 139 140 unsigned long int RequestTaskId(); 141 142 void ConsumerBufferWithGPU(sptr<SurfaceBuffer>& buffer); 143 void OnBufferAvailableWithCPU(); 144 IMAGE_EFFECT_EXPORT bool RenderBuffer(sptr<SurfaceBuffer> &inBuffer, sptr<SurfaceBuffer> &outBuffer, 145 int64_t ×tamp); 146 IMAGE_EFFECT_EXPORT GSError FlushBuffer(sptr<SurfaceBuffer> &buffer, sptr<SyncFence> &fence, bool isNeedAttach, 147 bool sendFence, int64_t ×tamp); 148 IMAGE_EFFECT_EXPORT GSError ReleaseBuffer(sptr<SurfaceBuffer> &buffer, sptr<SyncFence> &fence); 149 IMAGE_EFFECT_EXPORT void ProcessRender(BufferProcessInfo &bufferProcessInfo, bool &isNeedSwap, int64_t ×tamp); 150 IMAGE_EFFECT_EXPORT void ProcessSwapBuffers(BufferProcessInfo &bufferProcessInfo, int64_t ×tamp); 151 152 ErrorCode GetImageInfo(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 153 std::shared_ptr<ExifMetadata> &exifMetadata); 154 ErrorCode ConfigureFilters(std::shared_ptr<EffectBuffer> srcEffectBuffer, 155 std::shared_ptr<EffectBuffer> dstEffectBuffer); 156 ErrorCode GetImageInfoFromPixelMap(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 157 std::shared_ptr<ExifMetadata> &exifMetadata) const; 158 ErrorCode GetImageInfoFromSurface(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat) const; 159 ErrorCode GetImageInfoFromPath(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 160 std::shared_ptr<ExifMetadata> &exifMetadata) const; 161 ErrorCode GetImageInfoFromPicture(uint32_t &width, uint32_t &height, PixelFormat &pixelFormat, 162 std::shared_ptr<ExifMetadata> &exifMetadata) const; 163 164 void UpdateConsumedrBuffersNumber(); 165 void UpdateCycleBuffersNumber(); 166 167 void SetPathToSink(); 168 169 sptr<Surface> toProducerSurface_; // from ImageEffect to XComponent 170 sptr<Surface> fromProducerSurface_; // to camera hal 171 volatile int32_t imageEffectFlag_ = 0; 172 bool setCycleBuffersNumber_ = false; 173 bool setConsumerBufferSize_ = false; 174 175 GraphicTransformType toProducerTransform_ = GRAPHIC_ROTATE_BUTT; 176 177 // envSupportIpType 178 std::vector<IPType> ipType_ = { 179 { IPType::CPU, IPType::GPU }, 180 }; 181 182 std::map<ConfigType, Plugin::Any> config_ = { { ConfigType::IPTYPE, ipType_ } }; 183 184 EffectJsonPtr extraInfo_ = nullptr; 185 186 std::string name_; 187 188 class Impl; 189 std::shared_ptr<Impl> impl_; 190 std::mutex innerEffectMutex_; 191 RenderThread<> *m_renderThread{ nullptr }; 192 std::atomic_ullong m_currentTaskId{0}; 193 bool needPreFlush_ = false; 194 uint32_t failureCount_ = 0; 195 }; 196 197 IMAGE_EFFECT_EXPORT void MemoryCopyForSurfaceBuffer(sptr<SurfaceBuffer> &buffer, OHOS::sptr<SurfaceBuffer> &outBuffer); 198 IMAGE_EFFECT_EXPORT bool IsSurfaceBufferHebc(sptr<SurfaceBuffer> &buffer); 199 200 } // namespace Effect 201 } // namespace Media 202 } // namespace OHOS 203 #endif // IMAGE_EFFECT_IMAGE_EFFECT_H 204