1 /*
2 * Copyright (c) 2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "photo_asset_buffer_consumer.h"
16
17 #include "camera_log.h"
18 #include "video_key_info.h"
19 #include "camera_surface_buffer_util.h"
20 #include "hstream_capture.h"
21 #include "task_manager.h"
22 #include "picture_assembler.h"
23 #include "dp_utils.h"
24 #include "camera_server_photo_proxy.h"
25 #include "picture_proxy.h"
26 #include "camera_report_dfx_uitls.h"
27
28 namespace OHOS {
29 namespace CameraStandard {
30
PhotoAssetBufferConsumer(wptr<HStreamCapture> streamCapture)31 PhotoAssetBufferConsumer::PhotoAssetBufferConsumer(wptr<HStreamCapture> streamCapture) : streamCapture_(streamCapture)
32 {
33 MEDIA_INFO_LOG("PhotoAssetBufferConsumer new E");
34 }
35
~PhotoAssetBufferConsumer()36 PhotoAssetBufferConsumer::~PhotoAssetBufferConsumer()
37 {
38 MEDIA_INFO_LOG("PhotoAssetBufferConsumer ~ E");
39 }
40
OnBufferAvailable()41 void PhotoAssetBufferConsumer::OnBufferAvailable()
42 {
43 MEDIA_INFO_LOG("OnBufferAvailable E");
44 CAMERA_SYNC_TRACE;
45 sptr<HStreamCapture> streamCapture = streamCapture_.promote();
46 CHECK_RETURN_ELOG(streamCapture == nullptr, "streamCapture is null");
47 CHECK_RETURN_ELOG(streamCapture->photoTask_ == nullptr, "photoTask is null");
48 wptr<PhotoAssetBufferConsumer> thisPtr(this);
49 streamCapture->photoTask_->SubmitTask([thisPtr]() {
50 auto listener = thisPtr.promote();
51 CHECK_EXECUTE(listener, listener->ExecuteOnBufferAvailable());
52 });
53
54 MEDIA_INFO_LOG("OnBufferAvailable X");
55 }
56
ExecuteOnBufferAvailable()57 void PhotoAssetBufferConsumer::ExecuteOnBufferAvailable()
58 {
59 MEDIA_INFO_LOG("PA_ExecuteOnBufferAvailable E");
60 CAMERA_SYNC_TRACE;
61 sptr<HStreamCapture> streamCapture = streamCapture_.promote();
62 CHECK_RETURN_ELOG(streamCapture == nullptr, "streamCapture is null");
63 CHECK_RETURN_ELOG(streamCapture->surface_ == nullptr, "surface is null");
64 sptr<SurfaceBuffer> surfaceBuffer = nullptr;
65 int32_t fence = -1;
66 int64_t timestamp;
67 OHOS::Rect damage;
68 SurfaceError surfaceRet = streamCapture->surface_->AcquireBuffer(surfaceBuffer, fence, timestamp, damage);
69 CHECK_RETURN_ELOG(surfaceRet != SURFACE_ERROR_OK, "Failed to acquire surface buffer");
70 CameraSurfaceBufferUtil::DumpSurfaceBuffer(surfaceBuffer);
71 // deep copy surfaceBuffer
72 sptr<SurfaceBuffer> newSurfaceBuffer = CameraSurfaceBufferUtil::DeepCopyBuffer(surfaceBuffer);
73 // release surfaceBuffer to bufferQueue
74 streamCapture->surface_->ReleaseBuffer(surfaceBuffer, -1);
75 CHECK_RETURN_ELOG(newSurfaceBuffer == nullptr, "DeepCopyBuffer faild");
76 int32_t captureId = CameraSurfaceBufferUtil::GetMaskCaptureId(newSurfaceBuffer);
77 CameraReportDfxUtils::GetInstance()->SetFirstBufferEndInfo(captureId);
78 CameraReportDfxUtils::GetInstance()->SetPrepareProxyStartInfo(captureId);
79 int32_t auxiliaryCount = CameraSurfaceBufferUtil::GetImageCount(newSurfaceBuffer);
80 MEDIA_INFO_LOG("OnBufferAvailable captureId:%{public}d auxiliaryCount:%{public}d", captureId, auxiliaryCount);
81 // create photoProxy
82 sptr<CameraServerPhotoProxy> cameraPhotoProxy = new CameraServerPhotoProxy();
83 cameraPhotoProxy->GetServerPhotoProxyInfo(newSurfaceBuffer);
84
85 std::string uri;
86 int32_t cameraShotType = 0;
87 std::string burstKey;
88 bool isYuv = streamCapture->isYuvCapture_;
89 MEDIA_INFO_LOG("CreateMediaLibrary captureId:%{public}d isYuv::%{public}d", captureId, isYuv);
90 if (isYuv) {
91 StartWaitAuxiliaryTask(captureId, auxiliaryCount, timestamp, newSurfaceBuffer);
92 } else {
93 streamCapture->CreateMediaLibrary(cameraPhotoProxy, uri, cameraShotType, burstKey, timestamp);
94 MEDIA_INFO_LOG("CreateMediaLibrary uri:%{public}s", uri.c_str());
95 streamCapture->OnPhotoAssetAvailable(captureId, uri, cameraShotType, burstKey);
96 }
97
98 MEDIA_INFO_LOG("PA_ExecuteOnBufferAvailable X");
99 }
100
StartWaitAuxiliaryTask(const int32_t captureId,const int32_t auxiliaryCount,int64_t timestamp,sptr<SurfaceBuffer> & newSurfaceBuffer)101 void PhotoAssetBufferConsumer::StartWaitAuxiliaryTask(
102 const int32_t captureId, const int32_t auxiliaryCount, int64_t timestamp, sptr<SurfaceBuffer> &newSurfaceBuffer)
103 {
104 CAMERA_SYNC_TRACE;
105 MEDIA_INFO_LOG("StartWaitAuxiliaryTask E, captureId:%{public}d", captureId);
106 sptr<HStreamCapture> streamCapture = streamCapture_.promote();
107 CHECK_RETURN_ELOG(streamCapture == nullptr, "streamCapture is null");
108 {
109 std::lock_guard<std::mutex> lock(streamCapture->g_photoImageMutex);
110 // create and save photoProxy
111 streamCapture->captureIdCountMap_[captureId] = auxiliaryCount;
112 streamCapture->captureIdAuxiliaryCountMap_[captureId]++;
113 MEDIA_INFO_LOG("PhotoAssetBufferConsumer StartWaitAuxiliaryTask 4 captureId = %{public}d", captureId);
114 sptr<CameraServerPhotoProxy> photoProxy = new CameraServerPhotoProxy();
115 photoProxy->GetServerPhotoProxyInfo(newSurfaceBuffer);
116 photoProxy->SetDisplayName(CreateDisplayName(suffixJpeg));
117 streamCapture->photoProxyMap_[captureId] = photoProxy;
118 MEDIA_INFO_LOG("PhotoAssetBufferConsumer StartWaitAuxiliaryTask 5");
119
120 // create and save pictureProxy
121 std::shared_ptr<PictureIntf> pictureProxy = PictureProxy::CreatePictureProxy();
122 CHECK_RETURN_ELOG(pictureProxy == nullptr, "pictureProxy is nullptr");
123 pictureProxy->Create(newSurfaceBuffer);
124 MEDIA_INFO_LOG(
125 "PhotoAssetBufferConsumer StartWaitAuxiliaryTask MainSurface w=%{public}d, h=%{public}d, f=%{public}d",
126 newSurfaceBuffer->GetWidth(), newSurfaceBuffer->GetHeight(), newSurfaceBuffer->GetFormat());
127 streamCapture->captureIdPictureMap_[captureId] = pictureProxy;
128
129 // all AuxiliaryBuffer ready, do assamble
130 if (streamCapture->captureIdCountMap_[captureId] != 0 &&
131 streamCapture->captureIdAuxiliaryCountMap_[captureId] == streamCapture->captureIdCountMap_[captureId]) {
132 MEDIA_INFO_LOG(
133 "PhotoAssetBufferConsumer StartWaitAuxiliaryTask auxiliaryCount is complete, StopMonitor DoTimeout "
134 "captureId = %{public}d", captureId);
135 AssembleDeferredPicture(timestamp, captureId);
136 } else {
137 // start timeer to do assamble
138 uint32_t pictureHandle;
139 constexpr uint32_t delayMilli = 1 * 1000;
140 MEDIA_INFO_LOG(
141 "PhotoAssetBufferConsumer StartWaitAuxiliaryTask GetGlobalWatchdog StartMonitor, captureId=%{public}d",
142 captureId);
143 auto thisPtr = wptr<PhotoAssetBufferConsumer>(this);
144 DeferredProcessing::GetGlobalWatchdog().StartMonitor(
145 pictureHandle, delayMilli, [thisPtr, captureId, timestamp](uint32_t handle) {
146 MEDIA_INFO_LOG(
147 "PhotoAssetBufferConsumer PhotoAssetBufferConsumer-Watchdog executed, handle: %{public}d, "
148 "captureId=%{public}d", static_cast<int>(handle), captureId);
149 auto ptr = thisPtr.promote();
150 CHECK_RETURN(ptr == nullptr);
151 ptr->AssembleDeferredPicture(timestamp, captureId);
152 auto streamCapture = ptr->streamCapture_.promote();
153 if (streamCapture && streamCapture->captureIdAuxiliaryCountMap_.count(captureId)) {
154 streamCapture->captureIdAuxiliaryCountMap_[captureId] = -1;
155 MEDIA_INFO_LOG(
156 "PhotoAssetBufferConsumer StartWaitAuxiliaryTask captureIdAuxiliaryCountMap_ = -1, "
157 "captureId=%{public}d", captureId);
158 }
159 });
160 streamCapture->captureIdHandleMap_[captureId] = pictureHandle;
161 MEDIA_INFO_LOG(
162 "PhotoAssetBufferConsumer StartWaitAuxiliaryTask, pictureHandle: %{public}d, captureId=%{public}d "
163 "captureIdCountMap = %{public}d, captureIdAuxiliaryCountMap = %{public}d",
164 pictureHandle, captureId, streamCapture->captureIdCountMap_[captureId],
165 streamCapture->captureIdAuxiliaryCountMap_[captureId]);
166 }
167 }
168 MEDIA_INFO_LOG("StartWaitAuxiliaryTask X");
169 }
170
LoggingSurfaceBufferInfo(sptr<SurfaceBuffer> buffer,std::string bufName)171 inline void LoggingSurfaceBufferInfo(sptr<SurfaceBuffer> buffer, std::string bufName)
172 {
173 if (buffer) {
174 MEDIA_INFO_LOG("LoggingSurfaceBufferInfo %{public}s w=%{public}d, h=%{public}d, f=%{public}d",
175 bufName.c_str(), buffer->GetWidth(), buffer->GetHeight(), buffer->GetFormat());
176 }
177 };
178
CleanAfterTransPicture(int32_t captureId)179 void PhotoAssetBufferConsumer::CleanAfterTransPicture(int32_t captureId)
180 {
181 MEDIA_INFO_LOG("CleanAfterTransPicture E, captureId:%{public}d", captureId);
182 sptr<HStreamCapture> streamCapture = streamCapture_.promote();
183 CHECK_RETURN_ELOG(streamCapture == nullptr, "streamCapture is null");
184
185 streamCapture->photoProxyMap_.erase(captureId);
186 streamCapture->captureIdPictureMap_.erase(captureId);
187 streamCapture->captureIdGainmapMap_.erase(captureId);
188 streamCapture->captureIdDepthMap_.Erase(captureId);
189 streamCapture->captureIdExifMap_.erase(captureId);
190 streamCapture->captureIdDebugMap_.erase(captureId);
191 streamCapture->captureIdAuxiliaryCountMap_.erase(captureId);
192 streamCapture->captureIdCountMap_.erase(captureId);
193 streamCapture->captureIdHandleMap_.erase(captureId);
194 }
195
AssembleDeferredPicture(int64_t timestamp,int32_t captureId)196 void PhotoAssetBufferConsumer::AssembleDeferredPicture(int64_t timestamp, int32_t captureId)
197 {
198 CAMERA_SYNC_TRACE;
199 MEDIA_INFO_LOG("AssembleDeferredPicture E, captureId:%{public}d", captureId);
200 sptr<HStreamCapture> streamCapture = streamCapture_.promote();
201 CHECK_RETURN_ELOG(streamCapture == nullptr, "streamCapture is null");
202 std::lock_guard<std::mutex> lock(streamCapture->g_assembleImageMutex);
203 std::shared_ptr<PictureIntf> picture = streamCapture->captureIdPictureMap_[captureId];
204 if (streamCapture->captureIdExifMap_[captureId] && picture) {
205 MEDIA_INFO_LOG("AssembleDeferredPicture exifSurfaceBuffer");
206 auto buffer = streamCapture->captureIdExifMap_[captureId];
207 LoggingSurfaceBufferInfo(buffer, "exifSurfaceBuffer");
208 picture->SetExifMetadata(buffer);
209 streamCapture->captureIdExifMap_[captureId] = nullptr;
210 }
211 if (streamCapture->captureIdGainmapMap_[captureId] && picture) {
212 MEDIA_INFO_LOG("AssembleDeferredPicture gainmapSurfaceBuffer");
213 LoggingSurfaceBufferInfo(streamCapture->captureIdGainmapMap_[captureId], "gainmapSurfaceBuffer");
214 picture->SetAuxiliaryPicture(
215 streamCapture->captureIdGainmapMap_[captureId], CameraAuxiliaryPictureType::GAINMAP);
216 streamCapture->captureIdGainmapMap_[captureId] = nullptr;
217 }
218 sptr<SurfaceBuffer> depthBuffer = nullptr;
219 streamCapture->captureIdDepthMap_.FindOldAndSetNew(captureId, depthBuffer, nullptr);
220 if (depthBuffer && picture) {
221 MEDIA_INFO_LOG("AssembleDeferredPicture deepSurfaceBuffer");
222 LoggingSurfaceBufferInfo(depthBuffer, "deepSurfaceBuffer");
223 picture->SetAuxiliaryPicture(depthBuffer, CameraAuxiliaryPictureType::DEPTH_MAP);
224 }
225 if (streamCapture->captureIdDebugMap_[captureId] && picture) {
226 MEDIA_INFO_LOG("AssembleDeferredPicture debugSurfaceBuffer");
227 auto buffer = streamCapture->captureIdDebugMap_[captureId];
228 LoggingSurfaceBufferInfo(buffer, "debugSurfaceBuffer");
229 picture->SetMaintenanceData(buffer);
230 streamCapture->captureIdDebugMap_[captureId] = nullptr;
231 }
232 CHECK_RETURN_ELOG(!picture, "CreateMediaLibrary picture is nullptr");
233 std::string uri;
234 int32_t cameraShotType;
235 std::string burstKey = "";
236 MEDIA_DEBUG_LOG("AssembleDeferredPicture CreateMediaLibrary E");
237 streamCapture->CreateMediaLibrary(
238 picture, streamCapture->photoProxyMap_[captureId], uri, cameraShotType, burstKey, timestamp);
239 MEDIA_DEBUG_LOG("AssembleDeferredPicture CreateMediaLibrary X");
240 MEDIA_INFO_LOG("CreateMediaLibrary result %{public}s, type %{public}d", uri.c_str(), cameraShotType);
241 streamCapture->OnPhotoAssetAvailable(captureId, uri, cameraShotType, burstKey);
242 CleanAfterTransPicture(captureId);
243 MEDIA_INFO_LOG("AssembleDeferredPicture X, captureId:%{public}d", captureId);
244 }
245 } // namespace CameraStandard
246 } // namespace OHOS