1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "video_capture_sf_impl.h"
17 #include <map>
18 #include <cmath>
19 #include "media_log.h"
20 #include "media_errors.h"
21 #include "graphic_common.h"
22
23 namespace {
24 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "VideoCaptureSfmpl"};
25 constexpr int32_t DEFAULT_SURFACE_QUEUE_SIZE = 6;
26 constexpr int32_t DEFAULT_SURFACE_SIZE = 1024 * 1024;
27 constexpr int32_t DEFAULT_VIDEO_WIDTH = 1920;
28 constexpr int32_t DEFAULT_VIDEO_HEIGHT = 1080;
29 }
30
31 namespace OHOS {
32 namespace Media {
VideoCaptureSfImpl()33 VideoCaptureSfImpl::VideoCaptureSfImpl()
34 : videoWidth_(DEFAULT_VIDEO_WIDTH),
35 videoHeight_(DEFAULT_VIDEO_HEIGHT),
36 fence_(-1),
37 bufferAvailableCount_(0),
38 timestamp_(0),
39 damage_ {0},
40 surfaceBuffer_(nullptr),
41 started_(false),
42 paused_(false),
43 streamType_(VIDEO_STREAM_TYPE_UNKNOWN),
44 streamTypeUnknown_(true),
45 dataConSurface_(nullptr),
46 producerSurface_(nullptr)
47 {
48 }
49
~VideoCaptureSfImpl()50 VideoCaptureSfImpl::~VideoCaptureSfImpl()
51 {
52 (void)Stop();
53 }
54
Prepare()55 int32_t VideoCaptureSfImpl::Prepare()
56 {
57 MEDIA_LOGI("videoWidth %{public}d, videoHeight %{public}d", videoWidth_, videoHeight_);
58 sptr<Surface> consumerSurface = Surface::CreateSurfaceAsConsumer();
59 CHECK_AND_RETURN_RET_LOG(consumerSurface != nullptr, MSERR_NO_MEMORY, "create surface fail");
60
61 sptr<IBufferConsumerListener> listenerProxy = new (std::nothrow) ConsumerListenerProxy(*this);
62 CHECK_AND_RETURN_RET_LOG(listenerProxy != nullptr, MSERR_NO_MEMORY, "create consumer listener fail");
63
64 if (consumerSurface->RegisterConsumerListener(listenerProxy) != SURFACE_ERROR_OK) {
65 MEDIA_LOGW("register consumer listener fail");
66 }
67
68 sptr<IBufferProducer> producer = consumerSurface->GetProducer();
69 CHECK_AND_RETURN_RET_LOG(producer != nullptr, MSERR_NO_MEMORY, "get producer fail");
70 sptr<Surface> producerSurface = Surface::CreateSurfaceAsProducer(producer);
71 CHECK_AND_RETURN_RET_LOG(producerSurface != nullptr, MSERR_NO_MEMORY, "get producer fail");
72
73 dataConSurface_ = consumerSurface;
74 producerSurface_ = producerSurface;
75
76 SetSurfaceUserData();
77 return MSERR_OK;
78 }
79
Start()80 int32_t VideoCaptureSfImpl::Start()
81 {
82 std::unique_lock<std::mutex> lock(mutex_);
83 started_.store(true);
84 return MSERR_OK;
85 }
86
GetCurrentTime()87 uint64_t VideoCaptureSfImpl::GetCurrentTime()
88 {
89 constexpr uint32_t SEC_TO_NS = 1000000000; // second to nano second
90 struct timespec timestamp = {0, 0};
91 clock_gettime(CLOCK_MONOTONIC, ×tamp);
92 uint64_t time = (uint64_t)timestamp.tv_sec * SEC_TO_NS + (uint64_t)timestamp.tv_nsec;
93 return time;
94 }
95
Pause()96 int32_t VideoCaptureSfImpl::Pause()
97 {
98 std::lock_guard<std::mutex> lock1(pauseMutex_);
99 isPause_ = true;
100 pauseCount_++;
101 return MSERR_OK;
102 }
103
Resume()104 int32_t VideoCaptureSfImpl::Resume()
105 {
106 std::lock_guard<std::mutex> lock1(pauseMutex_);
107 isResume_ = true;
108 return MSERR_OK;
109 }
110
Stop()111 int32_t VideoCaptureSfImpl::Stop()
112 {
113 std::unique_lock<std::mutex> lock(mutex_);
114 started_.store(false);
115 if (bufferAvailableCount_ == 0) {
116 bufferAvailableCount_++;
117 }
118 bufferAvailableCondition_.notify_all();
119 if (dataConSurface_ != nullptr) {
120 if (dataConSurface_->UnregisterConsumerListener() != SURFACE_ERROR_OK) {
121 MEDIA_LOGW("deregister consumer listener fail");
122 }
123 dataConSurface_ = nullptr;
124 producerSurface_ = nullptr;
125 }
126 pauseTime_ = 0;
127 resumeTime_ = 0;
128 persistTime_ = 0;
129 totalPauseTime_ = 0;
130 pauseCount_ = 0;
131 isFirstBuffer_ = true;
132 return MSERR_OK;
133 }
134
UnLock(bool start)135 void VideoCaptureSfImpl::UnLock(bool start)
136 {
137 std::unique_lock<std::mutex> lock(mutex_);
138 resourceLock_ = start;
139 bufferAvailableCondition_.notify_all();
140 MEDIA_LOGI("Unlock any pending access to the resource: %{public}d", resourceLock_);
141 }
142
SetVideoWidth(uint32_t width)143 int32_t VideoCaptureSfImpl::SetVideoWidth(uint32_t width)
144 {
145 videoWidth_ = width;
146 return MSERR_OK;
147 }
148
SetVideoHeight(uint32_t height)149 int32_t VideoCaptureSfImpl::SetVideoHeight(uint32_t height)
150 {
151 videoHeight_ = height;
152 return MSERR_OK;
153 }
154
SetFrameRate(uint32_t frameRate)155 int32_t VideoCaptureSfImpl::SetFrameRate(uint32_t frameRate)
156 {
157 framerate_ = frameRate;
158 minInterval_ = 1000000000 / framerate_; // 1s = 1000000000ns
159 return MSERR_OK;
160 }
161
SetStreamType(VideoStreamType streamType)162 int32_t VideoCaptureSfImpl::SetStreamType(VideoStreamType streamType)
163 {
164 streamTypeUnknown_ = false;
165 streamType_ = streamType;
166 return MSERR_OK;
167 }
168
GetSurface()169 sptr<Surface> VideoCaptureSfImpl::GetSurface()
170 {
171 CHECK_AND_RETURN_RET_LOG(producerSurface_ != nullptr, nullptr, "surface not created");
172 return producerSurface_;
173 }
174
GetCodecBuffer()175 std::shared_ptr<EsAvcCodecBuffer> VideoCaptureSfImpl::GetCodecBuffer()
176 {
177 if (AcquireSurfaceBuffer() == MSERR_OK) {
178 if (streamTypeUnknown_) {
179 ProbeStreamType();
180 }
181 return DoGetCodecBuffer();
182 }
183
184 return nullptr;
185 }
186
GetFrameBufferInner()187 std::shared_ptr<VideoFrameBuffer> VideoCaptureSfImpl::GetFrameBufferInner()
188 {
189 if (streamTypeUnknown_) {
190 ProbeStreamType();
191 }
192 bufferNumber_++;
193 return DoGetFrameBuffer();
194 }
195
GetFrameBuffer()196 std::shared_ptr<VideoFrameBuffer> VideoCaptureSfImpl::GetFrameBuffer()
197 {
198 if (bufferNumber_ == 0 && streamType_ == VIDEO_STREAM_TYPE_ES_AVC) {
199 return GetFrameBufferInner();
200 } else {
201 if (AcquireSurfaceBuffer() == MSERR_OK) {
202 return GetFrameBufferInner();
203 }
204 }
205 return nullptr;
206 }
207
SetSurfaceUserData()208 void VideoCaptureSfImpl::SetSurfaceUserData()
209 {
210 SurfaceError ret = dataConSurface_->SetUserData("video_width", std::to_string(videoWidth_));
211 if (ret != SURFACE_ERROR_OK) {
212 MEDIA_LOGW("set video width fail");
213 }
214 ret = dataConSurface_->SetUserData("video_height", std::to_string(videoHeight_));
215 if (ret != SURFACE_ERROR_OK) {
216 MEDIA_LOGW("set video height fail");
217 }
218 ret = dataConSurface_->SetQueueSize(DEFAULT_SURFACE_QUEUE_SIZE);
219 if (ret != SURFACE_ERROR_OK) {
220 MEDIA_LOGW("set queue size fail");
221 }
222 ret = dataConSurface_->SetUserData("surface_size", std::to_string(DEFAULT_SURFACE_SIZE));
223 if (ret != SURFACE_ERROR_OK) {
224 MEDIA_LOGW("set surface size fail");
225 }
226 ret = dataConSurface_->SetDefaultWidthAndHeight(videoWidth_, videoHeight_);
227 if (ret != SURFACE_ERROR_OK) {
228 MEDIA_LOGW("set surface width and height fail");
229 }
230 }
231
GetSufferExtraData()232 int32_t VideoCaptureSfImpl::GetSufferExtraData()
233 {
234 CHECK_AND_RETURN_RET_LOG(surfaceBuffer_ != nullptr, MSERR_INVALID_OPERATION, "surfacebuffer is null");
235
236 SurfaceError surfaceRet;
237
238 surfaceRet = surfaceBuffer_->ExtraGet("dataSize", dataSize_);
239 CHECK_AND_RETURN_RET_LOG(surfaceRet == SURFACE_ERROR_OK, MSERR_INVALID_OPERATION, "get dataSize fail");
240 CHECK_AND_RETURN_RET_LOG(dataSize_ > 0, MSERR_INVALID_OPERATION, "illegal dataSize");
241 surfaceRet = surfaceBuffer_->ExtraGet("timeStamp", pts_);
242 CHECK_AND_RETURN_RET_LOG(surfaceRet == SURFACE_ERROR_OK, MSERR_INVALID_OPERATION, "get timeStamp fail");
243 surfaceRet = surfaceBuffer_->ExtraGet("isKeyFrame", isCodecFrame_);
244 CHECK_AND_RETURN_RET_LOG(surfaceRet == SURFACE_ERROR_OK, MSERR_INVALID_OPERATION, "get isKeyFrame fail");
245
246 MEDIA_LOGI("surfaceBuffer extraData dataSize_: %{public}d, pts: (%{public}" PRId64 ")", dataSize_, pts_);
247 MEDIA_LOGI("is this surfaceBuffer keyFrame ? : %{public}d", isCodecFrame_);
248
249 return MSERR_OK;
250 }
251
DropThisFrame(uint32_t fps,int64_t oldTimeStamp,int64_t newTimeStamp,bool cacheFlag)252 bool VideoCaptureSfImpl::DropThisFrame(uint32_t fps, int64_t oldTimeStamp, int64_t newTimeStamp, bool cacheFlag)
253 {
254 if (!cacheFlag) {
255 MEDIA_LOGW("Resume has cache buffer, drop buffer");
256 return TRUE;
257 }
258
259 if (newTimeStamp <= oldTimeStamp) {
260 MEDIA_LOGW("Invalid timestamp: not increased");
261 return TRUE;
262 }
263
264 if (fps == 0) {
265 MEDIA_LOGW("Invalid frame rate: 0");
266 return FALSE;
267 }
268
269 if ((INT64_MAX - minInterval_) < oldTimeStamp) {
270 MEDIA_LOGW("Invalid timestamp: too big");
271 return TRUE;
272 }
273
274 const int64_t deviations = 3000000; // 3ms
275 if (newTimeStamp < (oldTimeStamp - deviations + minInterval_)) {
276 MEDIA_LOGW("Drop this frame to make sure maximum frame rate");
277 return TRUE;
278 }
279 return FALSE;
280 }
281
CheckPauseResumeTime()282 bool VideoCaptureSfImpl::CheckPauseResumeTime()
283 {
284 std::lock_guard<std::mutex> lock1(pauseMutex_);
285 if (isPause_) {
286 pauseTime_ = pts_;
287 isPause_ = false;
288 isCheckRealTime_ = true;
289 MEDIA_LOGD("video pause timestamp %{public}" PRIu64 "", pauseTime_);
290 }
291
292 if (isResume_ && isCheckRealTime_) {
293 realTimeWhenResume_ = (int64_t)GetCurrentTime();
294 isCheckRealTime_ = false;
295 MEDIA_LOGD("video resume real timestamp %{public}" PRIu64 "", realTimeWhenResume_);
296 }
297
298 if (isResume_) {
299 resumeTime_ = pts_;
300 if ((realTimeWhenResume_ - resumeTime_) > minInterval_) {
301 MEDIA_LOGD("video has cached buffer timestamp %{public}" PRIu64 "", resumeTime_);
302 return false;
303 }
304
305 MEDIA_LOGD("video resume timestamp %{public}" PRIu64 "", resumeTime_);
306 // here subtract one more frame duration to avoid pause time equele to resume time and
307 // cause error in qtmux
308 persistTime_ = std::fabs(resumeTime_ - pauseTime_) - minInterval_;
309 totalPauseTime_ += persistTime_;
310 MEDIA_LOGD("video has %{public}d times pause, total PauseTime: %{public}" PRIu64 "",
311 pauseCount_ ,totalPauseTime_);
312 }
313
314 pts_ = pts_ - totalPauseTime_;
315 return true;
316 }
317
AcquireSurfaceBuffer()318 int32_t VideoCaptureSfImpl::AcquireSurfaceBuffer()
319 {
320 std::unique_lock<std::mutex> lock(mutex_);
321 while (1) {
322 if (!started_ || (dataConSurface_ == nullptr)) {
323 return MSERR_INVALID_OPERATION;
324 }
325
326 bufferAvailableCondition_.wait(lock, [this]() { return bufferAvailableCount_ > 0 || resourceLock_; });
327 if (resourceLock_) {
328 MEDIA_LOGI("flush start / eos, skip acquire buffer");
329 return MSERR_NO_MEMORY;
330 }
331
332 if (!started_ || (dataConSurface_ == nullptr)) {
333 return MSERR_INVALID_OPERATION;
334 }
335 if (dataConSurface_->AcquireBuffer(surfaceBuffer_, fence_, timestamp_, damage_) != SURFACE_ERROR_OK) {
336 return MSERR_UNKNOWN;
337 }
338
339 if (isFirstBuffer_) {
340 isFirstBuffer_ = false;
341 pixelFormat_ = surfaceBuffer_->GetFormat();
342 MEDIA_LOGI("the input pixel format is %{public}d", pixelFormat_);
343 }
344
345 int32_t ret = GetSufferExtraData();
346 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, MSERR_INVALID_OPERATION, "get ExtraData fail");
347
348 bool dropCacheBuffer = CheckPauseResumeTime();
349 bufferAvailableCount_--;
350
351 if (DropThisFrame(framerate_, previousTimestamp_, pts_, dropCacheBuffer)) {
352 MEDIA_LOGI("drop this frame!");
353 (void)dataConSurface_->ReleaseBuffer(surfaceBuffer_, fence_);
354 continue;
355 } else {
356 previousTimestamp_ = pts_;
357 isResume_ = false;
358 break;
359 }
360 };
361 return MSERR_OK;
362 }
363
OnBufferAvailable()364 void VideoCaptureSfImpl::ConsumerListenerProxy::OnBufferAvailable()
365 {
366 return owner_.OnBufferAvailable();
367 }
368
OnBufferAvailable()369 void VideoCaptureSfImpl::OnBufferAvailable()
370 {
371 if (dataConSurface_ == nullptr) {
372 return;
373 }
374 std::unique_lock<std::mutex> lock(mutex_);
375 if (bufferAvailableCount_ == 0) {
376 bufferAvailableCondition_.notify_all();
377 }
378 bufferAvailableCount_++;
379 }
380
ProbeStreamType()381 void VideoCaptureSfImpl::ProbeStreamType()
382 {
383 streamTypeUnknown_ = false;
384 // Identify whether it is an ES stream or a YUV stream from the code stream or from the buffer.
385 }
386 } // namespace Media
387 } // namespace OHOS
388