1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "av_thumbnail_generator.h"
17
18 #include "buffer/avbuffer_common.h"
19 #include "common/media_source.h"
20 #include "ibuffer_consumer_listener.h"
21 #include "graphic_common_c.h"
22 #include "media_errors.h"
23 #include "media_log.h"
24 #include "media_description.h"
25 #include "meta/meta.h"
26 #include "meta/meta_key.h"
27 #include "plugin/plugin_time.h"
28 #include "sync_fence.h"
29 #include "uri_helper.h"
30
31 #include "v1_0/cm_color_space.h"
32 #include "v1_0/hdr_static_metadata.h"
33 #include "v1_0/buffer_handle_meta_key_type.h"
34
35 namespace {
36 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_METADATA, "AVThumbnailGenerator" };
37 }
38
39 namespace OHOS {
40 namespace Media {
41 using namespace OHOS::HDI::Display::Graphic::Common::V1_0;
42 constexpr float BYTES_PER_PIXEL_YUV = 1.5;
43 constexpr int32_t RATE_UV = 2;
44 constexpr int32_t SHIFT_BITS_P010_2_NV12 = 8;
45 constexpr double VIDEO_FRAME_RATE = 2000.0;
46 constexpr int32_t MAX_WAIT_TIME_SECOND = 3;
47 constexpr uint32_t REQUEST_BUFFER_TIMEOUT = 0; // Requesting buffer overtimes 0ms means no retry
48 constexpr uint32_t ERROR_AGAIN_SLEEP_TIME_US = 1000;
49 const std::string AV_THUMBNAIL_GENERATOR_INPUT_BUFFER_QUEUE_NAME = "AVThumbnailGeneratorInputBufferQueue";
50
51 class ThumnGeneratorCodecCallback : public OHOS::MediaAVCodec::MediaCodecCallback {
52 public:
ThumnGeneratorCodecCallback(AVThumbnailGenerator * generator)53 explicit ThumnGeneratorCodecCallback(AVThumbnailGenerator *generator) : generator_(generator) {}
54
55 ~ThumnGeneratorCodecCallback() = default;
56
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)57 void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) override
58 {
59 generator_->OnError(errorType, errorCode);
60 }
61
OnOutputFormatChanged(const MediaAVCodec::Format & format)62 void OnOutputFormatChanged(const MediaAVCodec::Format &format) override
63 {
64 generator_->OnOutputFormatChanged(format);
65 }
66
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)67 void OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
68 {
69 generator_->OnInputBufferAvailable(index, buffer);
70 }
71
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)72 void OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
73 {
74 generator_->OnOutputBufferAvailable(index, buffer);
75 }
76
77 private:
78 AVThumbnailGenerator *generator_;
79 };
80
81 class ThumbnailGeneratorAVBufferAvailableListener : public OHOS::Media::IConsumerListener {
82 public:
ThumbnailGeneratorAVBufferAvailableListener(std::shared_ptr<AVThumbnailGenerator> generator)83 explicit ThumbnailGeneratorAVBufferAvailableListener(std::shared_ptr<AVThumbnailGenerator> generator)
84 : generator_(generator)
85 {
86 }
87
OnBufferAvailable()88 void OnBufferAvailable() override
89 {
90 if (auto generator = generator_.lock()) {
91 generator->AcquireAvailableInputBuffer();
92 } else {
93 MEDIA_LOGE("invalid AVThumbnailGenerator");
94 }
95 }
96 private:
97 std::weak_ptr<AVThumbnailGenerator> generator_;
98 };
99
AVThumbnailGenerator(std::shared_ptr<MediaDemuxer> & mediaDemuxer)100 AVThumbnailGenerator::AVThumbnailGenerator(std::shared_ptr<MediaDemuxer> &mediaDemuxer) : mediaDemuxer_(mediaDemuxer)
101 {
102 MEDIA_LOGI("Constructor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
103 }
104
~AVThumbnailGenerator()105 AVThumbnailGenerator::~AVThumbnailGenerator()
106 {
107 MEDIA_LOGI("Destructor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
108 Destroy();
109 }
110
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)111 void AVThumbnailGenerator::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode)
112 {
113 MEDIA_LOGE("OnError errorType:%{public}d, errorCode:%{public}d", static_cast<int32_t>(errorType), errorCode);
114 {
115 std::scoped_lock lock(mutex_, queueMutex_);
116 stopProcessing_ = true;
117 }
118 cond_.notify_all();
119 bufferAvailableCond_.notify_all();
120 }
121
InitDecoder()122 Status AVThumbnailGenerator::InitDecoder()
123 {
124 MEDIA_LOGD("Init decoder start.");
125 if (videoDecoder_ != nullptr) {
126 MEDIA_LOGD("AVThumbnailGenerator InitDecoder already.");
127 Format format;
128 format.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
129 videoDecoder_->SetParameter(format);
130 videoDecoder_->Start();
131 return Status::OK;
132 }
133 videoDecoder_ = MediaAVCodec::VideoDecoderFactory::CreateByMime(trackMime_);
134 CHECK_AND_RETURN_RET_LOG(videoDecoder_ != nullptr, Status::ERROR_NO_MEMORY, "Create videoDecoder_ is nullptr");
135 Format trackFormat{};
136 trackFormat.SetMeta(GetVideoTrackInfo());
137 trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width_);
138 trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height_);
139 MEDIA_LOGI("0x%{public}06" PRIXPTR " Init decoder trackFormat width:%{public}d, height:%{public}d",
140 FAKE_POINTER(this), width_, height_);
141 trackFormat.PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT,
142 static_cast<int32_t>(Plugins::VideoPixelFormat::NV12));
143 trackFormat.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
144 videoDecoder_->Configure(trackFormat);
145 std::shared_ptr<MediaAVCodec::MediaCodecCallback> mediaCodecCallback =
146 std::make_shared<ThumnGeneratorCodecCallback>(this);
147 videoDecoder_->SetCallback(mediaCodecCallback);
148 videoDecoder_->Prepare();
149 auto res = videoDecoder_->Start();
150 CHECK_AND_RETURN_RET(res == MSERR_OK, Status::ERROR_WRONG_STATE);
151 return Status::OK;
152 }
153
Init()154 int32_t AVThumbnailGenerator::Init()
155 {
156 CHECK_AND_RETURN_RET_LOG(inputBufferQueue_ == nullptr, MSERR_OK, "InputBufferQueue already create");
157
158 inputBufferQueue_ = AVBufferQueue::Create(0,
159 MemoryType::UNKNOWN_MEMORY, AV_THUMBNAIL_GENERATOR_INPUT_BUFFER_QUEUE_NAME, true);
160 CHECK_AND_RETURN_RET_LOG(inputBufferQueue_ != nullptr, MSERR_NO_MEMORY, "BufferQueue is nullptr");
161
162 inputBufferQueueProducer_ = inputBufferQueue_->GetProducer();
163 CHECK_AND_RETURN_RET_LOG(inputBufferQueueProducer_ != nullptr, MSERR_UNKNOWN, "QueueProducer is nullptr");
164
165 inputBufferQueueConsumer_ = inputBufferQueue_->GetConsumer();
166 CHECK_AND_RETURN_RET_LOG(inputBufferQueueConsumer_ != nullptr, MSERR_UNKNOWN, "QueueConsumer is nullptr");
167
168 sptr<IConsumerListener> listener = new ThumbnailGeneratorAVBufferAvailableListener(shared_from_this());
169 CHECK_AND_RETURN_RET_LOG(listener != nullptr, MSERR_NO_MEMORY, "listener is nullptr");
170 inputBufferQueueConsumer_->SetBufferAvailableListener(listener);
171
172 readTask_ = std::make_unique<Task>(std::string("AVThumbReadLoop"));
173 CHECK_AND_RETURN_RET_LOG(readTask_ != nullptr, MSERR_NO_MEMORY, "Task is nullptr");
174
175 readTask_->RegisterJob([this] {return ReadLoop();});
176 readTask_->Start();
177
178 return MSERR_OK;
179 }
180
AcquireAvailableInputBuffer()181 void AVThumbnailGenerator::AcquireAvailableInputBuffer()
182 {
183 CHECK_AND_RETURN_LOG(inputBufferQueueConsumer_ != nullptr, "QueueConsumer is nullptr");
184
185 std::shared_ptr<AVBuffer> filledInputBuffer;
186 Status ret = inputBufferQueueConsumer_->AcquireBuffer(filledInputBuffer);
187 CHECK_AND_RETURN_LOG(ret == Status::OK, "AcquireBuffer fail");
188
189 CHECK_AND_RETURN_LOG(filledInputBuffer->meta_ != nullptr, "filledInputBuffer meta is nullptr.");
190 uint32_t index;
191 CHECK_AND_RETURN_LOG(filledInputBuffer->meta_->GetData(Tag::BUFFER_INDEX, index), "get index failed.");
192
193 CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "videoDecoder_ is nullptr.");
194 if (videoDecoder_->QueueInputBuffer(index) != ERR_OK) {
195 MEDIA_LOGE("QueueInputBuffer failed, index: %{public}u, bufferid: %{public}" PRIu64
196 ", pts: %{public}" PRIu64", flag: %{public}u", index, filledInputBuffer->GetUniqueId(),
197 filledInputBuffer->pts_, filledInputBuffer->flag_);
198 } else {
199 MEDIA_LOGD("QueueInputBuffer success, index: %{public}u, bufferid: %{public}" PRIu64
200 ", pts: %{public}" PRIu64", flag: %{public}u", index, filledInputBuffer->GetUniqueId(),
201 filledInputBuffer->pts_, filledInputBuffer->flag_);
202 }
203 }
204
GetDuration()205 void AVThumbnailGenerator::GetDuration()
206 {
207 auto meta = mediaDemuxer_->GetGlobalMetaInfo();
208 CHECK_AND_RETURN_LOG(meta != nullptr, "Global info is nullptr");
209 (void)meta->Get<Tag::MEDIA_DURATION>(duration_);
210 MEDIA_LOGI("%{public}" PRId64, duration_);
211 }
212
GetVideoTrackInfo()213 std::shared_ptr<Meta> AVThumbnailGenerator::GetVideoTrackInfo()
214 {
215 CHECK_AND_RETURN_RET(trackInfo_ == nullptr, trackInfo_);
216 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "GetTargetTrackInfo demuxer is nullptr");
217 GetDuration();
218 std::vector<std::shared_ptr<Meta>> trackInfos = mediaDemuxer_->GetStreamMetaInfo();
219 size_t trackCount = trackInfos.size();
220 CHECK_AND_RETURN_RET_LOG(trackCount > 0, nullptr, "GetTargetTrackInfo trackCount is invalid");
221 for (size_t index = 0; index < trackCount; index++) {
222 if (!(trackInfos[index]->GetData(Tag::MIME_TYPE, trackMime_))) {
223 MEDIA_LOGW("GetTargetTrackInfo get mime type failed %{public}s", trackMime_.c_str());
224 continue;
225 }
226 if (trackMime_.find("video/") == 0) {
227 Plugins::MediaType mediaType;
228 CHECK_AND_RETURN_RET_LOG(trackInfos[index]->GetData(Tag::MEDIA_TYPE, mediaType), nullptr,
229 "GetTargetTrackInfo failed to get mediaType, index:%{public}zu", index);
230 CHECK_AND_RETURN_RET_LOG(
231 mediaType == Plugins::MediaType::VIDEO, nullptr,
232 "GetTargetTrackInfo mediaType is not video, index:%{public}zu, mediaType:%{public}d", index,
233 static_cast<int32_t>(mediaType));
234 CHECK_AND_RETURN_RET_LOG(trackInfos[index]->Get<Tag::VIDEO_FRAME_RATE>(frameRate_) && frameRate_ > 0,
235 nullptr, "failed to get video frame rate");
236 trackIndex_ = index;
237 MEDIA_LOGI("0x%{public}06" PRIXPTR " GetTrackInfo success trackIndex_:%{public}zu, trackMime_:%{public}s",
238 FAKE_POINTER(this), trackIndex_, trackMime_.c_str());
239 if (trackInfos[index]->Get<Tag::VIDEO_ROTATION>(rotation_)) {
240 MEDIA_LOGI("rotation %{public}d", static_cast<int32_t>(rotation_));
241 }
242 return trackInfos[trackIndex_];
243 }
244 }
245 MEDIA_LOGW("GetTargetTrackInfo FAILED.");
246 return nullptr;
247 }
248
OnOutputFormatChanged(const MediaAVCodec::Format & format)249 void AVThumbnailGenerator::OnOutputFormatChanged(const MediaAVCodec::Format &format)
250 {
251 MEDIA_LOGD("OnOutputFormatChanged");
252 outputFormat_ = format;
253 int32_t width = 0;
254 int32_t height = 0;
255 bool hasWidth = format.GetIntValue(Tag::VIDEO_PIC_WIDTH, width);
256 bool hasHeight = format.GetIntValue(Tag::VIDEO_PIC_HEIGHT, height);
257 CHECK_AND_RETURN_LOG(hasWidth && hasHeight, "OutputFormat doesn't have width or height");
258 width_ = width;
259 height_ = height;
260 }
261
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)262 void AVThumbnailGenerator::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
263 {
264 CHECK_AND_RETURN_LOG(buffer != nullptr && buffer->meta_ != nullptr, "meta_ is nullptr.");
265
266 MEDIA_LOGD("OnInputBufferAvailable enter. index: %{public}u, bufferid: %{public}" PRIu64", pts: %{public}" PRIu64
267 ", flag: %{public}u", index, buffer->GetUniqueId(), buffer->pts_, buffer->flag_);
268
269 buffer->meta_->SetData(Tag::BUFFER_INDEX, index);
270
271 CHECK_AND_RETURN_LOG(inputBufferQueueConsumer_ != nullptr, "QueueConsumer is nullptr");
272
273 if (stopProcessing_.load() || hasFetchedFrame_.load() || readErrorFlag_.load()) {
274 MEDIA_LOGD("stop or has fetched frame, need not queue input buffer");
275 return;
276 }
277
278 {
279 std::unique_lock<std::mutex> lock(queueMutex_);
280 if (inputBufferQueueConsumer_->IsBufferInQueue(buffer)) {
281 if (inputBufferQueueConsumer_->ReleaseBuffer(buffer) != Status::OK) {
282 MEDIA_LOGE("IsBufferInQueue ReleaseBuffer failed. index: %{public}u, bufferid: %{public}" PRIu64
283 ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(),
284 buffer->pts_, buffer->flag_);
285 return;
286 } else {
287 MEDIA_LOGD("IsBufferInQueue ReleaseBuffer success. index: %{public}u, bufferid: %{public}" PRIu64
288 ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(),
289 buffer->pts_, buffer->flag_);
290 }
291 } else {
292 uint32_t size = inputBufferQueueConsumer_->GetQueueSize() + 1;
293 MEDIA_LOGI("AttachBuffer enter. index: %{public}u, size: %{public}u , bufferid: %{public}" PRIu64,
294 index, size, buffer->GetUniqueId());
295 inputBufferQueueConsumer_->SetQueueSizeAndAttachBuffer(size, buffer, false);
296 bufferVector_.push_back(buffer);
297 }
298 isBufferAvailable_ = true;
299 }
300 bufferAvailableCond_.notify_all();
301 }
302
ReadLoop()303 int64_t AVThumbnailGenerator::ReadLoop()
304 {
305 std::shared_ptr<AVBuffer> emptyBuffer = nullptr;
306 {
307 std::unique_lock<std::mutex> lock(queueMutex_);
308 bufferAvailableCond_.wait(lock, [this] {
309 return stopProcessing_.load() ||
310 (!hasFetchedFrame_.load() && !readErrorFlag_.load() && isBufferAvailable_.load());
311 });
312
313 CHECK_AND_RETURN_RET_LOG(!stopProcessing_.load(), StopTask(), "Stop Readloop");
314
315 CHECK_AND_RETURN_RET_LOG(inputBufferQueueProducer_ != nullptr, StopTask(), "QueueProducer is nullptr");
316
317 AVBufferConfig avBufferConfig;
318 auto res = inputBufferQueueProducer_->RequestBuffer(emptyBuffer, avBufferConfig, REQUEST_BUFFER_TIMEOUT);
319 if (res != Status::OK) {
320 isBufferAvailable_ = false;
321 return 0;
322 }
323 }
324
325 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, StopTask(), "mediaDemuxer is nullptr");
326
327 auto readSampleRes = mediaDemuxer_->ReadSample(trackIndex_, emptyBuffer);
328 if (readSampleRes != Status::OK && readSampleRes != Status::END_OF_STREAM && readSampleRes != Status::ERROR_AGAIN) {
329 {
330 std::unique_lock<std::mutex> lock(mutex_);
331 readErrorFlag_ = true;
332 }
333 cond_.notify_all();
334 inputBufferQueueProducer_->PushBuffer(emptyBuffer, false);
335 return 0;
336 }
337 if (readSampleRes == Status::ERROR_AGAIN) {
338 inputBufferQueueProducer_->PushBuffer(emptyBuffer, false);
339 return ERROR_AGAIN_SLEEP_TIME_US;
340 }
341 inputBufferQueueProducer_->PushBuffer(emptyBuffer, true);
342 return 0;
343 }
344
StopTask()345 int64_t AVThumbnailGenerator::StopTask()
346 {
347 if (readTask_ != nullptr) {
348 readTask_->Stop();
349 }
350 return 0;
351 }
352
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)353 void AVThumbnailGenerator::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
354 {
355 MEDIA_LOGD("OnOutputBufferAvailable index:%{public}u , pts %{public}" PRId64, index, buffer->pts_);
356 CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "Video decoder not exist");
357 bool isEosBuffer = buffer->flag_ & (uint32_t)(AVBufferFlag::EOS);
358 bool isValidBuffer = buffer != nullptr && buffer->memory_ != nullptr &&
359 (buffer->memory_->GetSize() != 0 || buffer->memory_->GetSurfaceBuffer() != nullptr || isEosBuffer);
360 bool isValidState = !hasFetchedFrame_.load() && !stopProcessing_.load();
361 if (!isValidBuffer || !isValidState) {
362 MEDIA_LOGW("isValidBuffer %{public}d isValidState %{public}d", isValidBuffer, isValidState);
363 videoDecoder_->ReleaseOutputBuffer(index, false);
364 return;
365 }
366 bool isClosest = seekMode_ == Plugins::SeekMode::SEEK_CLOSEST;
367 bool isAvailableFrame = !isClosest || buffer->pts_ >= seekTime_ || isEosBuffer;
368 if (!isAvailableFrame) {
369 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
370 bufferIndex_ = index;
371 avBuffer_ = buffer;
372 return;
373 }
374 if (isAvailableFrame) {
375 {
376 std::unique_lock<std::mutex> lock(mutex_);
377 hasFetchedFrame_ = true;
378 }
379 if (isClosest && avBuffer_ != nullptr) {
380 int64_t preDiff = seekTime_ - avBuffer_->pts_;
381 int64_t nextDiff = buffer->pts_ - seekTime_;
382 if (preDiff > nextDiff && !(buffer->flag_ & (uint32_t)(AVBufferFlag::EOS))) {
383 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
384 bufferIndex_ = index;
385 avBuffer_ = buffer;
386 } else {
387 videoDecoder_->ReleaseOutputBuffer(index, false);
388 }
389 } else {
390 bufferIndex_ = index;
391 avBuffer_ = buffer;
392 }
393 MEDIA_LOGI("dstTime %{public}" PRId64 " resTime %{public}" PRId64, seekTime_, buffer->pts_);
394 cond_.notify_all();
395 PauseFetchFrame();
396 return;
397 }
398 videoDecoder_->ReleaseOutputBuffer(index, false);
399 }
400
FetchFrameAtTime(int64_t timeUs,int32_t option,const OutputConfiguration & param)401 std::shared_ptr<AVSharedMemory> AVThumbnailGenerator::FetchFrameAtTime(int64_t timeUs, int32_t option,
402 const OutputConfiguration ¶m)
403 {
404 MEDIA_LOGI("Fetch frame 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64 ", option:%{public}d,"
405 "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d",
406 FAKE_POINTER(this), timeUs, option, param.dstWidth, param.dstHeight,
407 static_cast<int32_t>(param.colorFormat));
408 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr");
409
410 readErrorFlag_ = false;
411 hasFetchedFrame_ = false;
412 isBufferAvailable_ = false;
413 outputConfig_ = param;
414 seekTime_ = timeUs;
415 trackInfo_ = GetVideoTrackInfo();
416 CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr.");
417 mediaDemuxer_->Resume();
418 mediaDemuxer_->SelectTrack(trackIndex_);
419 int64_t realSeekTime = timeUs;
420 auto res = SeekToTime(Plugins::Us2Ms(timeUs), static_cast<Plugins::SeekMode>(option), realSeekTime);
421 CHECK_AND_RETURN_RET_LOG(res == Status::OK, nullptr, "Seek fail");
422 CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed.");
423 bool fetchFrameRes = false;
424 {
425 std::unique_lock<std::mutex> lock(mutex_);
426
427 // wait up to 3s to fetch frame AVSharedMemory at time.
428 fetchFrameRes = cond_.wait_for(lock, std::chrono::seconds(MAX_WAIT_TIME_SECOND),
429 [this] { return hasFetchedFrame_.load() || readErrorFlag_.load() || stopProcessing_.load(); });
430 }
431 if (fetchFrameRes) {
432 HandleFetchFrameAtTimeRes();
433 } else {
434 PauseFetchFrame();
435 }
436 return fetchedFrameAtTime_;
437 }
438
HandleFetchFrameAtTimeRes()439 void AVThumbnailGenerator::HandleFetchFrameAtTimeRes()
440 {
441 CHECK_AND_RETURN_RET_LOG(!readErrorFlag_.load(), PauseFetchFrame(), "ReadSample error, exit fetchFrame");
442 CHECK_AND_RETURN_RET_LOG(!stopProcessing_.load(), PauseFetchFrame(), "Destroy or Decoder error, exit fetchFrame");
443 MEDIA_LOGI("0x%{public}06" PRIXPTR " Fetch frame OK width:%{public}d, height:%{public}d",
444 FAKE_POINTER(this), outputConfig_.dstWidth, outputConfig_.dstHeight);
445 ConvertToAVSharedMemory();
446 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
447 }
448
FetchFrameYuv(int64_t timeUs,int32_t option,const OutputConfiguration & param)449 std::shared_ptr<AVBuffer> AVThumbnailGenerator::FetchFrameYuv(int64_t timeUs, int32_t option,
450 const OutputConfiguration ¶m)
451 {
452 MEDIA_LOGI("Fetch frame 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64 ", option:%{public}d,"
453 "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d",
454 FAKE_POINTER(this), timeUs, option, param.dstWidth, param.dstHeight,
455 static_cast<int32_t>(param.colorFormat));
456 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr");
457 avBuffer_ = nullptr;
458 readErrorFlag_ = false;
459 hasFetchedFrame_ = false;
460 isBufferAvailable_ = false;
461 outputConfig_ = param;
462 seekTime_ = timeUs;
463 trackInfo_ = GetVideoTrackInfo();
464 CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr.");
465 mediaDemuxer_->SelectTrack(trackIndex_);
466 int64_t realSeekTime = timeUs;
467 auto res = SeekToTime(Plugins::Us2Ms(timeUs), static_cast<Plugins::SeekMode>(option), realSeekTime);
468 CHECK_AND_RETURN_RET_LOG(res == Status::OK, nullptr, "Seek fail");
469 CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed.");
470 bool fetchFrameRes = false;
471 {
472 std::unique_lock<std::mutex> lock(mutex_);
473
474 // wait up to 3s to fetch frame AVSharedMemory at time.
475 fetchFrameRes = cond_.wait_for(lock, std::chrono::seconds(MAX_WAIT_TIME_SECOND),
476 [this] { return hasFetchedFrame_.load() || readErrorFlag_.load() || stopProcessing_.load(); });
477 }
478 if (fetchFrameRes) {
479 HandleFetchFrameYuvRes();
480 } else {
481 HandleFetchFrameYuvFailed();
482 }
483 return avBuffer_;
484 }
485
HandleFetchFrameYuvRes()486 void AVThumbnailGenerator::HandleFetchFrameYuvRes()
487 {
488 CHECK_AND_RETURN_RET_LOG(!readErrorFlag_.load(), HandleFetchFrameYuvFailed(), "ReadSample error, exit fetchFrame");
489 CHECK_AND_RETURN_RET_LOG(!stopProcessing_.load(), HandleFetchFrameYuvFailed(),
490 "Destroy or Decoder error, exit fetchFrame");
491 MEDIA_LOGI("0x%{public}06" PRIXPTR " Fetch frame OK width:%{public}d, height:%{public}d",
492 FAKE_POINTER(this), outputConfig_.dstWidth, outputConfig_.dstHeight);
493 avBuffer_ = GenerateAlignmentAvBuffer();
494 if (avBuffer_ != nullptr) {
495 avBuffer_->meta_->Set<Tag::VIDEO_WIDTH>(width_);
496 avBuffer_->meta_->Set<Tag::VIDEO_HEIGHT>(height_);
497 avBuffer_->meta_->Set<Tag::VIDEO_ROTATION>(rotation_);
498 }
499 }
500
HandleFetchFrameYuvFailed()501 void AVThumbnailGenerator::HandleFetchFrameYuvFailed()
502 {
503 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
504 PauseFetchFrame();
505 }
506
SeekToTime(int64_t timeMs,Plugins::SeekMode option,int64_t realSeekTime)507 Status AVThumbnailGenerator::SeekToTime(int64_t timeMs, Plugins::SeekMode option, int64_t realSeekTime)
508 {
509 seekMode_ = option;
510 if (option == Plugins::SeekMode::SEEK_CLOSEST) {
511 option = Plugins::SeekMode::SEEK_PREVIOUS_SYNC;
512 }
513 timeMs = duration_ > 0 ? std::min(timeMs, Plugins::Us2Ms(duration_)) : timeMs;
514 auto res = mediaDemuxer_->SeekTo(timeMs, option, realSeekTime);
515 /* SEEK_NEXT_SYNC or SEEK_PREVIOUS_SYNC may cant find I frame and return seek failed
516 if seek failed, use SEEK_CLOSEST_SYNC seek again */
517 if (res != Status::OK && option != Plugins::SeekMode::SEEK_CLOSEST_SYNC) {
518 res = mediaDemuxer_->SeekTo(timeMs, Plugins::SeekMode::SEEK_CLOSEST_SYNC, realSeekTime);
519 seekMode_ = Plugins::SeekMode::SEEK_CLOSEST_SYNC;
520 }
521 return res;
522 }
523
ConvertToAVSharedMemory()524 void AVThumbnailGenerator::ConvertToAVSharedMemory()
525 {
526 auto surfaceBuffer = avBuffer_->memory_->GetSurfaceBuffer();
527 if (surfaceBuffer != nullptr) {
528 auto ret = GetYuvDataAlignStride(surfaceBuffer);
529 CHECK_AND_RETURN_LOG(ret == MSERR_OK, "Copy frame failed");
530 OutputFrame *frame = reinterpret_cast<OutputFrame *>(fetchedFrameAtTime_->GetBase());
531 frame->width_ = surfaceBuffer->GetWidth();
532 frame->height_ = surfaceBuffer->GetHeight();
533 frame->stride_ = frame->width_ * RATE_UV;
534 frame->bytesPerPixel_ = RATE_UV;
535 frame->size_ = frame->width_ * frame->height_ * BYTES_PER_PIXEL_YUV;
536 frame->rotation_ = static_cast<int32_t>(rotation_);
537 return;
538 }
539
540 int32_t width;
541 int32_t height;
542 outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width);
543 outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height);
544 if (width == 0 || height == 0) {
545 width = width_;
546 height = height_;
547 }
548
549 fetchedFrameAtTime_ = std::make_shared<AVSharedMemoryBase>(sizeof(OutputFrame) + avBuffer_->memory_->GetSize(),
550 AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory");
551 int32_t ret = fetchedFrameAtTime_->Init();
552 CHECK_AND_RETURN_LOG(ret == static_cast<int32_t>(Status::OK), "Create AVSharedmemory failed, ret:%{public}d", ret);
553 OutputFrame *frame = reinterpret_cast<OutputFrame *>(fetchedFrameAtTime_->GetBase());
554 frame->width_ = width;
555 frame->height_ = height;
556 frame->stride_ = width * RATE_UV;
557 frame->bytesPerPixel_ = RATE_UV;
558 frame->size_ = avBuffer_->memory_->GetSize();
559 frame->rotation_ = static_cast<int32_t>(rotation_);
560 fetchedFrameAtTime_->Write(avBuffer_->memory_->GetAddr(), frame->size_, sizeof(OutputFrame));
561 }
562
ConvertP010ToNV12(const sptr<SurfaceBuffer> & surfaceBuffer,uint8_t * dstNV12,int32_t strideWidth,int32_t strideHeight)563 void AVThumbnailGenerator::ConvertP010ToNV12(const sptr<SurfaceBuffer> &surfaceBuffer, uint8_t *dstNV12,
564 int32_t strideWidth, int32_t strideHeight)
565 {
566 int32_t width = surfaceBuffer->GetWidth();
567 int32_t height = surfaceBuffer->GetHeight();
568 uint8_t *srcP010 = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
569
570 // copy src Y component to dst
571 for (int32_t i = 0; i < height; i++) {
572 uint16_t *srcY = reinterpret_cast<uint16_t *>(srcP010 + strideWidth * i);
573 uint8_t *dstY = dstNV12 + width * i;
574 for (int32_t j = 0; j < width; j++) {
575 *dstY = static_cast<uint8_t>(*srcY >> SHIFT_BITS_P010_2_NV12);
576 srcY++;
577 dstY++;
578 }
579 }
580
581 uint8_t *maxDstAddr = dstNV12 + width * height + width * height / RATE_UV;
582 uint8_t *originSrcAddr = srcP010;
583 uint16_t *maxSrcAddr = reinterpret_cast<uint16_t *>(originSrcAddr) + surfaceBuffer->GetSize();
584
585 srcP010 = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + strideWidth * strideHeight;
586 dstNV12 = dstNV12 + width * height;
587
588 // copy src UV component to dst, height(UV) = height(Y) / 2;
589 for (int32_t i = 0; i < height / 2; i++) {
590 uint16_t *srcUV = reinterpret_cast<uint16_t *>(srcP010 + strideWidth * i);
591 uint8_t *dstUV = dstNV12 + width * i;
592 for (int32_t j = 0; j < width && srcUV < maxSrcAddr && dstUV < maxDstAddr; j++) {
593 *dstUV = static_cast<uint8_t>(*srcUV >> SHIFT_BITS_P010_2_NV12);
594 *(dstUV + 1) = static_cast<uint8_t>(*(srcUV + 1) >> SHIFT_BITS_P010_2_NV12);
595 srcUV += 2; // srcUV move by 2 to process U and V component
596 dstUV += 2; // dstUV move by 2 to process U and V component
597 }
598 }
599 }
600
GenerateAlignmentAvBuffer()601 std::shared_ptr<AVBuffer> AVThumbnailGenerator::GenerateAlignmentAvBuffer()
602 {
603 CHECK_AND_RETURN_RET_LOG(avBuffer_ != nullptr && avBuffer_->memory_ != nullptr, nullptr,
604 "Generate Alignment AvBuffer failed, avBuffer_ or avBuffer_->memory_ is nullptr.");
605 if (avBuffer_->memory_->GetSize() != 0 && avBuffer_->memory_->GetSurfaceBuffer() == nullptr) {
606 return GenerateAvBufferFromFCodec();
607 }
608 CHECK_AND_RETURN_RET_LOG(avBuffer_->memory_->GetSurfaceBuffer() != nullptr, nullptr,
609 "Memory size is 0, SurfaceBuffer is nullptr.");
610 auto srcSurfaceBuffer = avBuffer_->memory_->GetSurfaceBuffer();
611 auto width = srcSurfaceBuffer->GetWidth();
612 auto height = srcSurfaceBuffer->GetHeight();
613 bool isHdr = srcSurfaceBuffer->GetFormat() ==
614 static_cast<int32_t>(GraphicPixelFormat::GRAPHIC_PIXEL_FMT_YCBCR_P010);
615
616 sptr<SurfaceBuffer> dstSurfaceBuffer = SurfaceBuffer::Create();
617 BufferRequestConfig requestConfig = {
618 .width = width,
619 .height = height,
620 .strideAlignment = 0x2,
621 .format = srcSurfaceBuffer->GetFormat(), // always yuv
622 .usage = srcSurfaceBuffer->GetUsage(),
623 .timeout = 0,
624 };
625 CHECK_AND_RETURN_RET_LOG(dstSurfaceBuffer != nullptr, nullptr, "Create surfaceBuffer failed");
626 GSError allocRes = dstSurfaceBuffer->Alloc(requestConfig);
627 CHECK_AND_RETURN_RET_LOG(allocRes == 0, nullptr, "Alloc surfaceBuffer failed, ecode %{public}d", allocRes);
628
629 CopySurfaceBufferInfo(srcSurfaceBuffer, dstSurfaceBuffer);
630 int32_t copyRes = memcpy_s(dstSurfaceBuffer->GetVirAddr(), dstSurfaceBuffer->GetSize(),
631 srcSurfaceBuffer->GetVirAddr(), srcSurfaceBuffer->GetSize());
632 CHECK_AND_RETURN_RET_LOG(copyRes == EOK, nullptr, "copy surface buffer pixels failed, copyRes %{public}d", copyRes);
633 int32_t outputHeight;
634 auto hasSliceHeight = outputFormat_.GetIntValue(Tag::VIDEO_SLICE_HEIGHT, outputHeight);
635 if (!hasSliceHeight || outputHeight < height) {
636 outputHeight = height;
637 }
638
639 // create avBuffer from surfaceBuffer
640 std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(dstSurfaceBuffer);
641 bool ret = targetAvBuffer && targetAvBuffer->memory_ && targetAvBuffer->meta_ &&
642 targetAvBuffer->memory_->GetSurfaceBuffer() != nullptr;
643 CHECK_AND_RETURN_RET_LOG(ret, nullptr, "create avBuffer failed");
644 targetAvBuffer->meta_->Set<Tag::VIDEO_IS_HDR_VIVID>(isHdr);
645 targetAvBuffer->meta_->Set<Tag::VIDEO_SLICE_HEIGHT>(outputHeight);
646 return targetAvBuffer;
647 }
648
GenerateAvBufferFromFCodec()649 std::shared_ptr<AVBuffer> AVThumbnailGenerator::GenerateAvBufferFromFCodec()
650 {
651 AVBufferConfig avBufferConfig;
652 avBufferConfig.size = avBuffer_->memory_->GetSize();
653 avBufferConfig.memoryType = MemoryType::SHARED_MEMORY;
654 avBufferConfig.memoryFlag = MemoryFlag::MEMORY_READ_WRITE;
655 std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(avBufferConfig);
656 CHECK_AND_RETURN_RET_LOG(targetAvBuffer != nullptr, nullptr, "Create avBuffer failed");
657 targetAvBuffer->memory_->Write(avBuffer_->memory_->GetAddr(), avBuffer_->memory_->GetSize(), 0);
658 return targetAvBuffer;
659 }
660
GetYuvDataAlignStride(const sptr<SurfaceBuffer> & surfaceBuffer)661 int32_t AVThumbnailGenerator::GetYuvDataAlignStride(const sptr<SurfaceBuffer> &surfaceBuffer)
662 {
663 int32_t width = surfaceBuffer->GetWidth();
664 int32_t height = surfaceBuffer->GetHeight();
665 int32_t stride = surfaceBuffer->GetStride();
666 int32_t outputHeight;
667 auto hasSliceHeight = outputFormat_.GetIntValue(Tag::VIDEO_SLICE_HEIGHT, outputHeight);
668 if (!hasSliceHeight || outputHeight < height) {
669 outputHeight = height;
670 }
671 MEDIA_LOGD("GetYuvDataAlignStride stride:%{public}d, strideWidth:%{public}d, outputHeight:%{public}d", stride,
672 stride, outputHeight);
673
674 fetchedFrameAtTime_ =
675 std::make_shared<AVSharedMemoryBase>(sizeof(OutputFrame) + width * height * BYTES_PER_PIXEL_YUV,
676 AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory");
677 auto ret = fetchedFrameAtTime_->Init();
678 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Create AVSharedmemory failed, ret:%{public}d", ret);
679 uint8_t *dstPtr = static_cast<uint8_t *>(sizeof(OutputFrame) + fetchedFrameAtTime_->GetBase());
680 uint8_t *srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
681 int32_t format = surfaceBuffer->GetFormat();
682 if (format == static_cast<int32_t>(GraphicPixelFormat::GRAPHIC_PIXEL_FMT_YCBCR_P010)) {
683 ConvertP010ToNV12(surfaceBuffer, dstPtr, stride, outputHeight);
684 return MSERR_OK;
685 }
686
687 // copy src Y component to dst
688 for (int32_t y = 0; y < height; y++) {
689 auto ret = memcpy_s(dstPtr, width, srcPtr, width);
690 if (ret != EOK) {
691 MEDIA_LOGW("Memcpy Y component failed.");
692 }
693 srcPtr += stride;
694 dstPtr += width;
695 }
696
697 srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + stride * outputHeight;
698
699 // copy src UV component to dst, height(UV) = height(Y) / 2
700 for (int32_t uv = 0; uv < height / 2; uv++) {
701 auto ret = memcpy_s(dstPtr, width, srcPtr, width);
702 if (ret != EOK) {
703 MEDIA_LOGW("Memcpy UV component failed.");
704 }
705 srcPtr += stride;
706 dstPtr += width;
707 }
708 return MSERR_OK;
709 }
710
Reset()711 void AVThumbnailGenerator::Reset()
712 {
713 if (mediaDemuxer_ != nullptr) {
714 mediaDemuxer_->Reset();
715 }
716
717 if (videoDecoder_ != nullptr) {
718 videoDecoder_->Reset();
719 }
720
721 FlushBufferQueue();
722
723 hasFetchedFrame_ = false;
724 trackInfo_ = nullptr;
725 }
726
FlushBufferQueue()727 void AVThumbnailGenerator::FlushBufferQueue()
728 {
729 std::unique_lock<std::mutex> lock(queueMutex_);
730 if (inputBufferQueueConsumer_ != nullptr) {
731 for (auto &buffer : bufferVector_) {
732 inputBufferQueueConsumer_->DetachBuffer(buffer);
733 }
734 bufferVector_.clear();
735 inputBufferQueueConsumer_->SetQueueSize(0);
736 }
737 isBufferAvailable_ = false;
738 }
739
CopySurfaceBufferInfo(sptr<SurfaceBuffer> & source,sptr<SurfaceBuffer> & dst)740 void AVThumbnailGenerator::CopySurfaceBufferInfo(sptr<SurfaceBuffer> &source, sptr<SurfaceBuffer> &dst)
741 {
742 if (source == nullptr || dst == nullptr) {
743 MEDIA_LOGI("CopySurfaceBufferInfo failed, source or dst is nullptr");
744 return;
745 }
746 std::vector<uint8_t> hdrMetadataTypeVec;
747 std::vector<uint8_t> colorSpaceInfoVec;
748 std::vector<uint8_t> staticData;
749 std::vector<uint8_t> dynamicData;
750
751 if (source->GetMetadata(ATTRKEY_HDR_METADATA_TYPE, hdrMetadataTypeVec) == GSERROR_OK) {
752 dst->SetMetadata(ATTRKEY_HDR_METADATA_TYPE, hdrMetadataTypeVec);
753 }
754 if (source->GetMetadata(ATTRKEY_COLORSPACE_INFO, colorSpaceInfoVec) == GSERROR_OK) {
755 dst->SetMetadata(ATTRKEY_COLORSPACE_INFO, colorSpaceInfoVec);
756 }
757 if (GetSbStaticMetadata(source, staticData) && (staticData.size() > 0)) {
758 SetSbStaticMetadata(dst, staticData);
759 }
760 if (GetSbDynamicMetadata(source, dynamicData) && (dynamicData.size()) > 0) {
761 SetSbDynamicMetadata(dst, dynamicData);
762 }
763 }
764
GetSbStaticMetadata(const sptr<SurfaceBuffer> & buffer,std::vector<uint8_t> & staticMetadata)765 bool AVThumbnailGenerator::GetSbStaticMetadata(const sptr<SurfaceBuffer> &buffer, std::vector<uint8_t> &staticMetadata)
766 {
767 return buffer->GetMetadata(ATTRKEY_HDR_STATIC_METADATA, staticMetadata) == GSERROR_OK;
768 }
769
GetSbDynamicMetadata(const sptr<SurfaceBuffer> & buffer,std::vector<uint8_t> & dynamicMetadata)770 bool AVThumbnailGenerator::GetSbDynamicMetadata(const sptr<SurfaceBuffer> &buffer,
771 std::vector<uint8_t> &dynamicMetadata)
772 {
773 return buffer->GetMetadata(ATTRKEY_HDR_DYNAMIC_METADATA, dynamicMetadata) == GSERROR_OK;
774 }
775
SetSbStaticMetadata(sptr<SurfaceBuffer> & buffer,const std::vector<uint8_t> & staticMetadata)776 bool AVThumbnailGenerator::SetSbStaticMetadata(sptr<SurfaceBuffer> &buffer, const std::vector<uint8_t> &staticMetadata)
777 {
778 return buffer->SetMetadata(ATTRKEY_HDR_STATIC_METADATA, staticMetadata) == GSERROR_OK;
779 }
780
SetSbDynamicMetadata(sptr<SurfaceBuffer> & buffer,const std::vector<uint8_t> & dynamicMetadata)781 bool AVThumbnailGenerator::SetSbDynamicMetadata(sptr<SurfaceBuffer> &buffer,
782 const std::vector<uint8_t> &dynamicMetadata)
783 {
784 return buffer->SetMetadata(ATTRKEY_HDR_DYNAMIC_METADATA, dynamicMetadata) == GSERROR_OK;
785 }
786
Destroy()787 void AVThumbnailGenerator::Destroy()
788 {
789 {
790 std::unique_lock<std::mutex> lock(queueMutex_);
791 stopProcessing_ = true;
792 }
793 bufferAvailableCond_.notify_all();
794
795 if (readTask_ != nullptr) {
796 readTask_->Stop();
797 }
798
799 if (videoDecoder_ != nullptr) {
800 videoDecoder_->Stop();
801 videoDecoder_->Release();
802 }
803 mediaDemuxer_ = nullptr;
804 videoDecoder_ = nullptr;
805 MEDIA_LOGI("0x%{public}06" PRIXPTR " Finish Destroy.", FAKE_POINTER(this));
806 }
807
PauseFetchFrame()808 void AVThumbnailGenerator::PauseFetchFrame()
809 {
810 hasFetchedFrame_ = true;
811 mediaDemuxer_->Pause();
812 mediaDemuxer_->Flush();
813 videoDecoder_->Flush();
814 Format format;
815 format.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, frameRate_);
816 videoDecoder_->SetParameter(format);
817 FlushBufferQueue();
818 }
819 } // namespace Media
820 } // namespace OHOS