1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "av_thumbnail_generator.h"
17
18 #include "buffer/avbuffer_common.h"
19 #include "common/media_source.h"
20 #include "ibuffer_consumer_listener.h"
21 #include "graphic_common_c.h"
22 #include "media_errors.h"
23 #include "media_log.h"
24 #include "media_description.h"
25 #include "meta/meta.h"
26 #include "meta/meta_key.h"
27 #include "plugin/plugin_time.h"
28 #include "sync_fence.h"
29 #include "uri_helper.h"
30
31 #include "v1_0/cm_color_space.h"
32 #include "v1_0/hdr_static_metadata.h"
33 #include "v1_0/buffer_handle_meta_key_type.h"
34
35 namespace {
36 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_METADATA, "AVThumbnailGenerator" };
37 }
38
39 namespace OHOS {
40 namespace Media {
41 using namespace OHOS::HDI::Display::Graphic::Common::V1_0;
42 constexpr float BYTES_PER_PIXEL_YUV = 1.5;
43 constexpr int32_t RATE_UV = 2;
44 constexpr int32_t SHIFT_BITS_P010_2_NV12 = 8;
45 constexpr double VIDEO_FRAME_RATE = 2000.0;
46
47 class ThumnGeneratorCodecCallback : public OHOS::MediaAVCodec::MediaCodecCallback {
48 public:
ThumnGeneratorCodecCallback(AVThumbnailGenerator * generator)49 explicit ThumnGeneratorCodecCallback(AVThumbnailGenerator *generator) : generator_(generator) {}
50
51 ~ThumnGeneratorCodecCallback() = default;
52
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)53 void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) override
54 {
55 generator_->OnError(errorType, errorCode);
56 }
57
OnOutputFormatChanged(const MediaAVCodec::Format & format)58 void OnOutputFormatChanged(const MediaAVCodec::Format &format) override
59 {
60 generator_->OnOutputFormatChanged(format);
61 }
62
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)63 void OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
64 {
65 generator_->OnInputBufferAvailable(index, buffer);
66 }
67
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)68 void OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
69 {
70 generator_->OnOutputBufferAvailable(index, buffer);
71 }
72
73 private:
74 AVThumbnailGenerator *generator_;
75 };
76
AVThumbnailGenerator(std::shared_ptr<MediaDemuxer> & mediaDemuxer)77 AVThumbnailGenerator::AVThumbnailGenerator(std::shared_ptr<MediaDemuxer> &mediaDemuxer) : mediaDemuxer_(mediaDemuxer)
78 {
79 MEDIA_LOGI("Constructor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
80 }
81
~AVThumbnailGenerator()82 AVThumbnailGenerator::~AVThumbnailGenerator()
83 {
84 MEDIA_LOGI("Destructor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
85 Destroy();
86 }
87
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)88 void AVThumbnailGenerator::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode)
89 {
90 MEDIA_LOGE("OnError errorType:%{public}d, errorCode:%{public}d", static_cast<int32_t>(errorType), errorCode);
91 stopProcessing_ = true;
92 }
93
InitDecoder()94 Status AVThumbnailGenerator::InitDecoder()
95 {
96 MEDIA_LOGD("Init decoder start.");
97 if (videoDecoder_ != nullptr) {
98 MEDIA_LOGD("AVThumbnailGenerator InitDecoder already.");
99 Format format;
100 format.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
101 videoDecoder_->SetParameter(format);
102 videoDecoder_->Start();
103 return Status::OK;
104 }
105 videoDecoder_ = MediaAVCodec::VideoDecoderFactory::CreateByMime(trackMime_);
106 CHECK_AND_RETURN_RET_LOG(videoDecoder_ != nullptr, Status::ERROR_NO_MEMORY, "Create videoDecoder_ is nullptr");
107 Format trackFormat{};
108 trackFormat.SetMeta(GetVideoTrackInfo());
109 trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width_);
110 trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height_);
111 MEDIA_LOGI("0x%{public}06" PRIXPTR " Init decoder trackFormat width:%{public}d, height:%{public}d",
112 FAKE_POINTER(this), width_, height_);
113 trackFormat.PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT,
114 static_cast<int32_t>(Plugins::VideoPixelFormat::NV12));
115 trackFormat.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
116 videoDecoder_->Configure(trackFormat);
117 std::shared_ptr<MediaAVCodec::MediaCodecCallback> mediaCodecCallback =
118 std::make_shared<ThumnGeneratorCodecCallback>(this);
119 videoDecoder_->SetCallback(mediaCodecCallback);
120 videoDecoder_->Prepare();
121 auto res = videoDecoder_->Start();
122 CHECK_AND_RETURN_RET(res == MSERR_OK, Status::ERROR_WRONG_STATE);
123 return Status::OK;
124 }
125
GetVideoTrackInfo()126 std::shared_ptr<Meta> AVThumbnailGenerator::GetVideoTrackInfo()
127 {
128 CHECK_AND_RETURN_RET(trackInfo_ == nullptr, trackInfo_);
129 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "GetTargetTrackInfo demuxer is nullptr");
130 std::vector<std::shared_ptr<Meta>> trackInfos = mediaDemuxer_->GetStreamMetaInfo();
131 size_t trackCount = trackInfos.size();
132 CHECK_AND_RETURN_RET_LOG(trackCount > 0, nullptr, "GetTargetTrackInfo trackCount is invalid");
133 for (size_t index = 0; index < trackCount; index++) {
134 if (!(trackInfos[index]->GetData(Tag::MIME_TYPE, trackMime_))) {
135 MEDIA_LOGW("GetTargetTrackInfo get mime type failed %{public}s", trackMime_.c_str());
136 continue;
137 }
138 if (trackMime_.find("video/") == 0) {
139 Plugins::MediaType mediaType;
140 CHECK_AND_RETURN_RET_LOG(trackInfos[index]->GetData(Tag::MEDIA_TYPE, mediaType), nullptr,
141 "GetTargetTrackInfo failed to get mediaType, index:%{public}d", index);
142 CHECK_AND_RETURN_RET_LOG(
143 mediaType == Plugins::MediaType::VIDEO, nullptr,
144 "GetTargetTrackInfo mediaType is not video, index:%{public}d, mediaType:%{public}d", index,
145 static_cast<int32_t>(mediaType));
146 CHECK_AND_RETURN_RET_LOG(trackInfos[index]->Get<Tag::VIDEO_FRAME_RATE>(frameRate_) && frameRate_ > 0,
147 nullptr, "failed to get video frame rate");
148 trackIndex_ = index;
149 MEDIA_LOGI("0x%{public}06" PRIXPTR " GetTrackInfo success trackIndex_:%{public}d, trackMime_:%{public}s",
150 FAKE_POINTER(this), trackIndex_, trackMime_.c_str());
151 if (trackInfos[index]->Get<Tag::VIDEO_ROTATION>(rotation_)) {
152 MEDIA_LOGI("rotation %{public}d", static_cast<int32_t>(rotation_));
153 }
154 return trackInfos[trackIndex_];
155 }
156 }
157 MEDIA_LOGW("GetTargetTrackInfo FAILED.");
158 return nullptr;
159 }
160
OnOutputFormatChanged(const MediaAVCodec::Format & format)161 void AVThumbnailGenerator::OnOutputFormatChanged(const MediaAVCodec::Format &format)
162 {
163 MEDIA_LOGD("OnOutputFormatChanged");
164 outputFormat_ = format;
165 int32_t width = 0;
166 int32_t height = 0;
167 bool hasWidth = format.GetIntValue(Tag::VIDEO_PIC_WIDTH, width);
168 bool hasHeight = format.GetIntValue(Tag::VIDEO_PIC_HEIGHT, height);
169 CHECK_AND_RETURN_LOG(hasWidth && hasHeight, "OutputFormat doesn't have width or height");
170 width_ = width;
171 height_ = height;
172 }
173
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)174 void AVThumbnailGenerator::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
175 {
176 MEDIA_LOGD("OnInputBufferAvailable index:%{public}u", index);
177 if (stopProcessing_.load() || hasFetchedFrame_.load()) {
178 MEDIA_LOGD("stop or has fetched frame, need not queue input buffer");
179 return;
180 }
181 CHECK_AND_RETURN_LOG(mediaDemuxer_ != nullptr, "OnInputBufferAvailable demuxer is nullptr.");
182 CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "OnInputBufferAvailable decoder is nullptr.");
183 mediaDemuxer_->ReadSample(trackIndex_, buffer);
184 videoDecoder_->QueueInputBuffer(index);
185 }
186
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)187 void AVThumbnailGenerator::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
188 {
189 MEDIA_LOGD("OnOutputBufferAvailable index:%{public}u , pts %{public}ld", index, buffer->pts_);
190 CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "Video decoder not exist");
191 bool isValidBuffer = buffer != nullptr && buffer->memory_ != nullptr &&
192 (buffer->memory_->GetSize() != 0 || buffer->memory_->GetSurfaceBuffer() != nullptr);
193 bool isValidState = !hasFetchedFrame_.load() && !stopProcessing_.load();
194 if (!isValidBuffer || !isValidState) {
195 MEDIA_LOGW("isValidBuffer %{public}d isValidState %{public}d", isValidBuffer, isValidState);
196 videoDecoder_->ReleaseOutputBuffer(index, false);
197 return;
198 }
199 bool isClosest = seekMode_ == Plugins::SeekMode::SEEK_CLOSEST;
200 bool isAvailableFrame = !isClosest || buffer->pts_ >= seekTime_ ||
201 (buffer->flag_ & (uint32_t)(AVBufferFlag::EOS));
202 if (!isAvailableFrame) {
203 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
204 bufferIndex_ = index;
205 avBuffer_ = buffer;
206 return;
207 }
208 if (isAvailableFrame) {
209 hasFetchedFrame_ = true;
210 if (isClosest && avBuffer_ != nullptr) {
211 int64_t preDiff = seekTime_ - avBuffer_->pts_;
212 int64_t nextDiff = buffer->pts_ - seekTime_;
213 if (preDiff > nextDiff && !(buffer->flag_ & (uint32_t)(AVBufferFlag::EOS))) {
214 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
215 bufferIndex_ = index;
216 avBuffer_ = buffer;
217 } else {
218 videoDecoder_->ReleaseOutputBuffer(index, false);
219 }
220 } else {
221 bufferIndex_ = index;
222 avBuffer_ = buffer;
223 }
224 MEDIA_LOGI("dstTime %{public}ld resTime %{public}ld", seekTime_, buffer->pts_);
225 PauseFetchFrame();
226 cond_.notify_all();
227 return;
228 }
229 videoDecoder_->ReleaseOutputBuffer(index, false);
230 }
231
FetchFrameAtTime(int64_t timeUs,int32_t option,const OutputConfiguration & param)232 std::shared_ptr<AVSharedMemory> AVThumbnailGenerator::FetchFrameAtTime(int64_t timeUs, int32_t option,
233 const OutputConfiguration ¶m)
234 {
235 MEDIA_LOGI("Fetch frame 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64 ", option:%{public}d,"
236 "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d",
237 FAKE_POINTER(this), timeUs, option, param.dstWidth, param.dstHeight,
238 static_cast<int32_t>(param.colorFormat));
239 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr");
240
241 hasFetchedFrame_ = false;
242 outputConfig_ = param;
243 seekTime_ = timeUs;
244 trackInfo_ = GetVideoTrackInfo();
245 CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr.");
246 mediaDemuxer_->Resume();
247 mediaDemuxer_->SelectTrack(trackIndex_);
248 int64_t realSeekTime = timeUs;
249 auto res = SeekToTime(Plugins::Us2Ms(timeUs), static_cast<Plugins::SeekMode>(option), realSeekTime);
250 CHECK_AND_RETURN_RET_LOG(res == Status::OK, nullptr, "Seek fail");
251 CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed.");
252 {
253 std::unique_lock<std::mutex> lock(mutex_);
254
255 // wait up to 3s to fetch frame AVSharedMemory at time.
256 if (cond_.wait_for(lock, std::chrono::seconds(3), [this] { return hasFetchedFrame_.load(); })) {
257 MEDIA_LOGI("0x%{public}06" PRIXPTR " Fetch frame OK width:%{public}d, height:%{public}d",
258 FAKE_POINTER(this), outputConfig_.dstWidth, outputConfig_.dstHeight);
259 ConvertToAVSharedMemory();
260 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
261 } else {
262 PauseFetchFrame();
263 }
264 }
265 return fetchedFrameAtTime_;
266 }
267
FetchFrameYuv(int64_t timeUs,int32_t option,const OutputConfiguration & param)268 std::shared_ptr<AVBuffer> AVThumbnailGenerator::FetchFrameYuv(int64_t timeUs, int32_t option,
269 const OutputConfiguration ¶m)
270 {
271 MEDIA_LOGI("Fetch frame 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64 ", option:%{public}d,"
272 "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d",
273 FAKE_POINTER(this), timeUs, option, param.dstWidth, param.dstHeight,
274 static_cast<int32_t>(param.colorFormat));
275 CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr");
276 avBuffer_ = nullptr;
277 hasFetchedFrame_ = false;
278 outputConfig_ = param;
279 seekTime_ = timeUs;
280 trackInfo_ = GetVideoTrackInfo();
281 CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr.");
282 mediaDemuxer_->SelectTrack(trackIndex_);
283 int64_t realSeekTime = timeUs;
284 auto res = SeekToTime(Plugins::Us2Ms(timeUs), static_cast<Plugins::SeekMode>(option), realSeekTime);
285 CHECK_AND_RETURN_RET_LOG(res == Status::OK, nullptr, "Seek fail");
286 CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed.");
287 {
288 std::unique_lock<std::mutex> lock(mutex_);
289
290 // wait up to 3s to fetch frame AVSharedMemory at time.
291 if (cond_.wait_for(lock, std::chrono::seconds(3), [this] { return hasFetchedFrame_.load(); })) {
292 MEDIA_LOGI("0x%{public}06" PRIXPTR " Fetch frame OK width:%{public}d, height:%{public}d",
293 FAKE_POINTER(this), outputConfig_.dstWidth, outputConfig_.dstHeight);
294 avBuffer_ = GenerateAlignmentAvBuffer();
295 if (avBuffer_ != nullptr) {
296 avBuffer_->meta_->Set<Tag::VIDEO_WIDTH>(width_);
297 avBuffer_->meta_->Set<Tag::VIDEO_HEIGHT>(height_);
298 avBuffer_->meta_->Set<Tag::VIDEO_ROTATION>(rotation_);
299 }
300 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
301 } else {
302 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
303 PauseFetchFrame();
304 }
305 }
306 return avBuffer_;
307 }
308
SeekToTime(int64_t timeMs,Plugins::SeekMode option,int64_t realSeekTime)309 Status AVThumbnailGenerator::SeekToTime(int64_t timeMs, Plugins::SeekMode option, int64_t realSeekTime)
310 {
311 seekMode_ = option;
312 if (option == Plugins::SeekMode::SEEK_CLOSEST) {
313 option = Plugins::SeekMode::SEEK_PREVIOUS_SYNC;
314 }
315 auto res = mediaDemuxer_->SeekTo(timeMs, option, realSeekTime);
316 /* SEEK_NEXT_SYNC or SEEK_PREVIOUS_SYNC may cant find I frame and return seek failed
317 if seek failed, use SEEK_CLOSEST_SYNC seek again */
318 if (res != Status::OK && option != Plugins::SeekMode::SEEK_CLOSEST_SYNC) {
319 res = mediaDemuxer_->SeekTo(timeMs, Plugins::SeekMode::SEEK_CLOSEST_SYNC, realSeekTime);
320 seekMode_ = Plugins::SeekMode::SEEK_CLOSEST_SYNC;
321 }
322 return res;
323 }
324
ConvertToAVSharedMemory()325 void AVThumbnailGenerator::ConvertToAVSharedMemory()
326 {
327 auto surfaceBuffer = avBuffer_->memory_->GetSurfaceBuffer();
328 if (surfaceBuffer != nullptr) {
329 auto ret = GetYuvDataAlignStride(surfaceBuffer);
330 CHECK_AND_RETURN_LOG(ret == MSERR_OK, "Copy frame failed");
331 OutputFrame *frame = reinterpret_cast<OutputFrame *>(fetchedFrameAtTime_->GetBase());
332 frame->width_ = surfaceBuffer->GetWidth();
333 frame->height_ = surfaceBuffer->GetHeight();
334 frame->stride_ = frame->width_ * RATE_UV;
335 frame->bytesPerPixel_ = RATE_UV;
336 frame->size_ = frame->width_ * frame->height_ * BYTES_PER_PIXEL_YUV;
337 frame->rotation_ = static_cast<int32_t>(rotation_);
338 return;
339 }
340
341 int32_t width;
342 int32_t height;
343 outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width);
344 outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height);
345 if (width == 0 || height == 0) {
346 width = width_;
347 height = height_;
348 }
349
350 fetchedFrameAtTime_ = std::make_shared<AVSharedMemoryBase>(sizeof(OutputFrame) + avBuffer_->memory_->GetSize(),
351 AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory");
352 int32_t ret = fetchedFrameAtTime_->Init();
353 CHECK_AND_RETURN_LOG(ret == static_cast<int32_t>(Status::OK), "Create AVSharedmemory failed, ret:%{public}d", ret);
354 OutputFrame *frame = reinterpret_cast<OutputFrame *>(fetchedFrameAtTime_->GetBase());
355 frame->width_ = width;
356 frame->height_ = height;
357 frame->stride_ = width * RATE_UV;
358 frame->bytesPerPixel_ = RATE_UV;
359 frame->size_ = avBuffer_->memory_->GetSize();
360 frame->rotation_ = static_cast<int32_t>(rotation_);
361 fetchedFrameAtTime_->Write(avBuffer_->memory_->GetAddr(), frame->size_, sizeof(OutputFrame));
362 }
363
ConvertP010ToNV12(const sptr<SurfaceBuffer> & surfaceBuffer,uint8_t * dstNV12,int32_t strideWidth,int32_t strideHeight)364 void AVThumbnailGenerator::ConvertP010ToNV12(const sptr<SurfaceBuffer> &surfaceBuffer, uint8_t *dstNV12,
365 int32_t strideWidth, int32_t strideHeight)
366 {
367 int32_t width = surfaceBuffer->GetWidth();
368 int32_t height = surfaceBuffer->GetHeight();
369 uint8_t *srcP010 = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
370
371 // copy src Y component to dst
372 for (int32_t i = 0; i < height; i++) {
373 uint16_t *srcY = reinterpret_cast<uint16_t *>(srcP010 + strideWidth * i);
374 uint8_t *dstY = dstNV12 + width * i;
375 for (int32_t j = 0; j < width; j++) {
376 *dstY = static_cast<uint8_t>(*srcY >> SHIFT_BITS_P010_2_NV12);
377 srcY++;
378 dstY++;
379 }
380 }
381
382 uint8_t *maxDstAddr = dstNV12 + width * height + width * height / RATE_UV;
383 uint8_t *originSrcAddr = srcP010;
384 uint16_t *maxSrcAddr = reinterpret_cast<uint16_t *>(originSrcAddr) + surfaceBuffer->GetSize();
385
386 srcP010 = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + strideWidth * strideHeight;
387 dstNV12 = dstNV12 + width * height;
388
389 // copy src UV component to dst, height(UV) = height(Y) / 2;
390 for (int32_t i = 0; i < height / 2; i++) {
391 uint16_t *srcUV = reinterpret_cast<uint16_t *>(srcP010 + strideWidth * i);
392 uint8_t *dstUV = dstNV12 + width * i;
393 for (int32_t j = 0; j < width && srcUV < maxSrcAddr && dstUV < maxDstAddr; j++) {
394 *dstUV = static_cast<uint8_t>(*srcUV >> SHIFT_BITS_P010_2_NV12);
395 *(dstUV + 1) = static_cast<uint8_t>(*(srcUV + 1) >> SHIFT_BITS_P010_2_NV12);
396 srcUV += 2; // srcUV move by 2 to process U and V component
397 dstUV += 2; // dstUV move by 2 to process U and V component
398 }
399 }
400 }
401
GenerateAlignmentAvBuffer()402 std::shared_ptr<AVBuffer> AVThumbnailGenerator::GenerateAlignmentAvBuffer()
403 {
404 if (avBuffer_ == nullptr) {
405 MEDIA_LOGE("Generate Alignment AvBuffer failed, avBuffer_ is nullptr.");
406 return nullptr;
407 }
408 if (avBuffer_->memory_->GetSize() != 0 && avBuffer_->memory_->GetSurfaceBuffer() == nullptr) {
409 return GenerateAvBufferFromFCodec();
410 }
411 auto srcSurfaceBuffer = avBuffer_->memory_->GetSurfaceBuffer();
412 auto width = srcSurfaceBuffer->GetWidth();
413 auto height = srcSurfaceBuffer->GetHeight();
414 bool isHdr = srcSurfaceBuffer->GetFormat() ==
415 static_cast<int32_t>(GraphicPixelFormat::GRAPHIC_PIXEL_FMT_YCBCR_P010);
416 if (isHdr) {
417 sptr<SurfaceBuffer> dstSurfaceBuffer = SurfaceBuffer::Create();
418 BufferRequestConfig requestConfig = {
419 .width = width,
420 .height = height,
421 .strideAlignment = 0x2,
422 .format = srcSurfaceBuffer->GetFormat(), // always yuv
423 .usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA | BUFFER_USAGE_MEM_MMZ_CACHE,
424 .timeout = 0,
425 };
426 GSError allocRes = dstSurfaceBuffer->Alloc(requestConfig);
427 CHECK_AND_RETURN_RET_LOG(allocRes == 0, nullptr, "Alloc surfaceBuffer failed, ecode %{public}d", allocRes);
428
429 // create avBuffer from surfaceBuffer
430 std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(dstSurfaceBuffer);
431 bool ret = targetAvBuffer && targetAvBuffer->memory_ && targetAvBuffer->memory_->GetSurfaceBuffer() != nullptr;
432 CHECK_AND_RETURN_RET_LOG(ret, nullptr, "create avBuffer failed");
433 targetAvBuffer->meta_->Set<Tag::VIDEO_IS_HDR_VIVID>(true);
434 CopySurfaceBufferInfo(srcSurfaceBuffer, dstSurfaceBuffer);
435 CopySurfaceBufferPixels(srcSurfaceBuffer, targetAvBuffer);
436 return targetAvBuffer;
437 }
438
439 // not hdr, just copy pixels
440 std::shared_ptr<AVAllocator> allocator = AVAllocatorFactory::CreateSharedAllocator(MemoryFlag::MEMORY_READ_WRITE);
441 CHECK_AND_RETURN_RET_LOG(allocator != nullptr, nullptr, "create avBuffer allocator failed");
442 int32_t bufferSize = width * height * BYTES_PER_PIXEL_YUV;
443 std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(allocator, bufferSize);
444 CHECK_AND_RETURN_RET_LOG(targetAvBuffer, nullptr, "create avBuffer failed");
445 CopySurfaceBufferPixels(srcSurfaceBuffer, targetAvBuffer);
446 return targetAvBuffer;
447 }
448
GenerateAvBufferFromFCodec()449 std::shared_ptr<AVBuffer> AVThumbnailGenerator::GenerateAvBufferFromFCodec()
450 {
451 AVBufferConfig avBufferConfig;
452 avBufferConfig.size = avBuffer_->memory_->GetSize();
453 avBufferConfig.memoryType = MemoryType::SHARED_MEMORY;
454 avBufferConfig.memoryFlag = MemoryFlag::MEMORY_READ_WRITE;
455 std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(avBufferConfig);
456 CHECK_AND_RETURN_RET_LOG(targetAvBuffer != nullptr, nullptr, "Create avBuffer failed");
457 targetAvBuffer->memory_->Write(avBuffer_->memory_->GetAddr(), avBuffer_->memory_->GetSize(), 0);
458 return targetAvBuffer;
459 }
460
GetYuvDataAlignStride(const sptr<SurfaceBuffer> & surfaceBuffer)461 int32_t AVThumbnailGenerator::GetYuvDataAlignStride(const sptr<SurfaceBuffer> &surfaceBuffer)
462 {
463 int32_t width = surfaceBuffer->GetWidth();
464 int32_t height = surfaceBuffer->GetHeight();
465 int32_t stride = surfaceBuffer->GetStride();
466 int32_t outputHeight;
467 auto hasSliceHeight = outputFormat_.GetIntValue(Tag::VIDEO_SLICE_HEIGHT, outputHeight);
468 if (!hasSliceHeight || outputHeight < height) {
469 outputHeight = height;
470 }
471 MEDIA_LOGD("GetYuvDataAlignStride stride:%{public}d, strideWidth:%{public}d, outputHeight:%{public}d", stride,
472 stride, outputHeight);
473
474 fetchedFrameAtTime_ =
475 std::make_shared<AVSharedMemoryBase>(sizeof(OutputFrame) + width * height * BYTES_PER_PIXEL_YUV,
476 AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory");
477 auto ret = fetchedFrameAtTime_->Init();
478 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Create AVSharedmemory failed, ret:%{public}d");
479 uint8_t *dstPtr = static_cast<uint8_t *>(sizeof(OutputFrame) + fetchedFrameAtTime_->GetBase());
480 uint8_t *srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
481 int32_t format = surfaceBuffer->GetFormat();
482 if (format == static_cast<int32_t>(GraphicPixelFormat::GRAPHIC_PIXEL_FMT_YCBCR_P010)) {
483 ConvertP010ToNV12(surfaceBuffer, dstPtr, stride, outputHeight);
484 return MSERR_OK;
485 }
486
487 // copy src Y component to dst
488 for (int32_t y = 0; y < height; y++) {
489 auto ret = memcpy_s(dstPtr, width, srcPtr, width);
490 if (ret != EOK) {
491 MEDIA_LOGW("Memcpy Y component failed.");
492 }
493 srcPtr += stride;
494 dstPtr += width;
495 }
496
497 srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + stride * outputHeight;
498
499 // copy src UV component to dst, height(UV) = height(Y) / 2
500 for (int32_t uv = 0; uv < height / 2; uv++) {
501 auto ret = memcpy_s(dstPtr, width, srcPtr, width);
502 if (ret != EOK) {
503 MEDIA_LOGW("Memcpy UV component failed.");
504 }
505 srcPtr += stride;
506 dstPtr += width;
507 }
508 return MSERR_OK;
509 }
510
CopySurfaceBufferPixels(const sptr<SurfaceBuffer> & surfaceBuffer,std::shared_ptr<AVBuffer> & avBuffer)511 int32_t AVThumbnailGenerator::CopySurfaceBufferPixels(const sptr<SurfaceBuffer> &surfaceBuffer,
512 std::shared_ptr<AVBuffer> &avBuffer)
513 {
514 CHECK_AND_RETURN_RET(surfaceBuffer != nullptr && avBuffer != nullptr && avBuffer->memory_ != nullptr,
515 MSERR_INVALID_VAL);
516 int32_t width = surfaceBuffer->GetWidth();
517 int32_t height = surfaceBuffer->GetHeight();
518 int32_t stride = surfaceBuffer->GetStride();
519
520 bool isHdr = false;
521 (void)avBuffer->meta_->Get<Tag::VIDEO_IS_HDR_VIVID>(isHdr);
522
523 int32_t outputHeight;
524 auto hasSliceHeight = outputFormat_.GetIntValue(Tag::VIDEO_SLICE_HEIGHT, outputHeight);
525 if (!hasSliceHeight || outputHeight < height) {
526 outputHeight = height;
527 }
528 MEDIA_LOGI("width %{public}d stride %{public}d outputHeight %{public}d", width, stride, outputHeight);
529
530 uint8_t *srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
531 uint8_t *dstPtr = nullptr;
532 if (avBuffer->memory_->GetSurfaceBuffer() != nullptr) {
533 dstPtr = static_cast<uint8_t *>(avBuffer->memory_->GetSurfaceBuffer()->GetVirAddr());
534 } else {
535 dstPtr = avBuffer->memory_->GetAddr();
536 }
537
538 // copy src Y component to dst
539 int32_t lineByteCount = isHdr ? stride : width;
540 for (int32_t y = 0; y < height; y++) {
541 auto ret = memcpy_s(dstPtr, lineByteCount, srcPtr, lineByteCount);
542 TRUE_LOG(ret != EOK, MEDIA_LOGW, "Memcpy UV component failed.");
543 srcPtr += stride;
544 dstPtr += lineByteCount;
545 }
546
547 srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + stride * outputHeight;
548
549 // copy src UV component to dst, height(UV) = height(Y) / 2
550 for (int32_t uv = 0; uv < height / RATE_UV; uv++) {
551 auto ret = memcpy_s(dstPtr, lineByteCount, srcPtr, lineByteCount);
552 TRUE_LOG(ret != EOK, MEDIA_LOGW, "Memcpy UV component failed.");
553 srcPtr += stride;
554 dstPtr += lineByteCount;
555 }
556 return MSERR_OK;
557 }
558
Reset()559 void AVThumbnailGenerator::Reset()
560 {
561 if (mediaDemuxer_ != nullptr) {
562 mediaDemuxer_->Reset();
563 }
564
565 if (videoDecoder_ != nullptr) {
566 videoDecoder_->Reset();
567 }
568
569 hasFetchedFrame_ = false;
570 trackInfo_ = nullptr;
571 }
572
CopySurfaceBufferInfo(sptr<SurfaceBuffer> & source,sptr<SurfaceBuffer> & dst)573 void AVThumbnailGenerator::CopySurfaceBufferInfo(sptr<SurfaceBuffer> &source, sptr<SurfaceBuffer> &dst)
574 {
575 if (source == nullptr || dst == nullptr) {
576 MEDIA_LOGI("CopySurfaceBufferInfo failed, source or dst is nullptr");
577 return;
578 }
579 std::vector<uint8_t> hdrMetadataTypeVec;
580 std::vector<uint8_t> colorSpaceInfoVec;
581 std::vector<uint8_t> staticData;
582 std::vector<uint8_t> dynamicData;
583
584 if (source->GetMetadata(ATTRKEY_HDR_METADATA_TYPE, hdrMetadataTypeVec) == GSERROR_OK) {
585 dst->SetMetadata(ATTRKEY_HDR_METADATA_TYPE, hdrMetadataTypeVec);
586 }
587 if (source->GetMetadata(ATTRKEY_COLORSPACE_INFO, colorSpaceInfoVec) == GSERROR_OK) {
588 dst->SetMetadata(ATTRKEY_COLORSPACE_INFO, colorSpaceInfoVec);
589 }
590 if (GetSbStaticMetadata(source, staticData) && (staticData.size() > 0)) {
591 SetSbStaticMetadata(dst, staticData);
592 }
593 if (GetSbDynamicMetadata(source, dynamicData) && (dynamicData.size()) > 0) {
594 SetSbDynamicMetadata(dst, dynamicData);
595 }
596 }
597
GetSbStaticMetadata(const sptr<SurfaceBuffer> & buffer,std::vector<uint8_t> & staticMetadata)598 bool AVThumbnailGenerator::GetSbStaticMetadata(const sptr<SurfaceBuffer> &buffer, std::vector<uint8_t> &staticMetadata)
599 {
600 return buffer->GetMetadata(ATTRKEY_HDR_STATIC_METADATA, staticMetadata) == GSERROR_OK;
601 }
602
GetSbDynamicMetadata(const sptr<SurfaceBuffer> & buffer,std::vector<uint8_t> & dynamicMetadata)603 bool AVThumbnailGenerator::GetSbDynamicMetadata(const sptr<SurfaceBuffer> &buffer,
604 std::vector<uint8_t> &dynamicMetadata)
605 {
606 return buffer->GetMetadata(ATTRKEY_HDR_DYNAMIC_METADATA, dynamicMetadata) == GSERROR_OK;
607 }
608
SetSbStaticMetadata(sptr<SurfaceBuffer> & buffer,const std::vector<uint8_t> & staticMetadata)609 bool AVThumbnailGenerator::SetSbStaticMetadata(sptr<SurfaceBuffer> &buffer, const std::vector<uint8_t> &staticMetadata)
610 {
611 return buffer->SetMetadata(ATTRKEY_HDR_STATIC_METADATA, staticMetadata) == GSERROR_OK;
612 }
613
SetSbDynamicMetadata(sptr<SurfaceBuffer> & buffer,const std::vector<uint8_t> & dynamicMetadata)614 bool AVThumbnailGenerator::SetSbDynamicMetadata(sptr<SurfaceBuffer> &buffer,
615 const std::vector<uint8_t> &dynamicMetadata)
616 {
617 return buffer->SetMetadata(ATTRKEY_HDR_DYNAMIC_METADATA, dynamicMetadata) == GSERROR_OK;
618 }
619
Destroy()620 void AVThumbnailGenerator::Destroy()
621 {
622 stopProcessing_ = true;
623 if (videoDecoder_ != nullptr) {
624 videoDecoder_->Stop();
625 videoDecoder_->Release();
626 }
627 mediaDemuxer_ = nullptr;
628 videoDecoder_ = nullptr;
629 MEDIA_LOGI("0x%{public}06" PRIXPTR " Finish Destroy.", FAKE_POINTER(this));
630 }
631
PauseFetchFrame()632 void AVThumbnailGenerator::PauseFetchFrame()
633 {
634 hasFetchedFrame_ = true;
635 mediaDemuxer_->Pause();
636 mediaDemuxer_->Flush();
637 videoDecoder_->Flush();
638 Format format;
639 format.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, frameRate_);
640 videoDecoder_->SetParameter(format);
641 }
642 } // namespace Media
643 } // namespace OHOS