1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmetadatahelper_engine_gst_impl.h"
17 #include <gst/gst.h>
18 #include "media_errors.h"
19 #include "media_log.h"
20 #include "i_playbin_ctrler.h"
21 #include "avmeta_sinkprovider.h"
22 #include "avmeta_frame_extractor.h"
23 #include "avmeta_meta_collector.h"
24 #include "scope_guard.h"
25 #include "uri_helper.h"
26 #include "media_dfx.h"
27
28 namespace {
29 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaEngineGstImpl"};
30 const std::map<OHOS::Media::PlayBinState, std::string> AVMETADATA_STATE_MAP = {
31 {OHOS::Media::PLAYBIN_STATE_IDLE, "idle"},
32 {OHOS::Media::PLAYBIN_STATE_INITIALIZED, "initialized"},
33 {OHOS::Media::PLAYBIN_STATE_PREPARING, "preparing"},
34 {OHOS::Media::PLAYBIN_STATE_PREPARED, "prepared"},
35 {OHOS::Media::PLAYBIN_STATE_PLAYING, "playing"},
36 {OHOS::Media::PLAYBIN_STATE_PAUSED, "paused"},
37 {OHOS::Media::PLAYBIN_STATE_STOPPED, "stopped"},
38 {OHOS::Media::PLAYBIN_STATE_PLAYBACK_COMPLETE, "playbackcomplete"},
39 };
40 }
41
42 namespace OHOS {
43 namespace Media {
44 static const std::set<PixelFormat> SUPPORTED_PIXELFORMAT = {
45 PixelFormat::RGB_565, PixelFormat::RGB_888, PixelFormat::RGBA_8888
46 };
47
CheckFrameFetchParam(int64_t timeUsOrIndex,int32_t option,const OutputConfiguration & param)48 static bool CheckFrameFetchParam(int64_t timeUsOrIndex, int32_t option, const OutputConfiguration ¶m)
49 {
50 (void)timeUsOrIndex;
51 if ((option != AV_META_QUERY_CLOSEST) && (option != AV_META_QUERY_CLOSEST_SYNC) &&
52 (option != AV_META_QUERY_NEXT_SYNC) && (option != AV_META_QUERY_PREVIOUS_SYNC)) {
53 MEDIA_LOGE("Invalid query option: %{public}d", option);
54 return false;
55 }
56
57 if (SUPPORTED_PIXELFORMAT.count(param.colorFormat) == 0) {
58 MEDIA_LOGE("Unsupported pixelformat: %{public}d", param.colorFormat);
59 return false;
60 }
61
62 static constexpr int32_t maxDstWidth = 7680;
63 static constexpr int32_t minDstWidth = 32;
64 if (param.dstWidth > maxDstWidth || (param.dstWidth < minDstWidth && param.dstWidth != -1)) {
65 MEDIA_LOGE("Invalid dstWidth: %{public}d", param.dstWidth);
66 return false;
67 }
68
69 static constexpr int32_t maxDstHeight = 4320;
70 static constexpr int32_t minDstHeight = 32;
71 if (param.dstHeight > maxDstHeight || (param.dstHeight < minDstHeight && param.dstHeight != -1)) {
72 MEDIA_LOGE("Invalid dstHeight: %{public}d", param.dstHeight);
73 return false;
74 }
75
76 return true;
77 }
78
AVMetadataHelperEngineGstImpl()79 AVMetadataHelperEngineGstImpl::AVMetadataHelperEngineGstImpl()
80 {
81 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
82 }
83
~AVMetadataHelperEngineGstImpl()84 AVMetadataHelperEngineGstImpl::~AVMetadataHelperEngineGstImpl()
85 {
86 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
87 Reset();
88 }
89
SetSource(const std::string & uri,int32_t usage)90 int32_t AVMetadataHelperEngineGstImpl::SetSource(const std::string &uri, int32_t usage)
91 {
92 if ((usage != AVMetadataUsage::AV_META_USAGE_META_ONLY) &&
93 (usage != AVMetadataUsage::AV_META_USAGE_PIXEL_MAP)) {
94 MEDIA_LOGE("Invalid avmetadatahelper usage: %{public}d", usage);
95 return MSERR_INVALID_VAL;
96 }
97
98 UriHelper uriHelper(uri);
99 if (uriHelper.UriType() != UriHelper::URI_TYPE_FILE && uriHelper.UriType() != UriHelper::URI_TYPE_FD) {
100 MEDIA_LOGE("Unsupported uri type : %{public}s", uri.c_str());
101 return MSERR_UNSUPPORT;
102 }
103
104 MEDIA_LOGI("uri: %{public}s, usage: %{public}d", uri.c_str(), usage);
105
106 int32_t ret = SetSourceInternel(uri, usage);
107 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Failed to call SetSourceInternel");
108
109 MEDIA_LOGI("set source success");
110 return MSERR_OK;
111 }
112
ResolveMetadata(int32_t key)113 std::string AVMetadataHelperEngineGstImpl::ResolveMetadata(int32_t key)
114 {
115 MEDIA_LOGD("enter");
116 std::string result;
117
118 int32_t ret = ExtractMetadata();
119 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, result, "Failed to call ExtractMetadata");
120
121 if (collectedMeta_.count(key) == 0 || collectedMeta_.at(key).empty()) {
122 MEDIA_LOGE("The specified metadata %{public}d cannot be obtained from the specified stream.", key);
123 return result;
124 }
125
126 MEDIA_LOGD("exit");
127 result = collectedMeta_[key];
128 return result;
129 }
130
ResolveMetadata()131 std::unordered_map<int32_t, std::string> AVMetadataHelperEngineGstImpl::ResolveMetadata()
132 {
133 MEDIA_LOGD("enter");
134
135 int32_t ret = ExtractMetadata();
136 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, {}, "Failed to call ExtractMetadata");
137
138 MEDIA_LOGD("exit");
139 return collectedMeta_;
140 }
141
FetchArtPicture()142 std::shared_ptr<AVSharedMemory> AVMetadataHelperEngineGstImpl::FetchArtPicture()
143 {
144 MEDIA_LOGD("enter");
145
146 int32_t ret = ExtractMetadata();
147 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "Failed to call ExtractMetadata");
148
149 auto result = metaCollector_->FetchArtPicture();
150 MEDIA_LOGD("exit");
151 return result;
152 }
153
FetchFrameAtTime(int64_t timeUs,int32_t option,const OutputConfiguration & param)154 std::shared_ptr<AVSharedMemory> AVMetadataHelperEngineGstImpl::FetchFrameAtTime(
155 int64_t timeUs, int32_t option, const OutputConfiguration ¶m)
156 {
157 MEDIA_LOGD("enter");
158
159 if (usage_ != AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) {
160 MEDIA_LOGE("current instance is unavailable for fetch frame, check usage !");
161 return nullptr;
162 }
163
164 std::vector<std::shared_ptr<AVSharedMemory>> outFrames;
165 int32_t ret = FetchFrameInternel(timeUs, option, 1, param, outFrames);
166 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "fetch frame failed");
167
168 MEDIA_LOGD("exit");
169 return outFrames[0];
170 }
171
OnNotifyAutoPlugSort(GValueArray & factories)172 GValueArray *AVMetadataHelperEngineGstImpl::OnNotifyAutoPlugSort(GValueArray &factories)
173 {
174 GValueArray *result = g_value_array_new(factories.n_values);
175
176 for (uint32_t i = 0; i < factories.n_values; i++) {
177 GstElementFactory *factory =
178 static_cast<GstElementFactory *>(g_value_get_object(g_value_array_get_nth(&factories, i)));
179 GValue val = G_VALUE_INIT;
180 g_value_init(&val, G_TYPE_OBJECT);
181 g_value_set_object(&val, factory);
182 result = g_value_array_append(result, &val);
183 g_value_unset(&val);
184 }
185 return result;
186 }
187
SetSourceInternel(const std::string & uri,int32_t usage)188 int32_t AVMetadataHelperEngineGstImpl::SetSourceInternel(const std::string &uri, int32_t usage)
189 {
190 Reset();
191 ON_SCOPE_EXIT(0) { Reset(); };
192
193 uint8_t renderMode = IPlayBinCtrler::PlayBinRenderMode::NATIVE_STREAM;
194 renderMode = renderMode | IPlayBinCtrler::PlayBinRenderMode::DISABLE_TEXT;
195 auto notifier = std::bind(&AVMetadataHelperEngineGstImpl::OnNotifyMessage, this, std::placeholders::_1);
196 sinkProvider_ = std::make_shared<AVMetaSinkProvider>(usage);
197
198 IPlayBinCtrler::PlayBinCreateParam createParam = {
199 static_cast<IPlayBinCtrler::PlayBinRenderMode>(renderMode), notifier, sinkProvider_
200 };
201
202 playBinCtrler_ = IPlayBinCtrler::Create(IPlayBinCtrler::PlayBinKind::PLAYBIN2, createParam);
203 CHECK_AND_RETURN_RET_LOG(playBinCtrler_ != nullptr, MSERR_UNKNOWN, "Failed to call IPlayBinCtrler::Create");
204
205 int32_t ret = playBinCtrler_->SetSource(uri);
206 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Failed to call playBinCtrler_->SetSource");
207
208 metaCollector_ = std::make_unique<AVMetaMetaCollector>();
209 auto listener = std::bind(&AVMetadataHelperEngineGstImpl::OnNotifyElemSetup, this, std::placeholders::_1);
210 playBinCtrler_->SetElemSetupListener(listener);
211
212 listener = std::bind(&AVMetadataHelperEngineGstImpl::OnNotifyElemUnSetup, this, std::placeholders::_1);
213 playBinCtrler_->SetElemUnSetupListener(listener);
214
215 auto autoPlugSortListener =
216 std::bind(&AVMetadataHelperEngineGstImpl::OnNotifyAutoPlugSort, this, std::placeholders::_1);
217 playBinCtrler_->SetAutoPlugSortListener(autoPlugSortListener);
218
219 if (usage == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) {
220 auto vidSink = sinkProvider_->CreateVideoSink();
221 CHECK_AND_RETURN_RET_LOG(vidSink != nullptr, MSERR_UNKNOWN, "get video sink failed");
222 frameExtractor_ = std::make_unique<AVMetaFrameExtractor>();
223 ret = frameExtractor_->Init(playBinCtrler_, *vidSink);
224 if (ret != MSERR_OK) {
225 MEDIA_LOGE("frameExtractor init failed");
226 }
227 }
228
229 metaCollector_->Start();
230 usage_ = usage;
231
232 ret = PrepareInternel(true);
233 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Failed to call PrepareInternel");
234
235 std::string mimeType = metaCollector_->GetMetadata(AV_KEY_MIME_TYPE);
236 if (mimeType.empty()) {
237 MEDIA_LOGE("can not recognize the media source's mimetype, set source failed");
238 Reset();
239 return MSERR_INVALID_OPERATION;
240 }
241
242 CANCEL_SCOPE_EXIT_GUARD(0);
243 return MSERR_OK;
244 }
245
PrepareInternel(bool async)246 int32_t AVMetadataHelperEngineGstImpl::PrepareInternel(bool async)
247 {
248 CHECK_AND_RETURN_RET_LOG(playBinCtrler_ != nullptr, MSERR_INVALID_OPERATION, "set source firstly");
249
250 {
251 std::unique_lock<std::mutex> lock(mutex_);
252 if (status_ == PLAYBIN_STATE_PREPARED || status_ == PLAYBIN_STATE_PLAYING || status_ == PLAYBIN_STATE_PAUSED) {
253 return MSERR_OK;
254 }
255 }
256 asyncDone_ = false;
257 int32_t ret = playBinCtrler_->PrepareAsync();
258 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "prepare failed");
259
260 if (!async) {
261 metaCollector_->Stop(true);
262 static constexpr int32_t timeout = 3;
263 std::unique_lock<std::mutex> lock(mutex_);
264 cond_.wait_for(lock, std::chrono::seconds(timeout), [this]() {
265 return (status_ == PLAYBIN_STATE_PREPARED && asyncDone_) || errHappened_;
266 });
267 CHECK_AND_RETURN_RET_LOG(!errHappened_, MSERR_UNKNOWN, "prepare failed");
268 }
269
270 return MSERR_OK;
271 }
272
FetchFrameInternel(int64_t timeUsOrIndex,int32_t option,int32_t numFrames,const OutputConfiguration & param,std::vector<std::shared_ptr<AVSharedMemory>> & outFrames)273 int32_t AVMetadataHelperEngineGstImpl::FetchFrameInternel(int64_t timeUsOrIndex, int32_t option, int32_t numFrames,
274 const OutputConfiguration ¶m, std::vector<std::shared_ptr<AVSharedMemory>> &outFrames)
275 {
276 (void)numFrames;
277
278 if (!CheckFrameFetchParam(timeUsOrIndex, option, param)) {
279 MEDIA_LOGE("fetch frame's param invalid");
280 return MSERR_INVALID_OPERATION;
281 }
282
283 CHECK_AND_RETURN_RET_LOG(frameExtractor_ != nullptr, MSERR_INVALID_OPERATION, "frameExtractor is nullptr");
284
285 int32_t ret = ExtractMetadata();
286 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Failed to call ExtractMetadata");
287
288 if (collectedMeta_.find(AV_KEY_HAS_VIDEO) == collectedMeta_.end() ||
289 collectedMeta_[AV_KEY_HAS_VIDEO] != "yes") {
290 MEDIA_LOGE("There is no video track in the current media source !");
291 return MSERR_INVALID_OPERATION;
292 }
293
294 if (!metaCollector_->IsCollecteCompleted()) {
295 MEDIA_LOGE("extract meta failed, exit");
296 return MSERR_UNKNOWN;
297 }
298
299 ret = PrepareInternel(false);
300 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Failed to call PrepareInternel");
301
302 auto frame = frameExtractor_->ExtractFrame(timeUsOrIndex, option, param);
303 if (frame == nullptr) {
304 MEDIA_LOGE("fetch frame failed");
305 return MSERR_UNKNOWN;
306 }
307
308 if (firstFetch_) {
309 firstFetch_ = false;
310 }
311
312 outFrames.push_back(frame);
313 return MSERR_OK;
314 }
315
ExtractMetadata()316 int32_t AVMetadataHelperEngineGstImpl::ExtractMetadata()
317 {
318 CHECK_AND_RETURN_RET_LOG(metaCollector_ != nullptr, MSERR_INVALID_OPERATION, "metaCollector is nullptr");
319
320 if (!hasCollectMeta_) {
321 collectedMeta_ = metaCollector_->GetMetadata();
322 hasCollectMeta_ = true;
323 }
324 return MSERR_OK;
325 }
326
Reset()327 void AVMetadataHelperEngineGstImpl::Reset()
328 {
329 // If call Reset immediately after setSource, the state of multiple plug-ins (for example, qtdemux)
330 // will be confused, the ExtractMetadata call is added here to ensure stable state
331 (void)ExtractMetadata();
332 std::unique_lock<std::mutex> lock(mutex_);
333
334 if (metaCollector_ != nullptr) {
335 metaCollector_->Stop();
336 hasCollectMeta_ = false;
337 }
338
339 if (frameExtractor_ != nullptr) {
340 frameExtractor_->Reset();
341 }
342
343 if (playBinCtrler_ != nullptr) {
344 playBinCtrler_->SetElemSetupListener(nullptr);
345 playBinCtrler_->SetAutoPlugSortListener(nullptr);
346
347 auto tmp = playBinCtrler_;
348 playBinCtrler_ = nullptr;
349 // Some msg maybe be reported by the playbinCtrler_ during the playbinCtler_ destroying.
350 // unlock to avoid the deadlock.
351 lock.unlock();
352 tmp->Stop(false);
353 tmp = nullptr;
354 lock.lock();
355 }
356
357 sinkProvider_ = nullptr;
358 metaCollector_ = nullptr;
359 frameExtractor_ = nullptr;
360
361 errHappened_ = false;
362 status_ = PLAYBIN_STATE_IDLE;
363
364 firstFetch_ = true;
365 asyncDone_ = false;
366
367 lock.unlock();
368 lock.lock();
369 }
370
OnNotifyMessage(const PlayBinMessage & msg)371 void AVMetadataHelperEngineGstImpl::OnNotifyMessage(const PlayBinMessage &msg)
372 {
373 switch (msg.type) {
374 case PLAYBIN_MSG_STATE_CHANGE: {
375 std::unique_lock<std::mutex> lock(mutex_);
376 status_ = msg.code;
377 BehaviorEventWrite(
378 GetStatusDescription(static_cast<OHOS::Media::PlayBinState>(status_)), "AVMetadata");
379 cond_.notify_all();
380 if (msg.code == PLAYBIN_STATE_PREPARED) {
381 MEDIA_LOGI("prepare finished");
382 }
383 if (msg.code == PLAYBIN_STATE_STOPPED) {
384 MEDIA_LOGI("stop finished");
385 }
386 break;
387 }
388 case PLAYBIN_MSG_ERROR: {
389 std::unique_lock<std::mutex> lock(mutex_);
390 errHappened_ = true;
391 if (metaCollector_ != nullptr) {
392 metaCollector_->Stop();
393 }
394 if (frameExtractor_ != nullptr) {
395 frameExtractor_->Reset();
396 }
397 cond_.notify_all();
398 MEDIA_LOGE("error happened, cancel inprocessing job");
399 FaultEventWrite("error happened, cancel inprocessing job", "AVMetadata");
400 break;
401 }
402 case PLAYBIN_MSG_SEEKDONE: {
403 std::unique_lock<std::mutex> lock(mutex_);
404 if (frameExtractor_ != nullptr) {
405 frameExtractor_->NotifyPlayBinMsg(msg);
406 }
407 break;
408 }
409 case PLAYBIN_MSG_ASYNC_DONE: {
410 asyncDone_ = true;
411 cond_.notify_all();
412 MEDIA_LOGI("async done");
413 break;
414 }
415 default:
416 break;
417 }
418 }
419
OnNotifyElemSetup(GstElement & elem)420 void AVMetadataHelperEngineGstImpl::OnNotifyElemSetup(GstElement &elem)
421 {
422 const gchar *metadata = gst_element_get_metadata(&elem, GST_ELEMENT_METADATA_KLASS);
423 std::string metaStr(metadata);
424 if (metaStr.find("Codec/Decoder/Video") != std::string::npos) {
425 MEDIA_LOGI("Only need one frame!");
426 g_object_set(const_cast<GstElement *>(&elem), "only-one-frame-required", TRUE, nullptr);
427 MEDIA_LOGI("Set avmeta mode!");
428 g_object_set(const_cast<GstElement *>(&elem), "metadata-mode", TRUE, nullptr);
429 }
430
431 if (metaStr.find("Codec/Decoder/Video/Hardware") != std::string::npos) {
432 g_object_set(const_cast<GstElement *>(&elem), "player-scene", TRUE, nullptr);
433 }
434
435 std::unique_lock<std::mutex> lock(mutex_);
436 if (metaCollector_ != nullptr) {
437 metaCollector_->AddMetaSource(elem);
438 }
439 }
440
OnNotifyElemUnSetup(GstElement & elem)441 void AVMetadataHelperEngineGstImpl::OnNotifyElemUnSetup(GstElement &elem)
442 {
443 std::unique_lock<std::mutex> lock(mutex_);
444 if (metaCollector_ != nullptr) {
445 metaCollector_->RemoveMetaSource(elem);
446 }
447 }
448
GetStatusDescription(OHOS::Media::PlayBinState status)449 const std::string &AVMetadataHelperEngineGstImpl::GetStatusDescription(OHOS::Media::PlayBinState status)
450 {
451 static const std::string ILLEGAL_STATE = "PLAYER_STATUS_ILLEGAL";
452 if (status < OHOS::Media::PLAYBIN_STATE_IDLE || status > OHOS::Media::PLAYBIN_STATE_PLAYBACK_COMPLETE) {
453 return ILLEGAL_STATE;
454 }
455
456 return AVMETADATA_STATE_MAP.find(status)->second;
457 }
458 } // namespace Media
459 } // namespace OHOS