1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmetadatahelper_engine_gst_impl.h"
17 #include <gst/gst.h>
18 #include "media_errors.h"
19 #include "media_log.h"
20 #include "i_playbin_ctrler.h"
21 #include "avmeta_sinkprovider.h"
22 #include "avmeta_frame_extractor.h"
23 #include "avmeta_meta_collector.h"
24 #include "scope_guard.h"
25 #include "uri_helper.h"
26 #include "time_perf.h"
27
28 namespace {
29 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaEngineGstImpl"};
30 }
31
32 namespace OHOS {
33 namespace Media {
34 static const std::set<PixelFormat> SUPPORTED_PIXELFORMAT = {
35 PixelFormat::RGB_565, PixelFormat::RGB_888, PixelFormat::RGBA_8888
36 };
37
CheckFrameFetchParam(int64_t timeUsOrIndex,int32_t option,const OutputConfiguration & param)38 static bool CheckFrameFetchParam(int64_t timeUsOrIndex, int32_t option, const OutputConfiguration ¶m)
39 {
40 (void)timeUsOrIndex;
41 if ((option != AV_META_QUERY_CLOSEST) && (option != AV_META_QUERY_CLOSEST_SYNC) &&
42 (option != AV_META_QUERY_NEXT_SYNC) && (option != AV_META_QUERY_PREVIOUS_SYNC)) {
43 MEDIA_LOGE("Invalid query option: %{public}d", option);
44 return false;
45 }
46
47 if (SUPPORTED_PIXELFORMAT.count(param.colorFormat) == 0) {
48 MEDIA_LOGE("Unsupported pixelformat: %{public}d", param.colorFormat);
49 return false;
50 }
51
52 static constexpr int32_t maxDstWidth = 7680;
53 static constexpr int32_t minDstWidth = 32;
54 if (param.dstWidth > maxDstWidth || (param.dstWidth < minDstWidth && param.dstWidth != -1)) {
55 MEDIA_LOGE("Invalid dstWidth: %{public}d", param.dstWidth);
56 return false;
57 }
58
59 static constexpr int32_t maxDstHeight = 4320;
60 static constexpr int32_t minDstHeight = 32;
61 if (param.dstHeight > maxDstHeight || (param.dstHeight < minDstHeight && param.dstHeight != -1)) {
62 MEDIA_LOGE("Invalid dstHeight: %{public}d", param.dstHeight);
63 return false;
64 }
65
66 return true;
67 }
68
AVMetadataHelperEngineGstImpl()69 AVMetadataHelperEngineGstImpl::AVMetadataHelperEngineGstImpl()
70 {
71 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
72 }
73
~AVMetadataHelperEngineGstImpl()74 AVMetadataHelperEngineGstImpl::~AVMetadataHelperEngineGstImpl()
75 {
76 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
77 Reset();
78 CLEAN_PERF_RECORD(this);
79 }
80
SetSource(const std::string & uri,int32_t usage)81 int32_t AVMetadataHelperEngineGstImpl::SetSource(const std::string &uri, int32_t usage)
82 {
83 if ((usage != AVMetadataUsage::AV_META_USAGE_META_ONLY) &&
84 (usage != AVMetadataUsage::AV_META_USAGE_PIXEL_MAP)) {
85 MEDIA_LOGE("Invalid avmetadatahelper usage: %{public}d", usage);
86 return MSERR_INVALID_VAL;
87 }
88
89 UriHelper uriHelper(uri);
90 if (uriHelper.UriType() != UriHelper::URI_TYPE_FILE && uriHelper.UriType() != UriHelper::URI_TYPE_FD) {
91 MEDIA_LOGE("Unsupported uri type : %{public}s", uri.c_str());
92 return MSERR_UNSUPPORT;
93 }
94
95 MEDIA_LOGI("uri: %{public}s, usage: %{public}d", uri.c_str(), usage);
96
97 if (usage == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) {
98 ASYNC_PERF_START(this, "FirstFetchFrame");
99 }
100
101 int32_t ret = SetSourceInternel(uri, usage);
102 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
103
104 MEDIA_LOGI("set source success");
105 return MSERR_OK;
106 }
107
ResolveMetadata(int32_t key)108 std::string AVMetadataHelperEngineGstImpl::ResolveMetadata(int32_t key)
109 {
110 MEDIA_LOGD("enter");
111 std::string result;
112
113 int32_t ret = ExtractMetadata();
114 CHECK_AND_RETURN_RET(ret == MSERR_OK, result);
115
116 if (collectedMeta_.count(key) == 0 || collectedMeta_.at(key).empty()) {
117 MEDIA_LOGE("The specified metadata %{public}d cannot be obtained from the specified stream.", key);
118 return result;
119 }
120
121 MEDIA_LOGD("exit");
122 result = collectedMeta_[key];
123 return result;
124 }
125
ResolveMetadata()126 std::unordered_map<int32_t, std::string> AVMetadataHelperEngineGstImpl::ResolveMetadata()
127 {
128 MEDIA_LOGD("enter");
129
130 int32_t ret = ExtractMetadata();
131 CHECK_AND_RETURN_RET(ret == MSERR_OK, {});
132
133 MEDIA_LOGD("exit");
134 return collectedMeta_;
135 }
136
FetchArtPicture()137 std::shared_ptr<AVSharedMemory> AVMetadataHelperEngineGstImpl::FetchArtPicture()
138 {
139 MEDIA_LOGD("enter");
140
141 int32_t ret = ExtractMetadata();
142 CHECK_AND_RETURN_RET(ret == MSERR_OK, nullptr);
143
144 auto result = metaCollector_->FetchArtPicture();
145 MEDIA_LOGD("exit");
146 return result;
147 }
148
FetchFrameAtTime(int64_t timeUs,int32_t option,const OutputConfiguration & param)149 std::shared_ptr<AVSharedMemory> AVMetadataHelperEngineGstImpl::FetchFrameAtTime(
150 int64_t timeUs, int32_t option, const OutputConfiguration ¶m)
151 {
152 MEDIA_LOGD("enter");
153
154 if (usage_ != AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) {
155 MEDIA_LOGE("current instance is unavailable for fetch frame, check usage !");
156 return nullptr;
157 }
158
159 std::vector<std::shared_ptr<AVSharedMemory>> outFrames;
160 int32_t ret = FetchFrameInternel(timeUs, option, 1, param, outFrames);
161 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "fetch frame failed");
162
163 MEDIA_LOGD("exit");
164 return outFrames[0];
165 }
166
SetSourceInternel(const std::string & uri,int32_t usage)167 int32_t AVMetadataHelperEngineGstImpl::SetSourceInternel(const std::string &uri, int32_t usage)
168 {
169 Reset();
170 ON_SCOPE_EXIT(0) { Reset(); };
171
172 uint8_t renderMode = IPlayBinCtrler::PlayBinRenderMode::NATIVE_STREAM;
173 renderMode = renderMode | IPlayBinCtrler::PlayBinRenderMode::DISABLE_TEXT;
174 auto notifier = std::bind(&AVMetadataHelperEngineGstImpl::OnNotifyMessage, this, std::placeholders::_1);
175 sinkProvider_ = std::make_shared<AVMetaSinkProvider>(usage);
176
177 IPlayBinCtrler::PlayBinCreateParam createParam = {
178 static_cast<IPlayBinCtrler::PlayBinRenderMode>(renderMode), notifier, sinkProvider_
179 };
180
181 playBinCtrler_ = IPlayBinCtrler::Create(IPlayBinCtrler::PlayBinKind::PLAYBIN2, createParam);
182 CHECK_AND_RETURN_RET(playBinCtrler_ != nullptr, MSERR_UNKNOWN);
183
184 int32_t ret = playBinCtrler_->SetSource(uri);
185 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
186
187 metaCollector_ = std::make_unique<AVMetaMetaCollector>();
188 auto listener = std::bind(&AVMetadataHelperEngineGstImpl::OnNotifyElemSetup, this, std::placeholders::_1);
189 playBinCtrler_->SetElemSetupListener(listener);
190
191 if (usage == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) {
192 auto vidSink = sinkProvider_->CreateVideoSink();
193 CHECK_AND_RETURN_RET_LOG(vidSink != nullptr, MSERR_UNKNOWN, "get video sink failed");
194 frameExtractor_ = std::make_unique<AVMetaFrameExtractor>();
195 ret = frameExtractor_->Init(playBinCtrler_, *vidSink);
196 if (ret != MSERR_OK) {
197 MEDIA_LOGE("frameExtractor init failed");
198 }
199 gst_object_unref(vidSink);
200 }
201
202 metaCollector_->Start();
203 usage_ = usage;
204
205 ret = PrepareInternel(true);
206 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
207
208 std::string mimeType = metaCollector_->GetMetadata(AV_KEY_MIME_TYPE);
209 if (mimeType.empty()) {
210 MEDIA_LOGE("can not recognize the media source's mimetype, set source failed");
211 Reset();
212 return MSERR_INVALID_OPERATION;
213 }
214
215 CANCEL_SCOPE_EXIT_GUARD(0);
216 return MSERR_OK;
217 }
218
PrepareInternel(bool async)219 int32_t AVMetadataHelperEngineGstImpl::PrepareInternel(bool async)
220 {
221 CHECK_AND_RETURN_RET_LOG(playBinCtrler_ != nullptr, MSERR_INVALID_OPERATION, "set source firstly");
222
223 std::unique_lock<std::mutex> lock(mutex_);
224 if (status_ == PLAYBIN_STATE_PREPARED || status_ == PLAYBIN_STATE_PLAYING || status_ == PLAYBIN_STATE_PAUSED) {
225 return MSERR_OK;
226 }
227
228 int32_t ret = playBinCtrler_->PrepareAsync();
229 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "prepare failed");
230
231 if (!async) {
232 metaCollector_->Stop(true);
233 static constexpr int32_t timeout = 5;
234 cond_.wait_for(lock, std::chrono::seconds(timeout), [this]() {
235 return status_ == PLAYBIN_STATE_PREPARED || errHappened_;
236 });
237 CHECK_AND_RETURN_RET_LOG(!errHappened_, MSERR_UNKNOWN, "prepare failed");
238 }
239
240 return MSERR_OK;
241 }
242
FetchFrameInternel(int64_t timeUsOrIndex,int32_t option,int32_t numFrames,const OutputConfiguration & param,std::vector<std::shared_ptr<AVSharedMemory>> & outFrames)243 int32_t AVMetadataHelperEngineGstImpl::FetchFrameInternel(int64_t timeUsOrIndex, int32_t option, int32_t numFrames,
244 const OutputConfiguration ¶m, std::vector<std::shared_ptr<AVSharedMemory>> &outFrames)
245 {
246 (void)numFrames;
247
248 AUTO_PERF(this, "FetchFrame");
249
250 if (!CheckFrameFetchParam(timeUsOrIndex, option, param)) {
251 MEDIA_LOGE("fetch frame's param invalid");
252 return MSERR_INVALID_OPERATION;
253 }
254
255 CHECK_AND_RETURN_RET_LOG(frameExtractor_ != nullptr, MSERR_INVALID_OPERATION, "frameExtractor is nullptr");
256
257 int32_t ret = ExtractMetadata();
258 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
259
260 if (collectedMeta_.find(AV_KEY_HAS_VIDEO) == collectedMeta_.end() ||
261 collectedMeta_[AV_KEY_HAS_VIDEO] != "yes") {
262 MEDIA_LOGE("There is no video track in the current media source !");
263 return MSERR_INVALID_OPERATION;
264 }
265
266 if (!metaCollector_->IsCollecteCompleted()) {
267 MEDIA_LOGE("extract meta failed, exit");
268 return MSERR_UNKNOWN;
269 }
270
271 ret = PrepareInternel(false);
272 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
273
274 auto frame = frameExtractor_->ExtractFrame(timeUsOrIndex, option, param);
275 if (frame == nullptr) {
276 MEDIA_LOGE("fetch frame failed");
277 return MSERR_UNKNOWN;
278 }
279
280 if (firstFetch_) {
281 ASYNC_PERF_STOP(this, "FirstFetchFrame");
282 firstFetch_ = false;
283 }
284
285 outFrames.push_back(frame);
286 return MSERR_OK;
287 }
288
ExtractMetadata()289 int32_t AVMetadataHelperEngineGstImpl::ExtractMetadata()
290 {
291 CHECK_AND_RETURN_RET_LOG(metaCollector_ != nullptr, MSERR_INVALID_OPERATION, "metaCollector is nullptr");
292
293 if (!hasCollectMeta_) {
294 collectedMeta_ = metaCollector_->GetMetadata();
295 hasCollectMeta_ = true;
296 }
297 return MSERR_OK;
298 }
299
Reset()300 void AVMetadataHelperEngineGstImpl::Reset()
301 {
302 std::unique_lock<std::mutex> lock(mutex_);
303
304 if (metaCollector_ != nullptr) {
305 metaCollector_->Stop();
306 hasCollectMeta_ = false;
307 }
308
309 if (frameExtractor_ != nullptr) {
310 frameExtractor_->Reset();
311 }
312
313 if (playBinCtrler_ != nullptr) {
314 playBinCtrler_->SetElemSetupListener(nullptr);
315
316 auto tmp = playBinCtrler_;
317 playBinCtrler_ = nullptr;
318 // Some msg maybe be reported by the playbinCtrler_ during the playbinCtler_ destroying.
319 // unlock to avoid the deadlock.
320 lock.unlock();
321 tmp = nullptr;
322 lock.lock();
323 }
324
325 sinkProvider_ = nullptr;
326 metaCollector_ = nullptr;
327 frameExtractor_ = nullptr;
328
329 errHappened_ = false;
330 status_ = PLAYBIN_STATE_IDLE;
331
332 firstFetch_ = true;
333
334 lock.unlock();
335 lock.lock();
336 }
337
OnNotifyMessage(const PlayBinMessage & msg)338 void AVMetadataHelperEngineGstImpl::OnNotifyMessage(const PlayBinMessage &msg)
339 {
340 switch (msg.type) {
341 case PLAYBIN_MSG_STATE_CHANGE: {
342 std::unique_lock<std::mutex> lock(mutex_);
343 status_ = msg.code;
344 cond_.notify_all();
345 if (msg.code == PLAYBIN_STATE_PREPARED) {
346 MEDIA_LOGI("prepare finished");
347 }
348 if (msg.code == PLAYBIN_STATE_STOPPED) {
349 MEDIA_LOGI("stop finished");
350 }
351 break;
352 }
353 case PLAYBIN_MSG_ERROR: {
354 std::unique_lock<std::mutex> lock(mutex_);
355 errHappened_ = true;
356 if (metaCollector_ != nullptr) {
357 metaCollector_->Stop();
358 }
359 if (frameExtractor_ != nullptr) {
360 frameExtractor_->Reset();
361 }
362 cond_.notify_all();
363 MEDIA_LOGE("error happened, cancel inprocessing job");
364 break;
365 }
366 case PLAYBIN_MSG_SEEKDONE: {
367 std::unique_lock<std::mutex> lock(mutex_);
368 if (frameExtractor_ != nullptr) {
369 frameExtractor_->NotifyPlayBinMsg(msg);
370 }
371 break;
372 }
373 default:
374 break;
375 }
376 }
377
OnNotifyElemSetup(GstElement & elem)378 void AVMetadataHelperEngineGstImpl::OnNotifyElemSetup(GstElement &elem)
379 {
380 std::unique_lock<std::mutex> lock(mutex_);
381 if (metaCollector_ != nullptr) {
382 metaCollector_->AddMetaSource(elem);
383 }
384 }
385 } // namespace Media
386 } // namespace OHOS