1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmeta_elem_meta_collector.h"
17 #include <string_view>
18 #include <limits>
19 #include "avmetadatahelper.h"
20 #include "avsharedmemorybase.h"
21 #include "av_common.h"
22 #include "gst_meta_parser.h"
23 #include "gst_utils.h"
24 #include "media_errors.h"
25 #include "media_log.h"
26 #include "securec.h"
27
28 namespace {
29 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaElemCollector"};
30 }
31
32 namespace OHOS {
33 namespace Media {
34 #define AVMETA_KEY_TO_X_MAP_ITEM(key, innerKey) { key, innerKey }
35
36 static const std::unordered_map<int32_t, std::string_view> AVMETA_KEY_TO_X_MAP = {
37 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_ALBUM, INNER_META_KEY_ALBUM),
38 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_ALBUM_ARTIST, INNER_META_KEY_ALBUM_ARTIST),
39 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_ARTIST, INNER_META_KEY_ARTIST),
40 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_AUTHOR, INNER_META_KEY_AUTHOR),
41 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_COMPOSER, INNER_META_KEY_COMPOSER),
42 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_DATE_TIME, INNER_META_KEY_DATE_TIME),
43 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_DATE_TIME_FORMAT, ""),
44 /**
45 * The most of gst plugins don't send the GST_TAG_DURATION, we obtain this
46 * information from duration query.
47 */
48 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_DURATION, ""),
49 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_GENRE, INNER_META_KEY_GENRE),
50 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_HAS_AUDIO, ""),
51 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_HAS_VIDEO, ""),
52 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_MIME_TYPE, INNER_META_KEY_MIME_TYPE),
53 /**
54 * The GST_TAG_TRACK_COUNT does not means the actual track count, we obtain
55 * this information from pads count.
56 */
57 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_NUM_TRACKS, ""),
58 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_SAMPLE_RATE, INNER_META_KEY_SAMPLE_RATE),
59 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_TITLE, INNER_META_KEY_TITLE),
60 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_VIDEO_HEIGHT, INNER_META_KEY_VIDEO_HEIGHT),
61 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_VIDEO_WIDTH, INNER_META_KEY_VIDEO_WIDTH),
62 AVMETA_KEY_TO_X_MAP_ITEM(AV_KEY_VIDEO_ORIENTATION, INNER_META_KEY_VIDEO_ORIENTATION),
63 };
64
FormatDataTime(std::string & dataTime)65 std::string FormatDataTime(std::string &dataTime)
66 {
67 std::string::size_type position = dataTime.find(" ");
68 std::string data = "";
69 std::string time = "";
70 if (position == dataTime.npos) {
71 data = dataTime;
72 if (data.find("-") == data.npos) {
73 data += "-01-01";
74 } else if (data.find_first_of("-") == data.find_last_of("-")) {
75 data += "-01";
76 }
77 time += " 00:00:00";
78 } else {
79 data = dataTime.substr(0, position);
80 time = dataTime.substr(position);
81 if (data.find("-") == data.npos) {
82 data += "-01-01";
83 } else if (data.find_first_of("-") == data.find_last_of("-")) {
84 data += "-01";
85 }
86 if (time.find(":") == data.npos) {
87 time += ":00:00";
88 } else if (time.find_first_of(":") == time.find_last_of(":")) {
89 time += ":00";
90 } else {
91 time = time.substr(0, time.find("."));
92 }
93 }
94 MEDIA_LOGD("AV_KEY_DATE_TIME_FORMAT is: %{public}s%{public}s", data.c_str(), time.c_str());
95 return data + time;
96 }
97
PopulateMeta(Metadata & meta)98 void PopulateMeta(Metadata &meta)
99 {
100 for (const auto &item : AVMETA_KEY_TO_X_MAP) {
101 if (!meta.HasMeta(item.first)) {
102 if (item.first == AV_KEY_DATE_TIME_FORMAT && meta.HasMeta(AV_KEY_DATE_TIME)) {
103 std::string dataTime = meta.GetMeta(AV_KEY_DATE_TIME);
104 meta.SetMeta(item.first, FormatDataTime(dataTime));
105 } else {
106 meta.SetMeta(item.first, "");
107 }
108 }
109 }
110 }
111
112 struct AVMetaElemMetaCollector::TrackInfo {
113 bool valid = true;
114 Metadata uploadMeta;
115 Format innerMeta;
116 };
117
Create(AVMetaSourceType type,const MetaResCb & resCb)118 std::unique_ptr<AVMetaElemMetaCollector> AVMetaElemMetaCollector::Create(AVMetaSourceType type, const MetaResCb &resCb)
119 {
120 switch (type) {
121 case AVMetaSourceType::TYPEFIND:
122 return std::make_unique<TypeFindMetaCollector>(type, resCb);
123 case AVMetaSourceType::DEMUXER:
124 return std::make_unique<DemuxerMetaCollector>(type, resCb);
125 case AVMetaSourceType::PARSER:
126 return std::make_unique<ParserMetaCollector>(type, resCb);
127 default:
128 MEDIA_LOGE("unknown type: %{public}hhu", type);
129 break;
130 }
131
132 return nullptr;
133 }
134
AVMetaElemMetaCollector(AVMetaSourceType type,const MetaResCb & resCb)135 AVMetaElemMetaCollector::AVMetaElemMetaCollector(AVMetaSourceType type, const MetaResCb &resCb)
136 : type_(type), resCb_(resCb)
137 {
138 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR ", type: %{public}hhu", FAKE_POINTER(this), type_);
139 }
140
~AVMetaElemMetaCollector()141 AVMetaElemMetaCollector::~AVMetaElemMetaCollector()
142 {
143 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
144 }
145
Stop()146 void AVMetaElemMetaCollector::Stop()
147 {
148 std::unique_lock<std::mutex> lock(mutex_);
149 stopCollecting_ = true;
150
151 for (auto &[elem, signalId] : signalIds_) {
152 g_signal_handler_disconnect(elem, signalId);
153 }
154 signalIds_.clear();
155
156 for (auto &[pad, probeId] : padProbes_) {
157 gst_pad_remove_probe(pad, probeId);
158 }
159 padProbes_.clear();
160
161 lock.unlock();
162 lock.lock();
163 }
164
IsMetaCollected()165 bool AVMetaElemMetaCollector::IsMetaCollected()
166 {
167 std::unique_lock<std::mutex> lock(mutex_);
168 for (auto &[dummy, trackInfo] : trackInfos_) {
169 if (!trackInfo.valid) {
170 continue;
171 }
172 if (trackInfo.innerMeta.GetFormatMap().empty()) {
173 return false;
174 }
175 }
176
177 // at least the duration meta and track count or container mime.
178 if (fileUploadMeta_.tbl_.empty()) {
179 return false;
180 }
181
182 return true;
183 }
184
FetchArtPicture()185 std::shared_ptr<AVSharedMemory> AVMetaElemMetaCollector::FetchArtPicture()
186 {
187 std::unique_lock<std::mutex> lock(mutex_);
188
189 auto artPicMem = DoFetchArtPicture(fileInnerMeta_);
190 if (artPicMem != nullptr) {
191 return artPicMem;
192 }
193
194 for (auto &[dummy, trackInnerMeta] : trackInfos_) {
195 artPicMem = DoFetchArtPicture(trackInnerMeta.innerMeta);
196 if (artPicMem != nullptr) {
197 return artPicMem;
198 }
199 }
200
201 return artPicMem;
202 }
203
DoFetchArtPicture(const Format & innerMeta)204 std::shared_ptr<AVSharedMemory> AVMetaElemMetaCollector::DoFetchArtPicture(const Format &innerMeta)
205 {
206 if (!innerMeta.ContainKey(INNER_META_KEY_IMAGE)) {
207 return nullptr;
208 }
209
210 MEDIA_LOGD("has art picture");
211
212 uint8_t *addr = nullptr;
213 size_t size = 0;
214 (void)innerMeta.GetBuffer(INNER_META_KEY_IMAGE, &addr, size);
215
216 static constexpr size_t maxImageSize = 1 * 1024 * 1024;
217 if (addr == nullptr || size == 0 || size > maxImageSize) {
218 MEDIA_LOGW("invalid param, size = %{public}zu", size);
219 return nullptr;
220 }
221
222 auto artPicMem = AVSharedMemoryBase::CreateFromLocal(
223 static_cast<int32_t>(size), AVSharedMemory::FLAGS_READ_ONLY, "artpic");
224 CHECK_AND_RETURN_RET_LOG(artPicMem != nullptr, nullptr, "create art pic failed");
225
226 errno_t rc = memcpy_s(artPicMem->GetBase(), static_cast<size_t>(artPicMem->GetSize()), addr, size);
227 CHECK_AND_RETURN_RET_LOG(rc == EOK, nullptr, "memcpy_s failed");
228
229 return artPicMem;
230 }
231
AddProbeToPadList(GList & list)232 bool AVMetaElemMetaCollector::AddProbeToPadList(GList &list)
233 {
234 for (GList *padNode = g_list_first(&list); padNode != nullptr; padNode = padNode->next) {
235 CHECK_AND_CONTINUE(padNode->data != nullptr);
236
237 GstPad *pad = reinterpret_cast<GstPad *>(padNode->data);
238 CHECK_AND_RETURN_RET(AddProbeToPad(*pad), false);
239 }
240
241 return true;
242 }
243
AddProbeToPad(GstPad & pad)244 bool AVMetaElemMetaCollector::AddProbeToPad(GstPad &pad)
245 {
246 std::unique_lock<std::mutex> lock(mutex_);
247 if (stopCollecting_) {
248 MEDIA_LOGI("stop collecting...");
249 return false;
250 }
251
252 gulong probeId = gst_pad_add_probe(&pad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, ProbeCallback, this, nullptr);
253 CHECK_AND_RETURN_RET_LOG(probeId != 0, false, "add probe for %{public}s's pad %{public}s failed",
254 PAD_PARENT_NAME(&pad), PAD_NAME(&pad))
255
256 (void)padProbes_.emplace(&pad, probeId);
257 (void)trackInfos_.emplace(&pad, TrackInfo {});
258
259 // report the track count change when caps arrived.
260 trackcount_ += 1;
261 MEDIA_LOGD("add probe to pad %{public}s of %{public}s", PAD_NAME(&pad), PAD_PARENT_NAME(&pad));
262 return true;
263 }
264
ConnectSignal(GstElement & elem,std::string_view signal,GCallback callback)265 bool AVMetaElemMetaCollector::ConnectSignal(GstElement &elem, std::string_view signal, GCallback callback)
266 {
267 std::unique_lock<std::mutex> lock(mutex_);
268 if (stopCollecting_) {
269 MEDIA_LOGI("stop collecting...");
270 return false;
271 }
272
273 gulong signalId = g_signal_connect(&elem, signal.data(), callback, this);
274 CHECK_AND_RETURN_RET_LOG(signalId != 0, false, "connect signal '%{public}s' to %{public}s failed",
275 signal.data(), ELEM_NAME(&elem))
276
277 (void)signalIds_.emplace(&elem, signalId);
278 return true;
279 }
280
ProbeCallback(GstPad * pad,GstPadProbeInfo * info,gpointer usrdata)281 GstPadProbeReturn AVMetaElemMetaCollector::ProbeCallback(GstPad *pad, GstPadProbeInfo *info, gpointer usrdata)
282 {
283 CHECK_AND_RETURN_RET_LOG(pad != nullptr && info != nullptr && usrdata != nullptr,
284 GST_PAD_PROBE_OK, "param is invalid")
285
286 auto collector = reinterpret_cast<AVMetaElemMetaCollector *>(usrdata);
287 if (static_cast<unsigned int>(info->type) & GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) {
288 GstEvent *event = gst_pad_probe_info_get_event(info);
289 CHECK_AND_RETURN_RET_LOG(event != nullptr, GST_PAD_PROBE_OK, "event is null");
290 collector->OnEventProbe(*pad, *event);
291 }
292
293 return GST_PAD_PROBE_OK;
294 }
295
ParseTagList(const GstTagList & tagList,TrackInfo & trackInfo)296 void AVMetaElemMetaCollector::ParseTagList(const GstTagList &tagList, TrackInfo &trackInfo)
297 {
298 if (!trackInfo.valid) {
299 return;
300 }
301
302 GstTagScope scope = gst_tag_list_get_scope(&tagList);
303 MEDIA_LOGI("catch tag %{public}s event", scope == GST_TAG_SCOPE_GLOBAL ? "global" : "stream");
304
305 if (scope == GST_TAG_SCOPE_GLOBAL) {
306 if (globalTagCatched_) {
307 return;
308 }
309 globalTagCatched_ = true;
310 GstMetaParser::ParseTagList(tagList, fileInnerMeta_);
311 ConvertToAVMeta(fileInnerMeta_, fileUploadMeta_);
312 ReportMeta(fileUploadMeta_);
313 } else {
314 GstMetaParser::ParseTagList(tagList, trackInfo.innerMeta);
315 ConvertToAVMeta(trackInfo.innerMeta, trackInfo.uploadMeta);
316 ReportMeta(trackInfo.uploadMeta);
317 }
318 }
319
ParseCaps(const GstCaps & caps,TrackInfo & trackInfo)320 void AVMetaElemMetaCollector::ParseCaps(const GstCaps &caps, TrackInfo &trackInfo)
321 {
322 GstMetaParser::ParseStreamCaps(caps, trackInfo.innerMeta);
323
324 if (!EnsureTrackValid(trackInfo)) {
325 return;
326 }
327
328 fileUploadMeta_.SetMeta(AV_KEY_NUM_TRACKS, std::to_string(trackcount_));
329 ReportMeta(fileUploadMeta_);
330
331 ConvertToAVMeta(trackInfo.innerMeta, trackInfo.uploadMeta);
332 ReportMeta(trackInfo.uploadMeta);
333 }
334
EnsureTrackValid(TrackInfo & trackInfo)335 bool AVMetaElemMetaCollector::EnsureTrackValid(TrackInfo &trackInfo)
336 {
337 /**
338 * If the track can not supported, it would not be taken account into the
339 * total track counts. The ffmpeg will generate one track for one image of
340 * the metadata, the track's caps is image/png or image/jpeg, etc. For such
341 * tracks, them should be considered as invalid tracks.
342 */
343 int32_t trackType;
344 std::string mimeType;
345 if (!trackInfo.innerMeta.GetIntValue(INNER_META_KEY_TRACK_TYPE, trackType) ||
346 !trackInfo.innerMeta.GetStringValue(INNER_META_KEY_MIME_TYPE, mimeType)) {
347 trackInfo.valid = false;
348 trackcount_ -= 1;
349 fileUploadMeta_.SetMeta(AV_KEY_NUM_TRACKS, std::to_string(trackcount_));
350 ReportMeta(fileUploadMeta_);
351 return false;
352 }
353
354 if (trackType == MediaType::MEDIA_TYPE_VID) {
355 trackInfo.uploadMeta.SetMeta(AV_KEY_HAS_VIDEO, "yes");
356 } else if (trackType == MediaType::MEDIA_TYPE_AUD) {
357 trackInfo.uploadMeta.SetMeta(AV_KEY_HAS_AUDIO, "yes");
358 }
359
360 return true;
361 }
362
OnEventProbe(GstPad & pad,GstEvent & event)363 void AVMetaElemMetaCollector::OnEventProbe(GstPad &pad, GstEvent &event)
364 {
365 std::unique_lock<std::mutex> lock(mutex_);
366 if (stopCollecting_) {
367 MEDIA_LOGI("stop collecting...");
368 return;
369 }
370
371 auto it = trackInfos_.find(&pad);
372 CHECK_AND_RETURN_LOG(it != trackInfos_.end(), "unrecognized pad %{public}s", PAD_NAME(&pad));
373
374 if (GST_EVENT_TYPE(&event) == GST_EVENT_TAG) {
375 QueryDuration(pad);
376
377 GstTagList *tagList = nullptr;
378 gst_event_parse_tag(&event, &tagList);
379 CHECK_AND_RETURN_LOG(tagList != nullptr, "taglist is nullptr");
380 MEDIA_LOGI("catch tags at pad %{public}s", PAD_NAME(&pad));
381 ParseTagList(*tagList, it->second);
382 }
383
384 if (GST_EVENT_TYPE(&event) == GST_EVENT_CAPS) {
385 GstCaps *caps = nullptr;
386 gst_event_parse_caps(&event, &caps);
387 CHECK_AND_RETURN_LOG(caps != nullptr, "caps is nullptr");
388 MEDIA_LOGI("catch caps at pad %{public}s", PAD_NAME(&pad));
389 ParseCaps(*caps, it->second);
390 }
391 }
392
QueryDuration(GstPad & pad)393 void AVMetaElemMetaCollector::QueryDuration(GstPad &pad)
394 {
395 GstQuery *query = gst_query_new_duration(GST_FORMAT_TIME);
396 CHECK_AND_RETURN_LOG(query != nullptr, "query is failed");
397
398 gint64 streamDuration = 0;
399 if (gst_pad_query(&pad, query)) {
400 GstFormat format = GST_FORMAT_TIME;
401 gst_query_parse_duration(query, &format, &streamDuration);
402 if (!GST_CLOCK_TIME_IS_VALID(streamDuration)) {
403 streamDuration = 0;
404 }
405 }
406 gst_query_unref(query);
407
408 if (duration_ < streamDuration) {
409 duration_ = streamDuration;
410 MEDIA_LOGI("update duration to %{public}" PRIi64 "", duration_);
411
412 static constexpr int32_t NASEC_PER_MILLISEC = 1000000;
413 int64_t milliSecond = duration_ / NASEC_PER_MILLISEC; // ns -> ms
414 fileUploadMeta_.SetMeta(AV_KEY_DURATION, std::to_string(milliSecond));
415 ReportMeta(fileUploadMeta_);
416 }
417 }
418
ReportMeta(const Metadata & uploadMeta)419 void AVMetaElemMetaCollector::ReportMeta(const Metadata &uploadMeta)
420 {
421 if (resCb_ == nullptr) {
422 return;
423 }
424
425 mutex_.unlock();
426 resCb_(uploadMeta);
427 mutex_.lock();
428 }
429
ConvertToAVMeta(const Format & innerMeta,Metadata & avmeta) const430 void AVMetaElemMetaCollector::ConvertToAVMeta(const Format &innerMeta, Metadata &avmeta) const
431 {
432 for (const auto &[avKey, innerKey] : AVMETA_KEY_TO_X_MAP) {
433 if (innerKey.compare("") == 0) {
434 continue;
435 }
436
437 if (innerKey.compare(INNER_META_KEY_MIME_TYPE) == 0) { // only need the file mime type
438 continue;
439 }
440
441 if (!innerMeta.ContainKey(innerKey)) {
442 continue;
443 }
444
445 std::string strVal;
446 int32_t intVal;
447 FormatDataType type = innerMeta.GetValueType(innerKey);
448 switch (type) {
449 case FORMAT_TYPE_STRING:
450 innerMeta.GetStringValue(innerKey, strVal);
451 avmeta.SetMeta(avKey, strVal);
452 break;
453 case FORMAT_TYPE_INT32:
454 innerMeta.GetIntValue(innerKey, intVal);
455 avmeta.SetMeta(avKey, std::to_string(intVal));
456 break;
457 default:
458 break;
459 }
460 }
461 }
462
463 /**
464 * Detail Element Meta Collector Implementation Begin.
465 */
466
HaveTypeCallback(GstElement * elem,guint probability,GstCaps * caps,gpointer userData)467 void TypeFindMetaCollector::HaveTypeCallback(GstElement *elem, guint probability, GstCaps *caps, gpointer userData)
468 {
469 CHECK_AND_RETURN(elem != nullptr && caps != nullptr && userData != nullptr);
470 MEDIA_LOGD("typefind %{public}s have type, probalibity = %{public}u", ELEM_NAME(elem), probability);
471
472 TypeFindMetaCollector *collector = reinterpret_cast<TypeFindMetaCollector *>(userData);
473 collector->OnHaveType(*elem, *caps);
474 }
475
OnHaveType(const GstElement & elem,const GstCaps & caps)476 void TypeFindMetaCollector::OnHaveType(const GstElement &elem, const GstCaps &caps)
477 {
478 (void)elem;
479 std::unique_lock<std::mutex> lock(mutex_);
480 if (stopCollecting_) {
481 MEDIA_LOGI("stop collecting...");
482 return;
483 }
484
485 GstMetaParser::ParseFileMimeType(caps, fileInnerMeta_);
486
487 std::string mimeType;
488 (void)fileInnerMeta_.GetStringValue(INNER_META_KEY_MIME_TYPE, mimeType);
489 fileUploadMeta_.SetMeta(AV_KEY_MIME_TYPE, mimeType);
490
491 ReportMeta(fileUploadMeta_);
492 }
493
AddMetaSource(GstElement & elem)494 void TypeFindMetaCollector::AddMetaSource(GstElement &elem)
495 {
496 (void)ConnectSignal(elem, "have-type", G_CALLBACK(&TypeFindMetaCollector::HaveTypeCallback));
497 }
498
PadAddedCallback(GstElement * elem,GstPad * pad,gpointer userData)499 void DemuxerMetaCollector::PadAddedCallback(GstElement *elem, GstPad *pad, gpointer userData)
500 {
501 CHECK_AND_RETURN(elem != nullptr && pad != nullptr && userData != nullptr);
502
503 auto collector = reinterpret_cast<DemuxerMetaCollector *>(userData);
504 collector->OnPadAdded(*elem, *pad);
505 }
506
OnPadAdded(GstElement & src,GstPad & pad)507 void DemuxerMetaCollector::OnPadAdded(GstElement &src, GstPad &pad)
508 {
509 MEDIA_LOGD("demuxer %{public}s sinkpad %{public}s added", ELEM_NAME(&src), PAD_NAME(&pad));
510 (void)AddProbeToPad(pad);
511 }
512
AddMetaSource(GstElement & elem)513 void DemuxerMetaCollector::AddMetaSource(GstElement &elem)
514 {
515 CHECK_AND_RETURN(AddProbeToPadList(*elem.srcpads));
516
517 (void)ConnectSignal(elem, "pad-added", G_CALLBACK(&DemuxerMetaCollector::PadAddedCallback));
518 }
519
AddMetaSource(GstElement & elem)520 void ParserMetaCollector::AddMetaSource(GstElement &elem)
521 {
522 (void)AddProbeToPadList(*elem.srcpads);
523 }
524 } // namespace Media
525 } // namespace OHOS