1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "gst_meta_parser.h"
17 #include <functional>
18 #include <vector>
19 #include <sstream>
20 #include <iomanip>
21 #include "av_common.h"
22 #include "media_errors.h"
23 #include "media_log.h"
24 #include "gst/tag/tag.h"
25 #include "gst_utils.h"
26
27 namespace {
28 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "GstMetaParser"};
29 static GType GST_SAMPLE_TYPE = gst_sample_get_type();
30 static GType GST_DATE_TIME_TYPE = gst_date_time_get_type();
31 static GType GST_FRACTION_TYPE = gst_fraction_get_type();
32 constexpr size_t FORMATTED_TIME_NUM_SIZE = 2;
33 }
34
35 namespace OHOS {
36 namespace Media {
37 using MetaSetter = std::function<bool(const GValue &gval, const std::string_view &key, Format &metadata)>;
38
39 struct MetaParseItem {
40 std::string_view toKey;
41 GType valGType;
42 MetaSetter setter;
43 };
44
45 static bool ParseGValueSimple(const GValue &value, const MetaParseItem &item, Format &metadata);
46 static bool FractionMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
47 static bool ImageMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
48 static bool DateTimeMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
49 static bool OrientationMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
50
51 static const std::unordered_map<std::string_view, MetaParseItem> GST_TAG_PARSE_ITEMS = {
52 { GST_TAG_ALBUM, { INNER_META_KEY_ALBUM, G_TYPE_STRING, nullptr } },
53 { GST_TAG_ALBUM_ARTIST, { INNER_META_KEY_ALBUM_ARTIST, G_TYPE_STRING, nullptr } },
54 { GST_TAG_ARTIST, { INNER_META_KEY_ARTIST, G_TYPE_STRING, nullptr } },
55 { GST_TAG_COMPOSER, { INNER_META_KEY_COMPOSER, G_TYPE_STRING, nullptr } },
56 { GST_TAG_DATE_TIME, { INNER_META_KEY_DATE_TIME, GST_DATE_TIME_TYPE, DateTimeMetaSetter } },
57 { GST_TAG_GENRE, { INNER_META_KEY_GENRE, G_TYPE_STRING, nullptr } },
58 { GST_TAG_TITLE, { INNER_META_KEY_TITLE, G_TYPE_STRING, nullptr } },
59 { GST_TAG_AUTHOR, { INNER_META_KEY_AUTHOR, G_TYPE_STRING, nullptr } },
60 { GST_TAG_DURATION, { INNER_META_KEY_DURATION, G_TYPE_UINT64, nullptr } },
61 { GST_TAG_BITRATE, { INNER_META_KEY_BITRATE, G_TYPE_UINT, nullptr } },
62 { GST_TAG_IMAGE, { INNER_META_KEY_IMAGE, GST_SAMPLE_TYPE, ImageMetaSetter } },
63 { GST_TAG_LANGUAGE_CODE, { INNER_META_KEY_LANGUAGE, G_TYPE_STRING, nullptr } },
64 { GST_TAG_IMAGE_ORIENTATION, { INNER_META_KEY_VIDEO_ORIENTATION, G_TYPE_STRING, OrientationMetaSetter } },
65 { GST_TAG_BANDWIDTH, {INNER_META_KEY_BANDWIDTH, G_TYPE_UINT, nullptr } },
66 };
67
68 static const std::unordered_map<std::string_view, MetaParseItem> GST_CAPS_PARSE_ITEMS = {
69 { "width", { INNER_META_KEY_VIDEO_WIDTH, G_TYPE_INT, nullptr } },
70 { "height", { INNER_META_KEY_VIDEO_HEIGHT, G_TYPE_INT, nullptr } },
71 { "rate", { INNER_META_KEY_SAMPLE_RATE, G_TYPE_INT, nullptr } },
72 { "framerate", { INNER_META_KEY_FRAMERATE, GST_FRACTION_TYPE, FractionMetaSetter } },
73 { "channels", { INNER_META_KEY_CHANNEL_COUNT, G_TYPE_INT, nullptr } },
74 };
75
76 static const std::unordered_map<std::string_view, std::vector<std::string_view>> STREAM_CAPS_FIELDS = {
77 { "video", { "width", "height", "framerate", "format", "bandwidth" } },
78 { "audio", { "rate", "channels" } },
79 { "text", { "format" } },
80 };
81
82 static const std::unordered_map<std::string_view, std::string_view> FILE_MIME_TYPE_MAPPING = {
83 { "application/x-3gp", FILE_MIMETYPE_VIDEO_MP4 }, // mp4, m4a, mov, 3g2, 3gp
84 { "video/quicktime", FILE_MIMETYPE_VIDEO_MP4 },
85 { "video/quicktime, variant=(string)apple", FILE_MIMETYPE_VIDEO_MP4 },
86 { "video/quicktime, variant=(string)iso", FILE_MIMETYPE_VIDEO_MP4 },
87 { "video/quicktime, variant=(string)3gpp", FILE_MIMETYPE_VIDEO_MP4 },
88 { "video/quicktime, variant=(string)3g2", FILE_MIMETYPE_VIDEO_MP4 },
89 { "video/x-matroska", FILE_MIMETYPE_VIDEO_MKV}, // mkv
90 { "audio/x-matroska", FILE_MIMETYPE_AUDIO_MKV}, // mkv
91 { "video/mpegts", FILE_MIMETYPE_VIDEO_MPEGTS}, // mpeg-ts
92 { "video/webm", FILE_MIMETYPE_VIDEO_WEBM}, // webm
93 { "audio/webm", FILE_MIMETYPE_AUDIO_WEBM}, // webm
94 { "audio/x-m4a", FILE_MIMETYPE_AUDIO_MP4 },
95 { "audio/mpeg", FILE_MIMETYPE_AUDIO_AAC }, // aac
96 { "application/x-id3", FILE_MIMETYPE_AUDIO_MP3 }, // mp3
97 { "application/ogg", FILE_MIMETYPE_AUDIO_OGG }, // ogg
98 { "audio/ogg", FILE_MIMETYPE_AUDIO_OGG },
99 { "audio/x-flac", FILE_MIMETYPE_AUDIO_FLAC }, // flac
100 { "audio/x-wav", FILE_MIMETYPE_AUDIO_WAV } // wav
101 };
102
103 enum AUDIO_MPEG_TYPE {
104 AUDIO_MPEG_MP3,
105 AUDIO_MPEG_AAC
106 };
107
108 static const std::unordered_map<std::string_view, std::vector<std::string_view>> CODEC_MIME_TYPE_MAPPING = {
109 { "video/x-h264", {VIDEO_MIMETYPE_AVC} },
110 { "video/mpeg", {VIDEO_MIMETYPE_MPEG4} },
111 { "audio/mpeg", {AUDIO_MIMETYPE_MPEG, AUDIO_MIMETYPE_AAC} },
112 { "text/x-raw", {SUBTITLE_MIMETYPE_SRT} }
113 };
114
ParseGValue(const GValue & value,const MetaParseItem & item,Format & metadata)115 static void ParseGValue(const GValue &value, const MetaParseItem &item, Format &metadata)
116 {
117 if (G_VALUE_TYPE(&value) != item.valGType) {
118 MEDIA_LOGE("value type for key %{public}s is expected, curr is %{public}s, but expect %{public}s",
119 item.toKey.data(), g_type_name(G_VALUE_TYPE(&value)), g_type_name(item.valGType));
120 return;
121 }
122
123 bool ret = true;
124 if (item.setter != nullptr) {
125 ret = item.setter(value, item.toKey, metadata);
126 } else {
127 ret = ParseGValueSimple(value, item, metadata);
128 }
129
130 if (!ret) {
131 MEDIA_LOGE("parse gvalue failed for type: %{public}s, toKey: %{public}s",
132 g_type_name(item.valGType), item.toKey.data());
133 }
134 }
135
ParseTag(const GstTagList & tagList,guint tagIndex,Format & metadata)136 static void ParseTag(const GstTagList &tagList, guint tagIndex, Format &metadata)
137 {
138 const gchar *tag = gst_tag_list_nth_tag_name(&tagList, tagIndex);
139 if (tag == nullptr) {
140 return;
141 }
142
143 MEDIA_LOGD("visit tag: %{public}s", tag);
144 auto tagItemIt = GST_TAG_PARSE_ITEMS.find(tag);
145 if (tagItemIt == GST_TAG_PARSE_ITEMS.end()) {
146 return;
147 }
148
149 guint tagValSize = gst_tag_list_get_tag_size(&tagList, tag);
150 for (guint i = 0; i < tagValSize; i++) {
151 const GValue *value = gst_tag_list_get_value_index(&tagList, tag, i);
152 if (value == nullptr) {
153 MEDIA_LOGW("get value index %{public}d from tag %{public}s failed", i, tag);
154 continue;
155 }
156
157 ParseGValue(*value, tagItemIt->second, metadata);
158 }
159 }
160
ParseTagList(const GstTagList & tagList,Format & metadata)161 void GstMetaParser::ParseTagList(const GstTagList &tagList, Format &metadata)
162 {
163 gint tagCnt = gst_tag_list_n_tags(&tagList);
164 if (tagCnt <= 0) {
165 return;
166 }
167
168 for (guint tagIndex = 0; tagIndex < static_cast<guint>(tagCnt); tagIndex++) {
169 ParseTag(tagList, tagIndex, metadata);
170 }
171 }
172
ParseSingleCapsStructure(const GstStructure & structure,const std::vector<std::string_view> & target,Format & metadata)173 static void ParseSingleCapsStructure(const GstStructure &structure,
174 const std::vector<std::string_view> &target,
175 Format &metadata)
176 {
177 for (auto &field : target) {
178 if (!gst_structure_has_field(&structure, field.data())) {
179 continue;
180 }
181 const GValue *val = gst_structure_get_value(&structure, field.data());
182 if (val == nullptr) {
183 MEDIA_LOGE("get %{public}s filed's value failed", field.data());
184 continue;
185 }
186
187 auto it = GST_CAPS_PARSE_ITEMS.find(field);
188 if (it != GST_CAPS_PARSE_ITEMS.end()) {
189 ParseGValue(*val, it->second, metadata);
190 }
191 }
192 }
193
ParseStreamCaps(const GstCaps & caps,Format & metadata)194 void GstMetaParser::ParseStreamCaps(const GstCaps &caps, Format &metadata)
195 {
196 guint capsSize = gst_caps_get_size(&caps);
197 for (guint index = 0; index < capsSize; index++) {
198 const GstStructure *struc = gst_caps_get_structure(&caps, index);
199 std::string_view mimeType = STRUCTURE_NAME(struc);
200
201 std::string_view::size_type delimPos = mimeType.find('/');
202 if (delimPos == std::string_view::npos) {
203 MEDIA_LOGE("unrecognizable mimetype, %{public}s", mimeType.data());
204 continue;
205 }
206
207 MEDIA_LOGI("parse mimetype %{public}s's caps", mimeType.data());
208
209 std::string_view streamType = mimeType.substr(0, delimPos);
210 auto it = STREAM_CAPS_FIELDS.find(streamType);
211 if (it == STREAM_CAPS_FIELDS.end()) {
212 continue;
213 }
214
215 if (streamType.compare("video") == 0) {
216 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_VID);
217 } else if (streamType.compare("audio") == 0) {
218 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_AUD);
219 } else if (streamType.compare("text") == 0) {
220 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_SUBTITLE);
221 }
222
223 if (CODEC_MIME_TYPE_MAPPING.find(mimeType) != CODEC_MIME_TYPE_MAPPING.end()) {
224 if (mimeType == "audio/mpeg") {
225 gint mpegversion;
226 gst_structure_get_int(struc, "mpegversion", &mpegversion);
227 int32_t index = mpegversion == 1 ? AUDIO_MPEG_MP3 : AUDIO_MPEG_AAC;
228 mimeType = (CODEC_MIME_TYPE_MAPPING.find(mimeType)->second)[index];
229 } else {
230 mimeType = (CODEC_MIME_TYPE_MAPPING.find(mimeType)->second)[0];
231 }
232 }
233
234 metadata.PutStringValue(INNER_META_KEY_MIME_TYPE, mimeType);
235 ParseSingleCapsStructure(*struc, it->second, metadata);
236 }
237 }
238
ParseFileMimeType(const GstCaps & caps,Format & metadata)239 void GstMetaParser::ParseFileMimeType(const GstCaps &caps, Format &metadata)
240 {
241 const GstStructure *struc = gst_caps_get_structure(&caps, 0); // ignore the caps size
242 CHECK_AND_RETURN_LOG(struc != nullptr, "caps invalid");
243 auto mimeType = STRUCTURE_NAME(struc);
244 CHECK_AND_RETURN_LOG(mimeType != nullptr, "caps invalid");
245
246 const auto &it = FILE_MIME_TYPE_MAPPING.find(mimeType);
247 if (it == FILE_MIME_TYPE_MAPPING.end()) {
248 MEDIA_LOGE("unrecognizable file mimetype: %{public}s", mimeType);
249 return;
250 }
251
252 MEDIA_LOGI("file caps mime type: %{public}s, mapping to %{public}s", mimeType, it->second.data());
253 metadata.PutStringValue(INNER_META_KEY_MIME_TYPE, it->second);
254 }
255
ParseGValueSimple(const GValue & value,const MetaParseItem & item,Format & metadata)256 static bool ParseGValueSimple(const GValue &value, const MetaParseItem &item, Format &metadata)
257 {
258 bool ret = true;
259 switch (item.valGType) {
260 case G_TYPE_UINT: {
261 guint num = g_value_get_uint(&value);
262 ret = metadata.PutIntValue(item.toKey, static_cast<int32_t>(num));
263 MEDIA_LOGD("toKey: %{public}s, value: %{public}u", item.toKey.data(), num);
264 break;
265 }
266 case G_TYPE_INT: {
267 gint num = g_value_get_int(&value);
268 ret = metadata.PutIntValue(item.toKey, num);
269 MEDIA_LOGD("toKey: %{public}s, value: %{public}d", item.toKey.data(), num);
270 break;
271 }
272 case G_TYPE_UINT64: {
273 guint64 num = g_value_get_uint64(&value);
274 ret = metadata.PutLongValue(item.toKey, static_cast<int64_t>(num));
275 MEDIA_LOGD("toKey: %{public}s, value: %{public}" PRIu64, item.toKey.data(), num);
276 break;
277 }
278 case G_TYPE_INT64: {
279 gint64 num = g_value_get_int64(&value);
280 ret = metadata.PutLongValue(item.toKey, num);
281 MEDIA_LOGD("toKey: %{public}s, value: %{public}" PRIi64, item.toKey.data(), num);
282 break;
283 }
284 case G_TYPE_STRING: {
285 std::string_view str = g_value_get_string(&value);
286 CHECK_AND_RETURN_RET_LOG(!str.empty(), false, "Parse key %{public}s failed", item.toKey.data());
287 ret = metadata.PutStringValue(item.toKey, str);
288 MEDIA_LOGD("toKey: %{public}s, value: %{public}s", item.toKey.data(), str.data());
289 break;
290 }
291 default:
292 break;
293 }
294 return ret;
295 }
296
FractionMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)297 static bool FractionMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
298 {
299 gint numerator = gst_value_get_fraction_numerator(&gval);
300 gint denominator = gst_value_get_fraction_denominator(&gval);
301 CHECK_AND_RETURN_RET(denominator != 0, false);
302
303 float val = static_cast<float>(numerator) / denominator;
304 static constexpr int32_t FACTOR = 100;
305 bool ret = metadata.PutIntValue(key, static_cast<int32_t>(val * FACTOR));
306 if (!ret) {
307 MEDIA_LOGE("Parse gvalue for %{public}s failed, num = %{public}d, den = %{public}d",
308 key.data(), numerator, denominator);
309 } else {
310 MEDIA_LOGD("Parse gvalue for %{public}s ok, value = %{public}f", key.data(), val);
311 }
312 return ret;
313 }
314
ImageMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)315 static bool ImageMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
316 {
317 GstSample *sample = gst_value_get_sample(&gval);
318 CHECK_AND_RETURN_RET(sample != nullptr, false);
319
320 GstCaps *caps = gst_sample_get_caps(sample);
321 CHECK_AND_RETURN_RET(caps != nullptr, false);
322
323 GstStructure *structure = gst_caps_get_structure(caps, 0);
324 CHECK_AND_RETURN_RET(structure != nullptr, false);
325
326 const gchar *mime = gst_structure_get_name(structure);
327 CHECK_AND_RETURN_RET(mime != nullptr, false);
328
329 if (g_str_has_prefix(mime, "text")) {
330 MEDIA_LOGI("image is uri, ignored, %{public}s", mime);
331 return true;
332 }
333
334 GstBuffer *imageBuf = gst_sample_get_buffer(sample);
335 CHECK_AND_RETURN_RET(imageBuf != nullptr, false);
336
337 GstMapInfo mapInfo = GST_MAP_INFO_INIT;
338 if (!gst_buffer_map(imageBuf, &mapInfo, GST_MAP_READ)) {
339 MEDIA_LOGE("get buffer data failed");
340 return false;
341 }
342
343 static constexpr gsize minImageSize = 32;
344 CHECK_AND_RETURN_RET(mapInfo.data != nullptr && mapInfo.size > minImageSize, false);
345
346 bool ret = true;
347 if (metadata.ContainKey(key)) {
348 const GstStructure *imageInfo = gst_sample_get_info(sample);
349 gint type = GST_TAG_IMAGE_TYPE_NONE;
350 if (imageInfo != nullptr) {
351 gst_structure_get_enum(imageInfo, "image-type", GST_TYPE_TAG_IMAGE_TYPE, &type);
352 }
353 if (type != GST_TAG_IMAGE_TYPE_FRONT_COVER) {
354 MEDIA_LOGI("ignore the image type: %{public}d", type);
355 } else {
356 MEDIA_LOGI("got front cover image");
357 ret = metadata.PutBuffer(key, mapInfo.data, mapInfo.size);
358 }
359 } else {
360 ret = metadata.PutBuffer(key, mapInfo.data, mapInfo.size);
361 }
362
363 MEDIA_LOGD("Parse gvalue for %{public}s finish, ret = %{public}d, mime: %{public}s, "
364 "size: %{public}" G_GSIZE_FORMAT,
365 key.data(), ret, mime, mapInfo.size);
366
367 gst_buffer_unmap(imageBuf, &mapInfo);
368 return ret;
369 }
370
DateTimeMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)371 static bool DateTimeMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
372 {
373 GstDateTime *dateTime = static_cast<GstDateTime *>(g_value_dup_boxed(&gval));
374 CHECK_AND_RETURN_RET(dateTime != nullptr, false);
375
376 std::stringstream time;
377 if (gst_date_time_has_year(dateTime)) {
378 std::string year = std::to_string(gst_date_time_get_year(dateTime));
379 time << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << year;
380 }
381
382 if (gst_date_time_has_month(dateTime)) {
383 std::string month = std::to_string(gst_date_time_get_month(dateTime));
384 time << "-" << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << month;
385 }
386
387 if (gst_date_time_has_day(dateTime)) {
388 std::string day = std::to_string(gst_date_time_get_day(dateTime));
389 time << "-" << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << day;
390 }
391
392 if (gst_date_time_has_time(dateTime)) {
393 std::string hour = std::to_string(gst_date_time_get_hour(dateTime));
394 std::string minute = std::to_string(gst_date_time_get_minute(dateTime));
395 time << " " << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << hour << ":" <<
396 std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << minute;
397 }
398
399 if (gst_date_time_has_second(dateTime)) {
400 std::string second = std::to_string(gst_date_time_get_second(dateTime));
401 time << ":" << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << second;
402 }
403 std::string str = time.str();
404
405 gst_date_time_unref(dateTime);
406 dateTime = nullptr;
407
408 bool ret = metadata.PutStringValue(key, str);
409 MEDIA_LOGD("Key: %{public}s, value: %{public}s", key.data(), str.data());
410
411 return ret;
412 }
413
OrientationMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)414 static bool OrientationMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
415 {
416 std::string str = g_value_get_string(&gval);
417 CHECK_AND_RETURN_RET_LOG(!str.empty(), false, "Parse key %{public}s failed", key.data());
418 bool ret = false;
419
420 std::string::size_type pos = std::string::npos;
421 if ((pos = str.find("-")) == std::string::npos) {
422 ret = false;
423 } else {
424 std::string subStr = str.substr(pos + 1, str.length() - pos);
425 int32_t rotate = std::stol(subStr, nullptr, 10); // 10 : decimalism
426 MEDIA_LOGI("Get rotate str is %{public}s", subStr.c_str());
427 MEDIA_LOGI("Get rotate is %{public}d", rotate);
428 ret = metadata.PutIntValue(key, rotate);
429 }
430
431 return ret;
432 }
433 } // namespace Media
434 } // namespace OHOS
435