1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "gst_meta_parser.h"
17 #include <functional>
18 #include <vector>
19 #include <sstream>
20 #include <iomanip>
21 #include "av_common.h"
22 #include "media_errors.h"
23 #include "media_log.h"
24 #include "gst/tag/tag.h"
25 #include "gst_utils.h"
26
27 namespace {
28 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "GstMetaParser"};
29 static GType GST_SAMPLE_TYPE = gst_sample_get_type();
30 static GType GST_DATE_TIME_TYPE = gst_date_time_get_type();
31 static GType GST_FRACTION_TYPE = gst_fraction_get_type();
32 constexpr size_t FORMATTED_TIME_NUM_SIZE = 2;
33 }
34
35 namespace OHOS {
36 namespace Media {
37 using MetaSetter = std::function<bool(const GValue &gval, const std::string_view &key, Format &metadata)>;
38
39 struct MetaParseItem {
40 std::string_view toKey;
41 GType valGType;
42 MetaSetter setter;
43 };
44
45 static bool ParseGValueSimple(const GValue &value, const MetaParseItem &item, Format &metadata);
46 static bool FractionMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
47 static bool ImageMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
48 static bool DateTimeMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
49 static bool OrientationMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
50
51 static const std::unordered_map<std::string_view, MetaParseItem> GST_TAG_PARSE_ITEMS = {
52 { GST_TAG_ALBUM, { INNER_META_KEY_ALBUM, G_TYPE_STRING, nullptr } },
53 { GST_TAG_ALBUM_ARTIST, { INNER_META_KEY_ALBUM_ARTIST, G_TYPE_STRING, nullptr } },
54 { GST_TAG_ARTIST, { INNER_META_KEY_ARTIST, G_TYPE_STRING, nullptr } },
55 { GST_TAG_COMPOSER, { INNER_META_KEY_COMPOSER, G_TYPE_STRING, nullptr } },
56 { GST_TAG_DATE_TIME, { INNER_META_KEY_DATE_TIME, GST_DATE_TIME_TYPE, DateTimeMetaSetter } },
57 { GST_TAG_GENRE, { INNER_META_KEY_GENRE, G_TYPE_STRING, nullptr } },
58 { GST_TAG_TITLE, { INNER_META_KEY_TITLE, G_TYPE_STRING, nullptr } },
59 { GST_TAG_AUTHOR, { INNER_META_KEY_AUTHOR, G_TYPE_STRING, nullptr } },
60 { GST_TAG_DURATION, { INNER_META_KEY_DURATION, G_TYPE_UINT64, nullptr } },
61 { GST_TAG_BITRATE, { INNER_META_KEY_BITRATE, G_TYPE_UINT, nullptr } },
62 { GST_TAG_IMAGE, { INNER_META_KEY_IMAGE, GST_SAMPLE_TYPE, ImageMetaSetter } },
63 { GST_TAG_LANGUAGE_CODE, { INNER_META_KEY_LANGUAGE, G_TYPE_STRING, nullptr } },
64 { GST_TAG_IMAGE_ORIENTATION, { INNER_META_KEY_VIDEO_ORIENTATION, G_TYPE_STRING, OrientationMetaSetter } },
65 };
66
67 static const std::unordered_map<std::string_view, MetaParseItem> GST_CAPS_PARSE_ITEMS = {
68 { "width", { INNER_META_KEY_VIDEO_WIDTH, G_TYPE_INT, nullptr } },
69 { "height", { INNER_META_KEY_VIDEO_HEIGHT, G_TYPE_INT, nullptr } },
70 { "rate", { INNER_META_KEY_SAMPLE_RATE, G_TYPE_INT, nullptr } },
71 { "framerate", { INNER_META_KEY_FRAMERATE, GST_FRACTION_TYPE, FractionMetaSetter } },
72 { "channels", { INNER_META_KEY_CHANNEL_COUNT, G_TYPE_INT, nullptr } },
73 };
74
75 static const std::unordered_map<std::string_view, std::vector<std::string_view>> STREAM_CAPS_FIELDS = {
76 { "video", { "width", "height", "framerate", "format" } },
77 { "audio", { "rate", "channels" } },
78 { "text", { "format" } },
79 };
80
81 static const std::unordered_map<std::string_view, std::string_view> FILE_MIME_TYPE_MAPPING = {
82 { "application/x-3gp", FILE_MIMETYPE_VIDEO_MP4 }, // mp4, m4a, mov, 3g2, 3gp
83 { "video/quicktime", FILE_MIMETYPE_VIDEO_MP4 },
84 { "video/quicktime, variant=(string)apple", FILE_MIMETYPE_VIDEO_MP4 },
85 { "video/quicktime, variant=(string)iso", FILE_MIMETYPE_VIDEO_MP4 },
86 { "video/quicktime, variant=(string)3gpp", FILE_MIMETYPE_VIDEO_MP4 },
87 { "video/quicktime, variant=(string)3g2", FILE_MIMETYPE_VIDEO_MP4 },
88 { "video/x-matroska", FILE_MIMETYPE_VIDEO_MKV}, // mkv
89 { "audio/x-matroska", FILE_MIMETYPE_AUDIO_MKV}, // mkv
90 { "video/mpegts", FILE_MIMETYPE_VIDEO_MPEGTS}, // mpeg-ts
91 { "video/webm", FILE_MIMETYPE_VIDEO_WEBM}, // webm
92 { "audio/webm", FILE_MIMETYPE_AUDIO_WEBM}, // webm
93 { "audio/x-m4a", FILE_MIMETYPE_AUDIO_MP4 },
94 { "audio/mpeg", FILE_MIMETYPE_AUDIO_AAC }, // aac
95 { "application/x-id3", FILE_MIMETYPE_AUDIO_MP3 }, // mp3
96 { "application/ogg", FILE_MIMETYPE_AUDIO_OGG }, // ogg
97 { "audio/ogg", FILE_MIMETYPE_AUDIO_OGG },
98 { "audio/x-flac", FILE_MIMETYPE_AUDIO_FLAC }, // flac
99 { "audio/x-wav", FILE_MIMETYPE_AUDIO_WAV } // wav
100 };
101
ParseGValue(const GValue & value,const MetaParseItem & item,Format & metadata)102 static void ParseGValue(const GValue &value, const MetaParseItem &item, Format &metadata)
103 {
104 if (G_VALUE_TYPE(&value) != item.valGType) {
105 MEDIA_LOGE("value type for key %{public}s is expected, curr is %{public}s, but expect %{public}s",
106 item.toKey.data(), g_type_name(G_VALUE_TYPE(&value)), g_type_name(item.valGType));
107 return;
108 }
109
110 bool ret = true;
111 if (item.setter != nullptr) {
112 ret = item.setter(value, item.toKey, metadata);
113 } else {
114 ret = ParseGValueSimple(value, item, metadata);
115 }
116
117 if (!ret) {
118 MEDIA_LOGE("parse gvalue failed for type: %{public}s, toKey: %{public}s",
119 g_type_name(item.valGType), item.toKey.data());
120 }
121 }
122
ParseTag(const GstTagList & tagList,guint tagIndex,Format & metadata)123 static void ParseTag(const GstTagList &tagList, guint tagIndex, Format &metadata)
124 {
125 const gchar *tag = gst_tag_list_nth_tag_name(&tagList, tagIndex);
126 if (tag == nullptr) {
127 return;
128 }
129
130 MEDIA_LOGD("visit tag: %{public}s", tag);
131 auto tagItemIt = GST_TAG_PARSE_ITEMS.find(tag);
132 if (tagItemIt == GST_TAG_PARSE_ITEMS.end()) {
133 return;
134 }
135
136 guint tagValSize = gst_tag_list_get_tag_size(&tagList, tag);
137 for (guint i = 0; i < tagValSize; i++) {
138 const GValue *value = gst_tag_list_get_value_index(&tagList, tag, i);
139 if (value == nullptr) {
140 MEDIA_LOGW("get value index %{public}d from tag %{public}s failed", i, tag);
141 continue;
142 }
143
144 ParseGValue(*value, tagItemIt->second, metadata);
145 }
146 }
147
ParseTagList(const GstTagList & tagList,Format & metadata)148 void GstMetaParser::ParseTagList(const GstTagList &tagList, Format &metadata)
149 {
150 gint tagCnt = gst_tag_list_n_tags(&tagList);
151 if (tagCnt <= 0) {
152 return;
153 }
154
155 for (guint tagIndex = 0; tagIndex < static_cast<guint>(tagCnt); tagIndex++) {
156 ParseTag(tagList, tagIndex, metadata);
157 }
158 }
159
ParseSingleCapsStructure(const GstStructure & structure,const std::vector<std::string_view> & target,Format & metadata)160 static void ParseSingleCapsStructure(const GstStructure &structure,
161 const std::vector<std::string_view> &target,
162 Format &metadata)
163 {
164 for (auto &field : target) {
165 if (!gst_structure_has_field(&structure, field.data())) {
166 continue;
167 }
168 const GValue *val = gst_structure_get_value(&structure, field.data());
169 if (val == nullptr) {
170 MEDIA_LOGE("get %{public}s filed's value failed", field.data());
171 continue;
172 }
173
174 auto it = GST_CAPS_PARSE_ITEMS.find(field);
175 if (it != GST_CAPS_PARSE_ITEMS.end()) {
176 ParseGValue(*val, it->second, metadata);
177 }
178 }
179 }
180
ParseStreamCaps(const GstCaps & caps,Format & metadata)181 void GstMetaParser::ParseStreamCaps(const GstCaps &caps, Format &metadata)
182 {
183 guint capsSize = gst_caps_get_size(&caps);
184 for (guint index = 0; index < capsSize; index++) {
185 const GstStructure *struc = gst_caps_get_structure(&caps, index);
186 std::string_view mimeType = STRUCTURE_NAME(struc);
187
188 std::string_view::size_type delimPos = mimeType.find('/');
189 if (delimPos == std::string_view::npos) {
190 MEDIA_LOGE("unrecognizable mimetype, %{public}s", mimeType.data());
191 continue;
192 }
193
194 MEDIA_LOGI("parse mimetype %{public}s's caps", mimeType.data());
195
196 std::string_view streamType = mimeType.substr(0, delimPos);
197 auto it = STREAM_CAPS_FIELDS.find(streamType);
198 if (it == STREAM_CAPS_FIELDS.end()) {
199 continue;
200 }
201
202 if (streamType.compare("video") == 0) {
203 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_VID);
204 } else if (streamType.compare("audio") == 0) {
205 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_AUD);
206 } else if (streamType.compare("text") == 0) {
207 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_SUBTITLE);
208 }
209
210 metadata.PutStringValue(INNER_META_KEY_MIME_TYPE, mimeType);
211 ParseSingleCapsStructure(*struc, it->second, metadata);
212 }
213 }
214
ParseFileMimeType(const GstCaps & caps,Format & metadata)215 void GstMetaParser::ParseFileMimeType(const GstCaps &caps, Format &metadata)
216 {
217 const GstStructure *struc = gst_caps_get_structure(&caps, 0); // ignore the caps size
218 CHECK_AND_RETURN_LOG(struc != nullptr, "caps invalid");
219 auto mimeType = STRUCTURE_NAME(struc);
220 CHECK_AND_RETURN_LOG(mimeType != nullptr, "caps invalid");
221
222 const auto &it = FILE_MIME_TYPE_MAPPING.find(mimeType);
223 if (it == FILE_MIME_TYPE_MAPPING.end()) {
224 MEDIA_LOGE("unrecognizable file mimetype: %{public}s", mimeType);
225 return;
226 }
227
228 MEDIA_LOGI("file caps mime type: %{public}s, mapping to %{public}s", mimeType, it->second.data());
229 metadata.PutStringValue(INNER_META_KEY_MIME_TYPE, it->second);
230 }
231
ParseGValueSimple(const GValue & value,const MetaParseItem & item,Format & metadata)232 static bool ParseGValueSimple(const GValue &value, const MetaParseItem &item, Format &metadata)
233 {
234 bool ret = true;
235 switch (item.valGType) {
236 case G_TYPE_UINT: {
237 guint num = g_value_get_uint(&value);
238 ret = metadata.PutIntValue(item.toKey, static_cast<int32_t>(num));
239 MEDIA_LOGD("toKey: %{public}s, value: %{public}u", item.toKey.data(), num);
240 break;
241 }
242 case G_TYPE_INT: {
243 gint num = g_value_get_int(&value);
244 ret = metadata.PutIntValue(item.toKey, num);
245 MEDIA_LOGD("toKey: %{public}s, value: %{public}d", item.toKey.data(), num);
246 break;
247 }
248 case G_TYPE_UINT64: {
249 guint64 num = g_value_get_uint64(&value);
250 ret = metadata.PutLongValue(item.toKey, static_cast<int64_t>(num));
251 MEDIA_LOGD("toKey: %{public}s, value: %{public}" PRIu64, item.toKey.data(), num);
252 break;
253 }
254 case G_TYPE_INT64: {
255 gint64 num = g_value_get_int64(&value);
256 ret = metadata.PutLongValue(item.toKey, num);
257 MEDIA_LOGD("toKey: %{public}s, value: %{public}" PRIi64, item.toKey.data(), num);
258 break;
259 }
260 case G_TYPE_STRING: {
261 std::string_view str = g_value_get_string(&value);
262 CHECK_AND_RETURN_RET_LOG(!str.empty(), false, "Parse key %{public}s failed", item.toKey.data());
263 ret = metadata.PutStringValue(item.toKey, str);
264 MEDIA_LOGD("toKey: %{public}s, value: %{public}s", item.toKey.data(), str.data());
265 break;
266 }
267 default:
268 break;
269 }
270 return ret;
271 }
272
FractionMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)273 static bool FractionMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
274 {
275 gint numerator = gst_value_get_fraction_numerator(&gval);
276 gint denominator = gst_value_get_fraction_denominator(&gval);
277 CHECK_AND_RETURN_RET(denominator != 0, false);
278
279 float val = static_cast<float>(numerator) / denominator;
280 static constexpr int32_t FACTOR = 100;
281 bool ret = metadata.PutIntValue(key, static_cast<int32_t>(val * FACTOR));
282 if (!ret) {
283 MEDIA_LOGE("Parse gvalue for %{public}s failed, num = %{public}d, den = %{public}d",
284 key.data(), numerator, denominator);
285 } else {
286 MEDIA_LOGD("Parse gvalue for %{public}s ok, value = %{public}f", key.data(), val);
287 }
288 return ret;
289 }
290
ImageMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)291 static bool ImageMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
292 {
293 GstSample *sample = gst_value_get_sample(&gval);
294 CHECK_AND_RETURN_RET(sample != nullptr, false);
295
296 GstCaps *caps = gst_sample_get_caps(sample);
297 CHECK_AND_RETURN_RET(caps != nullptr, false);
298
299 GstStructure *structure = gst_caps_get_structure(caps, 0);
300 CHECK_AND_RETURN_RET(structure != nullptr, false);
301
302 const gchar *mime = gst_structure_get_name(structure);
303 CHECK_AND_RETURN_RET(mime != nullptr, false);
304
305 if (g_str_has_prefix(mime, "text")) {
306 MEDIA_LOGI("image is uri, ignored, %{public}s", mime);
307 return true;
308 }
309
310 GstBuffer *imageBuf = gst_sample_get_buffer(sample);
311 CHECK_AND_RETURN_RET(imageBuf != nullptr, false);
312
313 GstMapInfo mapInfo = GST_MAP_INFO_INIT;
314 if (!gst_buffer_map(imageBuf, &mapInfo, GST_MAP_READ)) {
315 MEDIA_LOGE("get buffer data failed");
316 return false;
317 }
318
319 static constexpr gsize minImageSize = 32;
320 CHECK_AND_RETURN_RET(mapInfo.data != nullptr && mapInfo.size > minImageSize, false);
321
322 bool ret = true;
323 if (metadata.ContainKey(key)) {
324 const GstStructure *imageInfo = gst_sample_get_info(sample);
325 gint type = GST_TAG_IMAGE_TYPE_NONE;
326 if (imageInfo != nullptr) {
327 gst_structure_get_enum(imageInfo, "image-type", GST_TYPE_TAG_IMAGE_TYPE, &type);
328 }
329 if (type != GST_TAG_IMAGE_TYPE_FRONT_COVER) {
330 MEDIA_LOGI("ignore the image type: %{public}d", type);
331 } else {
332 MEDIA_LOGI("got front cover image");
333 ret = metadata.PutBuffer(key, mapInfo.data, mapInfo.size);
334 }
335 } else {
336 ret = metadata.PutBuffer(key, mapInfo.data, mapInfo.size);
337 }
338
339 MEDIA_LOGD("Parse gvalue for %{public}s finish, ret = %{public}d, mime: %{public}s, "
340 "size: %{public}" G_GSIZE_FORMAT,
341 key.data(), ret, mime, mapInfo.size);
342
343 gst_buffer_unmap(imageBuf, &mapInfo);
344 return ret;
345 }
346
DateTimeMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)347 static bool DateTimeMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
348 {
349 GstDateTime *dateTime = static_cast<GstDateTime *>(g_value_dup_boxed(&gval));
350 CHECK_AND_RETURN_RET(dateTime != nullptr, false);
351
352 std::stringstream time;
353 if (gst_date_time_has_year(dateTime)) {
354 std::string year = std::to_string(gst_date_time_get_year(dateTime));
355 time << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << year;
356 }
357
358 if (gst_date_time_has_month(dateTime)) {
359 std::string month = std::to_string(gst_date_time_get_month(dateTime));
360 time << "-" << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << month;
361 }
362
363 if (gst_date_time_has_day(dateTime)) {
364 std::string day = std::to_string(gst_date_time_get_day(dateTime));
365 time << "-" << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << day;
366 }
367
368 if (gst_date_time_has_time(dateTime)) {
369 std::string hour = std::to_string(gst_date_time_get_hour(dateTime));
370 std::string minute = std::to_string(gst_date_time_get_minute(dateTime));
371 time << " " << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << hour << ":" <<
372 std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << minute;
373 }
374
375 if (gst_date_time_has_second(dateTime)) {
376 std::string second = std::to_string(gst_date_time_get_second(dateTime));
377 time << ":" << std::setfill('0') << std::setw(FORMATTED_TIME_NUM_SIZE) << second;
378 }
379 std::string str = time.str();
380
381 gst_date_time_unref(dateTime);
382 dateTime = nullptr;
383
384 bool ret = metadata.PutStringValue(key, str);
385 MEDIA_LOGD("Key: %{public}s, value: %{public}s", key.data(), str.data());
386
387 return ret;
388 }
389
OrientationMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)390 static bool OrientationMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
391 {
392 std::string str = g_value_get_string(&gval);
393 CHECK_AND_RETURN_RET_LOG(!str.empty(), false, "Parse key %{public}s failed", key.data());
394 bool ret = false;
395
396 std::string::size_type pos = std::string::npos;
397 if ((pos = str.find("-")) == std::string::npos) {
398 ret = false;
399 } else {
400 std::string subStr = str.substr(pos + 1, str.length() - pos);
401 int32_t rotate = std::stol(subStr, nullptr, 10); // 10 : decimalism
402 MEDIA_LOGI("Get rotate str is %{public}s", subStr.c_str());
403 MEDIA_LOGI("Get rotate is %{public}d", rotate);
404 ret = metadata.PutIntValue(key, rotate);
405 }
406
407 return ret;
408 }
409 } // namespace Media
410 } // namespace OHOS
411