1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "gst_meta_parser.h"
17 #include <functional>
18 #include <vector>
19 #include "av_common.h"
20 #include "media_errors.h"
21 #include "media_log.h"
22 #include "gst/tag/tag.h"
23 #include "gst_utils.h"
24
25 namespace {
26 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "GstMetaParser"};
27 static GType GST_SAMPLE_TYPE = gst_sample_get_type();
28 }
29
30 namespace OHOS {
31 namespace Media {
32 using MetaSetter = std::function<bool(const GValue &gval, const std::string_view &key, Format &metadata)>;
33
34 struct MetaParseItem {
35 std::string_view toKey;
36 GType valGType;
37 MetaSetter setter;
38 };
39
40 static bool ParseGValueSimple(const GValue &value, const MetaParseItem &item, Format &metadata);
41 static bool FractionMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
42 static bool ImageMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata);
43
44 static const std::unordered_map<std::string_view, MetaParseItem> GST_TAG_PARSE_ITEMS = {
45 { GST_TAG_ALBUM, { INNER_META_KEY_ALBUM, G_TYPE_STRING } },
46 { GST_TAG_ALBUM_ARTIST, { INNER_META_KEY_ALBUM_ARTIST, G_TYPE_STRING } },
47 { GST_TAG_ARTIST, { INNER_META_KEY_ARTIST, G_TYPE_STRING } },
48 { GST_TAG_COMPOSER, { INNER_META_KEY_COMPOSER, G_TYPE_STRING } },
49 { GST_TAG_GENRE, { INNER_META_KEY_GENRE, G_TYPE_STRING } },
50 { GST_TAG_TITLE, { INNER_META_KEY_TITLE, G_TYPE_STRING } },
51 { GST_TAG_AUTHOR, { INNER_META_KEY_AUTHOR, G_TYPE_STRING } },
52 { GST_TAG_DURATION, { INNER_META_KEY_DURATION, G_TYPE_UINT64 } },
53 { GST_TAG_BITRATE, { INNER_META_KEY_BITRATE, G_TYPE_UINT } },
54 { GST_TAG_IMAGE, { INNER_META_KEY_IMAGE, GST_SAMPLE_TYPE, ImageMetaSetter } },
55 };
56
57 static const std::unordered_map<std::string_view, MetaParseItem> GST_CAPS_PARSE_ITEMS = {
58 { "width", { INNER_META_KEY_VIDEO_WIDTH, G_TYPE_INT, nullptr } },
59 { "height", { INNER_META_KEY_VIDEO_HEIGHT, G_TYPE_INT } },
60 { "rate", { INNER_META_KEY_SAMPLE_RATE, G_TYPE_INT } },
61 { "framerate", { INNER_META_KEY_FRAMERATE, GST_TYPE_FRACTION, FractionMetaSetter } },
62 { "channels", { INNER_META_KEY_CHANNEL_COUNT, G_TYPE_INT } },
63 };
64
65 static const std::unordered_map<std::string_view, std::vector<std::string_view>> STREAM_CAPS_FIELDS = {
66 { "video", { "width", "height", "framrate", "format" } },
67 { "audio", { "rate", "channels" } },
68 { "text", { "format" } },
69 };
70
71 static const std::unordered_map<std::string_view, std::string_view> FILE_MIME_TYPE_MAPPING = {
72 { "application/x-3gp", FILE_MIMETYPE_VIDEO_MP4 }, // mp4, m4a, mov, 3g2, 3gp
73 { "video/quicktime", FILE_MIMETYPE_VIDEO_MP4 },
74 { "video/quicktime, variant=(string)apple", FILE_MIMETYPE_VIDEO_MP4 },
75 { "video/quicktime, variant=(string)iso", FILE_MIMETYPE_VIDEO_MP4 },
76 { "video/quicktime, variant=(string)3gpp", FILE_MIMETYPE_VIDEO_MP4 },
77 { "video/quicktime, variant=(string)3g2", FILE_MIMETYPE_VIDEO_MP4 },
78 { "audio/x-m4a", FILE_MIMETYPE_AUDIO_MP4 },
79 { "audio/mpeg", FILE_MIMETYPE_AUDIO_AAC }, // aac
80 { "application/x-id3", FILE_MIMETYPE_AUDIO_MP3 }, // mp3
81 { "application/ogg", FILE_MIMETYPE_AUDIO_OGG }, // ogg
82 { "audio/ogg", FILE_MIMETYPE_AUDIO_OGG },
83 { "audio/x-flac", FILE_MIMETYPE_AUDIO_FLAC }, // flac
84 { "audio/x-wav", FILE_MIMETYPE_AUDIO_WAV } // wav
85 };
86
ParseGValue(const GValue & value,const MetaParseItem & item,Format & metadata)87 static void ParseGValue(const GValue &value, const MetaParseItem &item, Format &metadata)
88 {
89 if (G_VALUE_TYPE(&value) != item.valGType) {
90 MEDIA_LOGE("value type for key %{public}s is expected, curr is %{public}s, but expect %{public}s",
91 item.toKey.data(), g_type_name(G_VALUE_TYPE(&value)), g_type_name(item.valGType));
92 return;
93 }
94
95 bool ret = true;
96 if (item.setter != nullptr) {
97 ret = item.setter(value, item.toKey, metadata);
98 } else {
99 ret = ParseGValueSimple(value, item, metadata);
100 }
101
102 if (!ret) {
103 MEDIA_LOGE("parse gvalue failed for type: %{public}s, toKey: %{public}s",
104 g_type_name(item.valGType), item.toKey.data());
105 }
106 }
107
ParseTag(const GstTagList & tagList,guint tagIndex,Format & metadata)108 static void ParseTag(const GstTagList &tagList, guint tagIndex, Format &metadata)
109 {
110 const gchar *tag = gst_tag_list_nth_tag_name(&tagList, tagIndex);
111 if (tag == nullptr) {
112 return;
113 }
114
115 MEDIA_LOGD("visit tag: %{public}s", tag);
116 auto tagItemIt = GST_TAG_PARSE_ITEMS.find(tag);
117 if (tagItemIt == GST_TAG_PARSE_ITEMS.end()) {
118 return;
119 }
120
121 guint tagValSize = gst_tag_list_get_tag_size(&tagList, tag);
122 for (guint i = 0; i < tagValSize; i++) {
123 const GValue *value = gst_tag_list_get_value_index(&tagList, tag, i);
124 if (value == nullptr) {
125 MEDIA_LOGW("get value index %{public}d from tag %{public}s failed", i, tag);
126 continue;
127 }
128
129 ParseGValue(*value, tagItemIt->second, metadata);
130 }
131 }
132
ParseTagList(const GstTagList & tagList,Format & metadata)133 void GstMetaParser::ParseTagList(const GstTagList &tagList, Format &metadata)
134 {
135 gint tagCnt = gst_tag_list_n_tags(&tagList);
136 if (tagCnt <= 0) {
137 return;
138 }
139
140 for (guint tagIndex = 0; tagIndex < static_cast<guint>(tagCnt); tagIndex++) {
141 ParseTag(tagList, tagIndex, metadata);
142 }
143 }
144
ParseSingleCapsStructure(const GstStructure & structure,const std::vector<std::string_view> & target,Format & metadata)145 static void ParseSingleCapsStructure(const GstStructure &structure,
146 const std::vector<std::string_view> &target,
147 Format &metadata)
148 {
149 for (auto &field : target) {
150 if (!gst_structure_has_field(&structure, field.data())) {
151 continue;
152 }
153 const GValue *val = gst_structure_get_value(&structure, field.data());
154 if (val == nullptr) {
155 MEDIA_LOGE("get %{public}s filed's value failed", field.data());
156 continue;
157 }
158
159 auto it = GST_CAPS_PARSE_ITEMS.find(field);
160 if (it != GST_CAPS_PARSE_ITEMS.end()) {
161 ParseGValue(*val, it->second, metadata);
162 }
163 }
164 }
165
ParseStreamCaps(const GstCaps & caps,Format & metadata)166 void GstMetaParser::ParseStreamCaps(const GstCaps &caps, Format &metadata)
167 {
168 guint capsSize = gst_caps_get_size(&caps);
169 for (guint index = 0; index < capsSize; index++) {
170 const GstStructure *struc = gst_caps_get_structure(&caps, index);
171 std::string_view mimeType = STRUCTURE_NAME(struc);
172
173 std::string_view::size_type delimPos = mimeType.find('/');
174 if (delimPos == std::string_view::npos) {
175 MEDIA_LOGE("unrecognizable mimetype, %{public}s", mimeType.data());
176 continue;
177 }
178
179 MEDIA_LOGI("parse mimetype %{public}s's caps", mimeType.data());
180
181 std::string_view streamType = mimeType.substr(0, delimPos);
182 auto it = STREAM_CAPS_FIELDS.find(streamType);
183 if (it == STREAM_CAPS_FIELDS.end()) {
184 continue;
185 }
186
187 if (streamType.compare("video") == 0) {
188 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_VID);
189 } else if (streamType.compare("audio") == 0) {
190 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_AUD);
191 } else if (streamType.compare("text") == 0) {
192 metadata.PutIntValue(INNER_META_KEY_TRACK_TYPE, MediaType::MEDIA_TYPE_SUBTITLE);
193 }
194
195 metadata.PutStringValue(INNER_META_KEY_MIME_TYPE, mimeType);
196 ParseSingleCapsStructure(*struc, it->second, metadata);
197 }
198 }
199
ParseFileMimeType(const GstCaps & caps,Format & metadata)200 void GstMetaParser::ParseFileMimeType(const GstCaps &caps, Format &metadata)
201 {
202 const GstStructure *struc = gst_caps_get_structure(&caps, 0); // ignore the caps size
203 CHECK_AND_RETURN_LOG(struc != nullptr, "caps invalid");
204 auto mimeType = STRUCTURE_NAME(struc);
205 CHECK_AND_RETURN_LOG(mimeType != nullptr, "caps invalid");
206
207 const auto &it = FILE_MIME_TYPE_MAPPING.find(mimeType);
208 if (it == FILE_MIME_TYPE_MAPPING.end()) {
209 MEDIA_LOGE("unrecognizable file mimetype: %{public}s", mimeType);
210 return;
211 }
212
213 MEDIA_LOGI("file caps mime type: %{public}s, mapping to %{public}s", mimeType, it->second.data());
214 metadata.PutStringValue(INNER_META_KEY_MIME_TYPE, it->second);
215 }
216
ParseGValueSimple(const GValue & value,const MetaParseItem & item,Format & metadata)217 static bool ParseGValueSimple(const GValue &value, const MetaParseItem &item, Format &metadata)
218 {
219 bool ret = true;
220 switch (item.valGType) {
221 case G_TYPE_UINT: {
222 guint num = g_value_get_uint(&value);
223 ret = metadata.PutIntValue(item.toKey, static_cast<int32_t>(num));
224 MEDIA_LOGD("toKey: %{public}s, value: %{public}u", item.toKey.data(), num);
225 break;
226 }
227 case G_TYPE_INT: {
228 gint num = g_value_get_int(&value);
229 ret = metadata.PutIntValue(item.toKey, num);
230 MEDIA_LOGD("toKey: %{public}s, value: %{public}u", item.toKey.data(), num);
231 break;
232 }
233 case G_TYPE_UINT64: {
234 guint64 num = g_value_get_uint64(&value);
235 ret = metadata.PutLongValue(item.toKey, static_cast<int64_t>(num));
236 MEDIA_LOGD("toKey: %{public}s, value: %{public}" PRIu64, item.toKey.data(), num);
237 break;
238 }
239 case G_TYPE_INT64: {
240 gint64 num = g_value_get_int64(&value);
241 ret = metadata.PutLongValue(item.toKey, num);
242 MEDIA_LOGD("toKey: %{public}s, value: %{public}" PRIi64, item.toKey.data(), num);
243 break;
244 }
245 case G_TYPE_STRING: {
246 std::string_view str = g_value_get_string(&value);
247 CHECK_AND_RETURN_RET_LOG(!str.empty(), false, "Parse key %{public}s failed", item.toKey.data());
248 ret = metadata.PutStringValue(item.toKey, str);
249 MEDIA_LOGD("toKey: %{public}s, value: %{public}s", item.toKey.data(), str.data());
250 break;
251 }
252 default:
253 break;
254 }
255 return true;
256 }
257
FractionMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)258 static bool FractionMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
259 {
260 gint numerator = gst_value_get_fraction_numerator(&gval);
261 gint denominator = gst_value_get_fraction_denominator(&gval);
262 CHECK_AND_RETURN_RET(denominator != 0, false);
263
264 float val = static_cast<float>(numerator) / denominator;
265 static constexpr int32_t FACTOR = 100;
266 bool ret = metadata.PutIntValue(key, static_cast<int32_t>(val * FACTOR));
267 if (!ret) {
268 MEDIA_LOGE("Parse gvalue for %{public}s failed, num = %{public}d, den = %{public}d",
269 key.data(), numerator, denominator);
270 } else {
271 MEDIA_LOGD("Parse gvalue for %{public}s ok, value = %{public}f", key.data(), val);
272 }
273 return ret;
274 }
275
ImageMetaSetter(const GValue & gval,const std::string_view & key,Format & metadata)276 static bool ImageMetaSetter(const GValue &gval, const std::string_view &key, Format &metadata)
277 {
278 GstSample *sample = gst_value_get_sample(&gval);
279 CHECK_AND_RETURN_RET(sample != nullptr, false);
280
281 GstCaps *caps = gst_sample_get_caps(sample);
282 CHECK_AND_RETURN_RET(caps != nullptr, false);
283
284 GstStructure *structure = gst_caps_get_structure(caps, 0);
285 CHECK_AND_RETURN_RET(structure != nullptr, false);
286
287 const gchar *mime = gst_structure_get_name(structure);
288 CHECK_AND_RETURN_RET(mime != nullptr, false);
289
290 if (g_str_has_prefix(mime, "text")) {
291 MEDIA_LOGI("image is uri, ignored, %{public}s", mime);
292 return true;
293 }
294
295 GstBuffer *imageBuf = gst_sample_get_buffer(sample);
296 CHECK_AND_RETURN_RET(imageBuf != nullptr, false);
297
298 GstMapInfo mapInfo = { 0 };
299 if (!gst_buffer_map(imageBuf, &mapInfo, GST_MAP_READ)) {
300 MEDIA_LOGE("get buffer data failed");
301 return false;
302 }
303
304 static constexpr gsize minImageSize = 32;
305 CHECK_AND_RETURN_RET(mapInfo.data != nullptr && mapInfo.size > minImageSize, false);
306
307 bool ret = true;
308 if (metadata.ContainKey(key)) {
309 const GstStructure *imageInfo = gst_sample_get_info(sample);
310 gint type = GST_TAG_IMAGE_TYPE_NONE;
311 if (imageInfo != nullptr) {
312 gst_structure_get_enum(imageInfo, "image-type", GST_TYPE_TAG_IMAGE_TYPE, &type);
313 }
314 if (type != GST_TAG_IMAGE_TYPE_FRONT_COVER) {
315 MEDIA_LOGI("ignore the image type: %{public}d", type);
316 } else {
317 MEDIA_LOGI("got front cover image");
318 ret = metadata.PutBuffer(key, mapInfo.data, mapInfo.size);
319 }
320 } else {
321 ret = metadata.PutBuffer(key, mapInfo.data, mapInfo.size);
322 }
323
324 MEDIA_LOGD("Parse gvalue for %{public}s finish, ret = %{public}d, mime: %{public}s, "
325 "size: %{public}" G_GSIZE_FORMAT,
326 key.data(), ret, mime, mapInfo.size);
327
328 gst_buffer_unmap(imageBuf, &mapInfo);
329 return ret;
330 }
331 } // namespace Media
332 } // namespace OHOS