1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmeta_frame_converter.h"
17 #include <gst/app/gstappsrc.h>
18 #include <gst/video/video-info.h>
19 #include "media_errors.h"
20 #include "media_log.h"
21 #include "gst_utils.h"
22 #include "gst_shmem_memory.h"
23 #include "scope_guard.h"
24 #include "time_perf.h"
25
26 namespace {
27 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaFrameConv"};
28 static constexpr int32_t KEEP_ORIGINAL_WIDTH_OR_HEIGHT = -1;
29 }
30
31 namespace OHOS {
32 namespace Media {
33 struct PixelFormatInfo {
34 PixelFormat format;
35 std::string_view gstVideoFormat;
36 uint8_t bytesPerPixel;
37 };
38
39 static const std::unordered_map<PixelFormat, PixelFormatInfo> PIXELFORMAT_INFO = {
40 { PixelFormat::RGB_565, { PixelFormat::RGB_565, "RGB16", 2 } },
41 { PixelFormat::RGB_888, { PixelFormat::RGB_888, "RGB", 3 } },
42 { PixelFormat::RGBA_8888, { PixelFormat::RGBA_8888, "RGBA", 4 } },
43 };
44
AVMetaFrameConverter()45 AVMetaFrameConverter::AVMetaFrameConverter()
46 {
47 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
48 }
49
~AVMetaFrameConverter()50 AVMetaFrameConverter::~AVMetaFrameConverter()
51 {
52 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
53 (void)Reset();
54 CLEAN_PERF_RECORD(this);
55 }
56
Init(const OutputConfiguration & config)57 int32_t AVMetaFrameConverter::Init(const OutputConfiguration &config)
58 {
59 std::unique_lock<std::mutex> lock(mutex_);
60
61 int32_t ret = SetupConvPipeline();
62 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
63
64 ret = SetupConvSrc();
65 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
66
67 ret = SetupConvSink(config);
68 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
69
70 ret = SetupMsgProcessor();
71 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
72
73 return MSERR_OK;
74 }
75
Convert(GstCaps & inCaps,GstBuffer & inBuf)76 std::shared_ptr<AVSharedMemory> AVMetaFrameConverter::Convert(GstCaps &inCaps, GstBuffer &inBuf)
77 {
78 AUTO_PERF(this, "ConvertFrame");
79
80 std::unique_lock<std::mutex> lock(mutex_);
81
82 int32_t ret = PrepareConvert(inCaps);
83 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "prepare convert failed");
84
85 GstFlowReturn flowRet = GST_FLOW_ERROR;
86 g_signal_emit_by_name(G_OBJECT(appSrc_), "push-buffer", &inBuf, &flowRet);
87 CHECK_AND_RETURN_RET_LOG(flowRet == GST_FLOW_OK, nullptr, "push buffer failed");
88
89 cond_.wait(lock, [this]() { return lastResult_ != nullptr || !startConverting_; });
90
91 return GetConvertResult();
92 }
93
PrepareConvert(GstCaps & inCaps)94 int32_t AVMetaFrameConverter::PrepareConvert(GstCaps &inCaps)
95 {
96 ON_SCOPE_EXIT(0) { (void)GetConvertResult(); };
97
98 if (lastCaps_ == nullptr || !gst_caps_is_equal(lastCaps_, &inCaps)) {
99 MEDIA_LOGI("caps changed");
100 gchar *capstr = gst_caps_to_string(&inCaps);
101 if (capstr != nullptr) {
102 MEDIA_LOGI("current caps: %{public}s", capstr);
103 g_free(capstr);
104 capstr = nullptr;
105 }
106 int32_t ret = ChangeState(GST_STATE_READY);
107 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
108
109 g_object_set(G_OBJECT(appSrc_), "caps", &inCaps, nullptr);
110 if (lastCaps_ != nullptr) {
111 gst_caps_unref(lastCaps_);
112 }
113 lastCaps_ = gst_caps_ref(&inCaps);
114
115 ret = ChangeState(GST_STATE_PLAYING);
116 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
117 }
118
119 CANCEL_SCOPE_EXIT_GUARD(0);
120 startConverting_ = true;
121
122 return MSERR_OK;
123 }
124
GetConvertResult()125 std::shared_ptr<AVSharedMemory> AVMetaFrameConverter::GetConvertResult()
126 {
127 startConverting_ = false;
128
129 if (lastResult_ == nullptr) {
130 MEDIA_LOGE("last result frame is nullptr");
131 return nullptr;
132 }
133
134 ON_SCOPE_EXIT(0) {
135 gst_buffer_unref(lastResult_);
136 lastResult_ = nullptr;
137 };
138
139 GstMapInfo info = GST_MAP_INFO_INIT;
140 gboolean ret = gst_buffer_map(lastResult_, &info, GST_MAP_READ);
141 CHECK_AND_RETURN_RET_LOG(ret, nullptr, "map failed, exit convert frame");
142
143 ON_SCOPE_EXIT(1) { gst_buffer_unmap(lastResult_, &info); };
144
145 GstVideoMeta *videoMeta = gst_buffer_get_video_meta(lastResult_);
146 CHECK_AND_RETURN_RET_LOG(videoMeta != nullptr, nullptr, "get video meta failed");
147
148 GstShMemMemory *mem = reinterpret_cast<GstShMemMemory *>(info.memory);
149 CHECK_AND_RETURN_RET_LOG(mem != nullptr, nullptr, "mem is nullptr");
150 CHECK_AND_RETURN_RET_LOG(mem->mem != nullptr, nullptr, "mem->mem is nullptr");
151 CHECK_AND_RETURN_RET_LOG(mem->mem->GetBase() != nullptr, nullptr, "addr is nullptr");
152
153 std::shared_ptr<AVSharedMemory> result = mem->mem;
154 if (!(result->GetSize() > 0 && static_cast<uint32_t>(result->GetSize()) >= sizeof(OutputFrame))) {
155 MEDIA_LOGE("size is incorrect");
156 return nullptr;
157 }
158
159 auto frame = reinterpret_cast<OutputFrame *>(result->GetBase());
160 frame->bytesPerPixel_ = PIXELFORMAT_INFO.at(outConfig_.colorFormat).bytesPerPixel;
161 frame->width_ = static_cast<int32_t>(videoMeta->width);
162 frame->height_ = static_cast<int32_t>(videoMeta->height);
163 frame->stride_ = videoMeta->stride[0];
164 frame->size_ = frame->stride_ * frame->height_;
165
166 CHECK_AND_RETURN_RET_LOG(result->GetSize() >= frame->GetFlattenedSize(), nullptr, "size is incorrect");
167
168 MEDIA_LOGI("======================Convert Frame Finished=========================");
169 MEDIA_LOGI("output width = %{public}d, stride = %{public}d, height = %{public}d, format = %{public}d",
170 frame->width_, frame->stride_, frame->height_, outConfig_.colorFormat);
171 return result;
172 }
173
UninstallPipeline()174 void AVMetaFrameConverter::UninstallPipeline()
175 {
176 if (pipeline_ != nullptr) {
177 ChangeState(GST_STATE_NULL);
178 gst_object_unref(pipeline_);
179 pipeline_ = nullptr;
180 }
181
182 if (pipeline_ != nullptr) {
183 gst_object_unref(pipeline_);
184 pipeline_ = nullptr;
185 }
186
187 if (vidShMemSink_ != nullptr) {
188 gst_object_unref(vidShMemSink_);
189 vidShMemSink_ = nullptr;
190 }
191
192 if (appSrc_ != nullptr) {
193 gst_object_unref(appSrc_);
194 appSrc_ = nullptr;
195 }
196
197 currState_ = GST_STATE_NULL;
198 }
199
Reset()200 int32_t AVMetaFrameConverter::Reset()
201 {
202 std::unique_lock<std::mutex> lock(mutex_);
203
204 /**
205 * Must flush before change state and delete the msgProcessor, otherwise deadlock will
206 * happened when try to destroy the msgprocessor.
207 */
208 auto tempMsgProc = std::move(msgProcessor_);
209 lock.unlock();
210 tempMsgProc->FlushBegin();
211 tempMsgProc->Reset();
212 tempMsgProc = nullptr;
213 lock.lock();
214
215 UninstallPipeline();
216
217 if (lastResult_ != nullptr) {
218 gst_buffer_unref(lastResult_);
219 lastResult_ = nullptr;
220 }
221
222 if (lastCaps_ != nullptr) {
223 gst_caps_unref(lastCaps_);
224 lastCaps_ = nullptr;
225 }
226
227 for (auto &sample : allResults_) {
228 gst_buffer_unref(sample);
229 }
230 allResults_.clear();
231
232 startConverting_ = false;
233 cond_.notify_all();
234
235 return MSERR_OK;
236 }
237
SetupConvPipeline()238 int32_t AVMetaFrameConverter::SetupConvPipeline()
239 {
240 pipeline_ = GST_PIPELINE_CAST(gst_pipeline_new("conv_pipeline"));
241 CHECK_AND_RETURN_RET(pipeline_ != nullptr, MSERR_INVALID_OPERATION);
242 GstBin *bin = GST_BIN_CAST(pipeline_);
243
244 appSrc_ = GST_ELEMENT_CAST(gst_object_ref(gst_element_factory_make("appsrc", "conv_src")));
245 CHECK_AND_RETURN_RET(appSrc_ != nullptr, MSERR_INVALID_OPERATION);
246 CHECK_AND_RETURN_RET(gst_bin_add(bin, appSrc_), MSERR_INVALID_OPERATION);
247
248 GstElement *scale = gst_element_factory_make("videoscale", "conv_scale");
249 CHECK_AND_RETURN_RET(scale != nullptr, MSERR_INVALID_OPERATION);
250 CHECK_AND_RETURN_RET(gst_bin_add(bin, scale), MSERR_INVALID_OPERATION);
251
252 GstElement *conv = gst_element_factory_make("videoconvert", "conv_conv");
253 CHECK_AND_RETURN_RET(conv != nullptr, MSERR_INVALID_OPERATION);
254 CHECK_AND_RETURN_RET(gst_bin_add(bin, conv), MSERR_INVALID_OPERATION);
255
256 vidShMemSink_ = GST_ELEMENT_CAST(gst_object_ref(gst_element_factory_make("sharedmemsink", "conv_sink")));
257 CHECK_AND_RETURN_RET(vidShMemSink_ != nullptr, MSERR_INVALID_OPERATION);
258 CHECK_AND_RETURN_RET(gst_bin_add(bin, vidShMemSink_), MSERR_INVALID_OPERATION);
259
260 gboolean ret = gst_element_link_pads_full(appSrc_, "src", scale, "sink", GST_PAD_LINK_CHECK_NOTHING);
261 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
262 ret = gst_element_link_pads_full(scale, "src", conv, "sink", GST_PAD_LINK_CHECK_NOTHING);
263 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
264 ret = gst_element_link_pads_full(conv, "src", vidShMemSink_, "sink", GST_PAD_LINK_CHECK_NOTHING);
265 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
266
267 MEDIA_LOGI("setup converter pipeline success");
268 return MSERR_OK;
269 }
270
SetupConvSrc()271 int32_t AVMetaFrameConverter::SetupConvSrc()
272 {
273 g_object_set(appSrc_, "is-live", TRUE, nullptr);
274 g_object_set(appSrc_, "stream-type", GST_APP_STREAM_TYPE_STREAM, nullptr);
275 g_object_set(appSrc_, "do-timestamp", TRUE, nullptr);
276 return MSERR_OK;
277 }
278
SetupConvSink(const OutputConfiguration & outConfig)279 int32_t AVMetaFrameConverter::SetupConvSink(const OutputConfiguration &outConfig)
280 {
281 if (PIXELFORMAT_INFO.count(outConfig.colorFormat) == 0) {
282 MEDIA_LOGE("pixelformat unsupported: %{public}d", outConfig.colorFormat);
283 return MSERR_INVALID_VAL;
284 }
285
286 MEDIA_LOGI("target out config: width: %{public}d, height: %{public}d, format: %{public}d",
287 outConfig.dstWidth, outConfig.dstHeight, outConfig.colorFormat);
288
289 const char *formatStr = PIXELFORMAT_INFO.at(outConfig.colorFormat).gstVideoFormat.data();
290 GstStructure *struc = gst_structure_new("video/x-raw", "format", G_TYPE_STRING, formatStr, nullptr);
291 CHECK_AND_RETURN_RET(struc != nullptr, MSERR_NO_MEMORY);
292
293 if (outConfig.dstHeight != KEEP_ORIGINAL_WIDTH_OR_HEIGHT) {
294 gst_structure_set(struc, "height", G_TYPE_INT, outConfig.dstHeight, nullptr);
295 }
296
297 if (outConfig.dstWidth != KEEP_ORIGINAL_WIDTH_OR_HEIGHT) {
298 gst_structure_set(struc, "width", G_TYPE_INT, outConfig.dstWidth, nullptr);
299 }
300
301 GstCaps *caps = gst_caps_new_full(struc, nullptr);
302 CHECK_AND_RETURN_RET(caps != nullptr, MSERR_NO_MEMORY);
303
304 g_object_set(G_OBJECT(vidShMemSink_), "caps", caps, nullptr);
305 gst_caps_unref(caps);
306 caps = nullptr;
307 g_object_set(G_OBJECT(vidShMemSink_), "mem-prefix-size", sizeof(OutputFrame), nullptr);
308
309 GstMemSinkCallbacks callbacks = { nullptr, nullptr, OnNotifyNewSample };
310 gst_mem_sink_set_callback(GST_MEM_SINK_CAST(vidShMemSink_), &callbacks, this, nullptr);
311
312 outConfig_ = outConfig;
313 return MSERR_OK;
314 }
315
SetupMsgProcessor()316 int32_t AVMetaFrameConverter::SetupMsgProcessor()
317 {
318 GstBus *bus = gst_pipeline_get_bus(pipeline_);
319 CHECK_AND_RETURN_RET_LOG(bus != nullptr, MSERR_UNKNOWN, "can not get bus");
320
321 auto msgNotifier = std::bind(&AVMetaFrameConverter::OnNotifyMessage, this, std::placeholders::_1);
322 msgProcessor_ = std::make_unique<GstMsgProcessor>(*bus, msgNotifier);
323 gst_object_unref(bus);
324 bus = nullptr;
325
326 int32_t ret = msgProcessor_->Init();
327 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
328
329 // only concern the msg from pipeline
330 msgProcessor_->AddMsgFilter(ELEM_NAME(GST_ELEMENT_CAST(pipeline_)));
331
332 return MSERR_OK;
333 }
334
ChangeState(GstState targetState)335 int32_t AVMetaFrameConverter::ChangeState(GstState targetState)
336 {
337 MEDIA_LOGI("begin change state to %{public}s", gst_element_state_get_name(targetState));
338
339 GstStateChangeReturn stateRet = gst_element_set_state(GST_ELEMENT_CAST(pipeline_), targetState);
340 if (stateRet == GST_STATE_CHANGE_FAILURE) {
341 MEDIA_LOGE("change conv pipeline to ready failed");
342 return MSERR_INVALID_OPERATION;
343 }
344
345 return MSERR_OK;
346 }
347
OnNotifyMessage(const InnerMessage & msg)348 void AVMetaFrameConverter::OnNotifyMessage(const InnerMessage &msg)
349 {
350 MEDIA_LOGD("msg: %{public}d", msg.type);
351 std::unique_lock<std::mutex> lock(mutex_);
352
353 switch (msg.type) {
354 case InnerMsgType::INNER_MSG_STATE_CHANGED: {
355 MEDIA_LOGD("state is %{public}s", gst_element_state_get_name(currState_));
356 currState_ = static_cast<GstState>(msg.detail2);
357 cond_.notify_all();
358 break;
359 }
360 case InnerMsgType::INNER_MSG_ERROR: {
361 startConverting_ = false;
362 MEDIA_LOGE("error happened");
363 cond_.notify_all();
364 break;
365 }
366 default:
367 break;
368 }
369 }
370
OnNotifyNewSample(GstMemSink * elem,GstBuffer * sample,gpointer userData)371 GstFlowReturn AVMetaFrameConverter::OnNotifyNewSample(GstMemSink *elem, GstBuffer *sample, gpointer userData)
372 {
373 CHECK_AND_RETURN_RET(sample != nullptr, GST_FLOW_ERROR);
374 ON_SCOPE_EXIT(0) { gst_buffer_unref(sample); };
375
376 CHECK_AND_RETURN_RET(userData != nullptr, GST_FLOW_ERROR);
377 CHECK_AND_RETURN_RET(elem != nullptr, GST_FLOW_ERROR);
378 auto thiz = reinterpret_cast<AVMetaFrameConverter *>(userData);
379
380 std::unique_lock<std::mutex> lock(thiz->mutex_);
381 if (thiz->lastResult_ != nullptr) {
382 gst_buffer_unref(thiz->lastResult_);
383 thiz->lastResult_ = nullptr;
384 }
385
386 MEDIA_LOGI("sample buffer arrived, pts: %{public}" PRIu64 "", GST_BUFFER_PTS(sample));
387
388 thiz->lastResult_ = gst_buffer_ref(sample);
389 CHECK_AND_RETURN_RET(thiz->lastResult_ != nullptr, GST_FLOW_ERROR);
390 // increase the refcount to avoid the buffer to be released to bufferpool.
391 thiz->allResults_.push_back(gst_buffer_ref(sample));
392
393 thiz->cond_.notify_all();
394 return GST_FLOW_OK;
395 }
396 } // namespace Media
397 } // namespace OHOS
398