1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmeta_frame_converter.h"
17 #include <gst/app/gstappsrc.h>
18 #include <gst/video/video-info.h>
19 #include "media_errors.h"
20 #include "media_log.h"
21 #include "gst_utils.h"
22 #include "gst_shmem_memory.h"
23 #include "scope_guard.h"
24
25 namespace {
26 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaFrameConv"};
27 static constexpr int32_t KEEP_ORIGINAL_WIDTH_OR_HEIGHT = -1;
28 }
29
30 namespace OHOS {
31 namespace Media {
32 struct PixelFormatInfo {
33 PixelFormat format;
34 std::string_view gstVideoFormat;
35 uint8_t bytesPerPixel;
36 };
37
38 static const std::unordered_map<PixelFormat, PixelFormatInfo> PIXELFORMAT_INFO = {
39 { PixelFormat::RGB_565, { PixelFormat::RGB_565, "RGB16", 2 } },
40 { PixelFormat::RGB_888, { PixelFormat::RGB_888, "RGB", 3 } },
41 { PixelFormat::RGBA_8888, { PixelFormat::RGBA_8888, "RGBA", 4 } },
42 };
43
AVMetaFrameConverter()44 AVMetaFrameConverter::AVMetaFrameConverter()
45 {
46 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
47 }
48
~AVMetaFrameConverter()49 AVMetaFrameConverter::~AVMetaFrameConverter()
50 {
51 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
52 (void)Reset();
53 }
54
Init(const OutputConfiguration & config)55 int32_t AVMetaFrameConverter::Init(const OutputConfiguration &config)
56 {
57 std::unique_lock<std::mutex> lock(mutex_);
58
59 int32_t ret = SetupConvPipeline();
60 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
61
62 ret = SetupConvSrc();
63 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
64
65 ret = SetupConvSink(config);
66 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
67
68 ret = SetupMsgProcessor();
69 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
70
71 return MSERR_OK;
72 }
73
Convert(GstCaps & inCaps,GstBuffer & inBuf)74 std::shared_ptr<AVSharedMemory> AVMetaFrameConverter::Convert(GstCaps &inCaps, GstBuffer &inBuf)
75 {
76 std::unique_lock<std::mutex> lock(mutex_);
77
78 int32_t ret = PrepareConvert(inCaps);
79 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "prepare convert failed");
80
81 GstFlowReturn flowRet = GST_FLOW_ERROR;
82 g_signal_emit_by_name(G_OBJECT(appSrc_), "push-buffer", &inBuf, &flowRet);
83 CHECK_AND_RETURN_RET_LOG(flowRet == GST_FLOW_OK, nullptr, "push buffer failed");
84
85 cond_.wait(lock, [this]() { return lastResult_ != nullptr || !startConverting_; });
86
87 return GetConvertResult();
88 }
89
PrepareConvert(GstCaps & inCaps)90 int32_t AVMetaFrameConverter::PrepareConvert(GstCaps &inCaps)
91 {
92 ON_SCOPE_EXIT(0) { (void)GetConvertResult(); };
93
94 if (lastCaps_ == nullptr || !gst_caps_is_equal(lastCaps_, &inCaps)) {
95 MEDIA_LOGI("caps changed");
96 gchar *capstr = gst_caps_to_string(&inCaps);
97 if (capstr != nullptr) {
98 MEDIA_LOGI("current caps: %{public}s", capstr);
99 g_free(capstr);
100 capstr = nullptr;
101 }
102 int32_t ret = ChangeState(GST_STATE_READY);
103 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
104
105 g_object_set(G_OBJECT(appSrc_), "caps", &inCaps, nullptr);
106 if (lastCaps_ != nullptr) {
107 gst_caps_unref(lastCaps_);
108 }
109 lastCaps_ = gst_caps_ref(&inCaps);
110
111 ret = ChangeState(GST_STATE_PLAYING);
112 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
113 }
114
115 CANCEL_SCOPE_EXIT_GUARD(0);
116 startConverting_ = true;
117
118 return MSERR_OK;
119 }
120
GetConvertResult()121 std::shared_ptr<AVSharedMemory> AVMetaFrameConverter::GetConvertResult()
122 {
123 startConverting_ = false;
124
125 if (lastResult_ == nullptr) {
126 MEDIA_LOGE("last result frame is nullptr");
127 return nullptr;
128 }
129
130 ON_SCOPE_EXIT(0) {
131 gst_buffer_unref(lastResult_);
132 lastResult_ = nullptr;
133 };
134
135 GstMapInfo info = GST_MAP_INFO_INIT;
136 gboolean ret = gst_buffer_map(lastResult_, &info, GST_MAP_READ);
137 CHECK_AND_RETURN_RET_LOG(ret, nullptr, "map failed, exit convert frame");
138
139 ON_SCOPE_EXIT(1) { gst_buffer_unmap(lastResult_, &info); };
140
141 GstVideoMeta *videoMeta = gst_buffer_get_video_meta(lastResult_);
142 CHECK_AND_RETURN_RET_LOG(videoMeta != nullptr, nullptr, "get video meta failed");
143
144 GstShMemMemory *mem = reinterpret_cast<GstShMemMemory *>(info.memory);
145 CHECK_AND_RETURN_RET_LOG(mem != nullptr, nullptr, "mem is nullptr");
146 CHECK_AND_RETURN_RET_LOG(mem->mem != nullptr, nullptr, "mem->mem is nullptr");
147 CHECK_AND_RETURN_RET_LOG(mem->mem->GetBase() != nullptr, nullptr, "addr is nullptr");
148
149 std::shared_ptr<AVSharedMemory> result = mem->mem;
150 if (!(result->GetSize() > 0 && static_cast<uint32_t>(result->GetSize()) >= sizeof(OutputFrame))) {
151 MEDIA_LOGE("size is incorrect");
152 return nullptr;
153 }
154
155 auto frame = reinterpret_cast<OutputFrame *>(result->GetBase());
156 frame->bytesPerPixel_ = PIXELFORMAT_INFO.at(outConfig_.colorFormat).bytesPerPixel;
157 frame->width_ = static_cast<int32_t>(videoMeta->width);
158 frame->height_ = static_cast<int32_t>(videoMeta->height);
159 frame->stride_ = videoMeta->stride[0];
160 frame->size_ = frame->stride_ * frame->height_;
161
162 CHECK_AND_RETURN_RET_LOG(result->GetSize() >= frame->GetFlattenedSize(), nullptr, "size is incorrect");
163
164 MEDIA_LOGI("======================Convert Frame Finished=========================");
165 MEDIA_LOGI("output width = %{public}d, stride = %{public}d, height = %{public}d, format = %{public}d",
166 frame->width_, frame->stride_, frame->height_, outConfig_.colorFormat);
167 return result;
168 }
169
UninstallPipeline()170 void AVMetaFrameConverter::UninstallPipeline()
171 {
172 if (pipeline_ != nullptr) {
173 ChangeState(GST_STATE_NULL);
174 gst_object_unref(pipeline_);
175 pipeline_ = nullptr;
176 }
177
178 if (pipeline_ != nullptr) {
179 gst_object_unref(pipeline_);
180 pipeline_ = nullptr;
181 }
182
183 if (vidShMemSink_ != nullptr) {
184 gst_object_unref(vidShMemSink_);
185 vidShMemSink_ = nullptr;
186 }
187
188 if (appSrc_ != nullptr) {
189 gst_object_unref(appSrc_);
190 appSrc_ = nullptr;
191 }
192
193 currState_ = GST_STATE_NULL;
194 }
195
Reset()196 int32_t AVMetaFrameConverter::Reset()
197 {
198 std::unique_lock<std::mutex> lock(mutex_);
199
200 /**
201 * Must flush before change state and delete the msgProcessor, otherwise deadlock will
202 * happened when try to destroy the msgprocessor.
203 */
204 auto tempMsgProc = std::move(msgProcessor_);
205 lock.unlock();
206 tempMsgProc->FlushBegin();
207 tempMsgProc->Reset();
208 tempMsgProc = nullptr;
209 lock.lock();
210
211 UninstallPipeline();
212
213 if (lastResult_ != nullptr) {
214 gst_buffer_unref(lastResult_);
215 lastResult_ = nullptr;
216 }
217
218 if (lastCaps_ != nullptr) {
219 gst_caps_unref(lastCaps_);
220 lastCaps_ = nullptr;
221 }
222
223 for (auto &sample : allResults_) {
224 gst_buffer_unref(sample);
225 }
226 allResults_.clear();
227
228 startConverting_ = false;
229 cond_.notify_all();
230
231 return MSERR_OK;
232 }
233
SetupConvPipeline()234 int32_t AVMetaFrameConverter::SetupConvPipeline()
235 {
236 pipeline_ = GST_PIPELINE_CAST(gst_pipeline_new("conv_pipeline"));
237 CHECK_AND_RETURN_RET(pipeline_ != nullptr, MSERR_INVALID_OPERATION);
238 GstBin *bin = GST_BIN_CAST(pipeline_);
239
240 appSrc_ = GST_ELEMENT_CAST(gst_object_ref(gst_element_factory_make("appsrc", "conv_src")));
241 CHECK_AND_RETURN_RET(appSrc_ != nullptr, MSERR_INVALID_OPERATION);
242 CHECK_AND_RETURN_RET(gst_bin_add(bin, appSrc_), MSERR_INVALID_OPERATION);
243
244 GstElement *scale = gst_element_factory_make("videoscale", "conv_scale");
245 CHECK_AND_RETURN_RET(scale != nullptr, MSERR_INVALID_OPERATION);
246 CHECK_AND_RETURN_RET(gst_bin_add(bin, scale), MSERR_INVALID_OPERATION);
247
248 GstElement *conv = gst_element_factory_make("videoconvert", "conv_conv");
249 CHECK_AND_RETURN_RET(conv != nullptr, MSERR_INVALID_OPERATION);
250 CHECK_AND_RETURN_RET(gst_bin_add(bin, conv), MSERR_INVALID_OPERATION);
251
252 vidShMemSink_ = GST_ELEMENT_CAST(gst_object_ref(gst_element_factory_make("sharedmemsink", "conv_sink")));
253 CHECK_AND_RETURN_RET(vidShMemSink_ != nullptr, MSERR_INVALID_OPERATION);
254 CHECK_AND_RETURN_RET(gst_bin_add(bin, vidShMemSink_), MSERR_INVALID_OPERATION);
255
256 gboolean ret = gst_element_link_pads_full(appSrc_, "src", scale, "sink", GST_PAD_LINK_CHECK_NOTHING);
257 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
258 ret = gst_element_link_pads_full(scale, "src", conv, "sink", GST_PAD_LINK_CHECK_NOTHING);
259 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
260 ret = gst_element_link_pads_full(conv, "src", vidShMemSink_, "sink", GST_PAD_LINK_CHECK_NOTHING);
261 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
262
263 MEDIA_LOGI("setup converter pipeline success");
264 return MSERR_OK;
265 }
266
SetupConvSrc()267 int32_t AVMetaFrameConverter::SetupConvSrc()
268 {
269 g_object_set(appSrc_, "is-live", TRUE, nullptr);
270 g_object_set(appSrc_, "stream-type", GST_APP_STREAM_TYPE_STREAM, nullptr);
271 g_object_set(appSrc_, "do-timestamp", TRUE, nullptr);
272 return MSERR_OK;
273 }
274
SetupConvSink(const OutputConfiguration & outConfig)275 int32_t AVMetaFrameConverter::SetupConvSink(const OutputConfiguration &outConfig)
276 {
277 CHECK_AND_RETURN_RET_LOG(PIXELFORMAT_INFO.count(outConfig.colorFormat) != 0, MSERR_INVALID_VAL,
278 "pixelformat unsupported: %{public}d", outConfig.colorFormat);
279
280 MEDIA_LOGI("target out config: width: %{public}d, height: %{public}d, format: %{public}d",
281 outConfig.dstWidth, outConfig.dstHeight, outConfig.colorFormat);
282
283 const char *formatStr = PIXELFORMAT_INFO.at(outConfig.colorFormat).gstVideoFormat.data();
284 GstStructure *struc = gst_structure_new("video/x-raw", "format", G_TYPE_STRING, formatStr, nullptr);
285 CHECK_AND_RETURN_RET(struc != nullptr, MSERR_NO_MEMORY);
286
287 if (outConfig.dstHeight != KEEP_ORIGINAL_WIDTH_OR_HEIGHT) {
288 gst_structure_set(struc, "height", G_TYPE_INT, outConfig.dstHeight, nullptr);
289 }
290
291 if (outConfig.dstWidth != KEEP_ORIGINAL_WIDTH_OR_HEIGHT) {
292 gst_structure_set(struc, "width", G_TYPE_INT, outConfig.dstWidth, nullptr);
293 }
294
295 GstCaps *caps = gst_caps_new_full(struc, nullptr);
296 CHECK_AND_RETURN_RET(caps != nullptr, MSERR_NO_MEMORY);
297
298 g_object_set(G_OBJECT(vidShMemSink_), "caps", caps, nullptr);
299 gst_caps_unref(caps);
300 caps = nullptr;
301 g_object_set(G_OBJECT(vidShMemSink_), "mem-prefix-size", sizeof(OutputFrame), nullptr);
302
303 GstMemSinkCallbacks callbacks = { nullptr, nullptr, OnNotifyNewSample };
304 gst_mem_sink_set_callback(GST_MEM_SINK_CAST(vidShMemSink_), &callbacks, this, nullptr);
305
306 outConfig_ = outConfig;
307 return MSERR_OK;
308 }
309
SetupMsgProcessor()310 int32_t AVMetaFrameConverter::SetupMsgProcessor()
311 {
312 GstBus *bus = gst_pipeline_get_bus(pipeline_);
313 CHECK_AND_RETURN_RET_LOG(bus != nullptr, MSERR_UNKNOWN, "can not get bus");
314
315 auto msgNotifier = std::bind(&AVMetaFrameConverter::OnNotifyMessage, this, std::placeholders::_1);
316 msgProcessor_ = std::make_unique<GstMsgProcessor>(*bus, msgNotifier);
317 gst_object_unref(bus);
318 bus = nullptr;
319
320 int32_t ret = msgProcessor_->Init();
321 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
322
323 // only concern the msg from pipeline
324 msgProcessor_->AddMsgFilter(ELEM_NAME(GST_ELEMENT_CAST(pipeline_)));
325
326 return MSERR_OK;
327 }
328
ChangeState(GstState targetState)329 int32_t AVMetaFrameConverter::ChangeState(GstState targetState)
330 {
331 MEDIA_LOGI("begin change state to %{public}s", gst_element_state_get_name(targetState));
332
333 GstStateChangeReturn stateRet = gst_element_set_state(GST_ELEMENT_CAST(pipeline_), targetState);
334 CHECK_AND_RETURN_RET_LOG(stateRet != GST_STATE_CHANGE_FAILURE, MSERR_INVALID_OPERATION,
335 "change conv pipeline to ready failed");
336
337 return MSERR_OK;
338 }
339
OnNotifyMessage(const InnerMessage & msg)340 void AVMetaFrameConverter::OnNotifyMessage(const InnerMessage &msg)
341 {
342 MEDIA_LOGD("msg: %{public}d", msg.type);
343 std::unique_lock<std::mutex> lock(mutex_);
344
345 switch (msg.type) {
346 case InnerMsgType::INNER_MSG_STATE_CHANGED: {
347 MEDIA_LOGD("state is %{public}s", gst_element_state_get_name(currState_));
348 currState_ = static_cast<GstState>(msg.detail2);
349 cond_.notify_all();
350 break;
351 }
352 case InnerMsgType::INNER_MSG_ERROR: {
353 startConverting_ = false;
354 MEDIA_LOGE("error happened");
355 cond_.notify_all();
356 break;
357 }
358 default:
359 break;
360 }
361 }
362
OnNotifyNewSample(GstMemSink * elem,GstBuffer * sample,gpointer userData)363 GstFlowReturn AVMetaFrameConverter::OnNotifyNewSample(GstMemSink *elem, GstBuffer *sample, gpointer userData)
364 {
365 CHECK_AND_RETURN_RET(sample != nullptr, GST_FLOW_ERROR);
366 ON_SCOPE_EXIT(0) { gst_buffer_unref(sample); };
367
368 CHECK_AND_RETURN_RET(userData != nullptr, GST_FLOW_ERROR);
369 CHECK_AND_RETURN_RET(elem != nullptr, GST_FLOW_ERROR);
370 auto thiz = reinterpret_cast<AVMetaFrameConverter *>(userData);
371
372 std::unique_lock<std::mutex> lock(thiz->mutex_);
373 if (thiz->lastResult_ != nullptr) {
374 gst_buffer_unref(thiz->lastResult_);
375 thiz->lastResult_ = nullptr;
376 }
377
378 MEDIA_LOGI("sample buffer arrived, pts: %{public}" PRIu64 "", GST_BUFFER_PTS(sample));
379
380 thiz->lastResult_ = gst_buffer_ref(sample);
381 CHECK_AND_RETURN_RET(thiz->lastResult_ != nullptr, GST_FLOW_ERROR);
382 // increase the refcount to avoid the buffer to be released to bufferpool.
383 thiz->allResults_.push_back(gst_buffer_ref(sample));
384
385 thiz->cond_.notify_all();
386 return GST_FLOW_OK;
387 }
388 } // namespace Media
389 } // namespace OHOS
390