1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmeta_frame_converter.h"
17 #include <gst/app/gstappsrc.h>
18 #include <gst/video/video-info.h>
19 #include "media_errors.h"
20 #include "media_log.h"
21 #include "gst_utils.h"
22 #include "gst_shmem_memory.h"
23 #include "scope_guard.h"
24 #include "time_perf.h"
25
26 namespace {
27 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaFrameConv"};
28 static constexpr int32_t KEEP_ORIGINAL_WIDTH_OR_HEIGHT = -1;
29 }
30
31 namespace OHOS {
32 namespace Media {
33 struct PixelFormatInfo {
34 PixelFormat format;
35 std::string_view gstVideoFormat;
36 uint8_t bytesPerPixel;
37 };
38
39 static const std::unordered_map<PixelFormat, PixelFormatInfo> PIXELFORMAT_INFO = {
40 { PixelFormat::RGB_565, { PixelFormat::RGB_565, "RGB16", 2 } },
41 { PixelFormat::RGB_888, { PixelFormat::RGB_888, "RGB", 3 } },
42 { PixelFormat::RGBA_8888, { PixelFormat::RGBA_8888, "RGBA", 4 } },
43 };
44
AVMetaFrameConverter()45 AVMetaFrameConverter::AVMetaFrameConverter()
46 {
47 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
48 }
49
~AVMetaFrameConverter()50 AVMetaFrameConverter::~AVMetaFrameConverter()
51 {
52 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
53 (void)Reset();
54 CLEAN_PERF_RECORD(this);
55 }
56
Init(const OutputConfiguration & config)57 int32_t AVMetaFrameConverter::Init(const OutputConfiguration &config)
58 {
59 std::unique_lock<std::mutex> lock(mutex_);
60
61 int32_t ret = SetupConvPipeline();
62 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
63
64 ret = SetupConvSrc();
65 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
66
67 ret = SetupConvSink(config);
68 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
69
70 ret = SetupMsgProcessor();
71 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
72
73 return MSERR_OK;
74 }
75
Convert(GstCaps & inCaps,GstBuffer & inBuf)76 std::shared_ptr<AVSharedMemory> AVMetaFrameConverter::Convert(GstCaps &inCaps, GstBuffer &inBuf)
77 {
78 AUTO_PERF(this, "ConvertFrame");
79
80 std::unique_lock<std::mutex> lock(mutex_);
81
82 int32_t ret = PrepareConvert(inCaps);
83 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "prepare convert failed");
84
85 GstFlowReturn flowRet = GST_FLOW_ERROR;
86 g_signal_emit_by_name(G_OBJECT(appSrc_), "push-buffer", &inBuf, &flowRet);
87 CHECK_AND_RETURN_RET_LOG(flowRet == GST_FLOW_OK, nullptr, "push buffer failed");
88
89 cond_.wait(lock, [this]() { return lastResult_ != nullptr || !startConverting_; });
90
91 return GetConvertResult();
92 }
93
PrepareConvert(GstCaps & inCaps)94 int32_t AVMetaFrameConverter::PrepareConvert(GstCaps &inCaps)
95 {
96 ON_SCOPE_EXIT(0) { (void)GetConvertResult(); };
97
98 if (lastCaps_ == nullptr || !gst_caps_is_equal(lastCaps_, &inCaps)) {
99 MEDIA_LOGI("caps changed");
100 MEDIA_LOGI("current caps: %{public}s", gst_caps_to_string(&inCaps));
101 int32_t ret = ChangeState(GST_STATE_READY);
102 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
103
104 g_object_set(G_OBJECT(appSrc_), "caps", &inCaps, nullptr);
105 if (lastCaps_ != nullptr) {
106 gst_caps_unref(lastCaps_);
107 }
108 lastCaps_ = gst_caps_ref(&inCaps);
109
110 ret = ChangeState(GST_STATE_PLAYING);
111 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
112 }
113
114 CANCEL_SCOPE_EXIT_GUARD(0);
115 startConverting_ = true;
116
117 return MSERR_OK;
118 }
119
GetConvertResult()120 std::shared_ptr<AVSharedMemory> AVMetaFrameConverter::GetConvertResult()
121 {
122 startConverting_ = false;
123
124 if (lastResult_ == nullptr) {
125 MEDIA_LOGE("last result frame is nullptr");
126 return nullptr;
127 }
128
129 ON_SCOPE_EXIT(0) {
130 gst_buffer_unref(lastResult_);
131 lastResult_ = nullptr;
132 };
133
134 GstMapInfo info = GST_MAP_INFO_INIT;
135 gboolean ret = gst_buffer_map(lastResult_, &info, GST_MAP_READ);
136 CHECK_AND_RETURN_RET_LOG(ret, nullptr, "map failed, exit convert frame");
137
138 ON_SCOPE_EXIT(1) { gst_buffer_unmap(lastResult_, &info); };
139
140 GstVideoMeta *videoMeta = gst_buffer_get_video_meta(lastResult_);
141 CHECK_AND_RETURN_RET_LOG(videoMeta != nullptr, nullptr, "get video meta failed");
142
143 GstShMemMemory *mem = reinterpret_cast<GstShMemMemory *>(info.memory);
144 CHECK_AND_RETURN_RET_LOG(mem != nullptr && mem->mem != nullptr, nullptr, "mem is nullptr");
145 CHECK_AND_RETURN_RET_LOG(mem->mem->GetBase() != nullptr, nullptr, "addr is nullptr");
146
147 std::shared_ptr<AVSharedMemory> result = mem->mem;
148 if (!(result->GetSize() > 0 && static_cast<uint32_t>(result->GetSize()) >= sizeof(OutputFrame))) {
149 MEDIA_LOGE("size is incorrect");
150 return nullptr;
151 }
152
153 auto frame = reinterpret_cast<OutputFrame *>(result->GetBase());
154 frame->bytesPerPixel_ = PIXELFORMAT_INFO.at(outConfig_.colorFormat).bytesPerPixel;
155 frame->width_ = static_cast<int32_t>(videoMeta->width);
156 frame->height_ = static_cast<int32_t>(videoMeta->height);
157 frame->stride_ = videoMeta->stride[0];
158 frame->size_ = frame->stride_ * frame->height_;
159
160 CHECK_AND_RETURN_RET_LOG(result->GetSize() >= frame->GetFlattenedSize(), nullptr, "size is incorrect");
161
162 MEDIA_LOGI("======================Convert Frame Finished=========================");
163 MEDIA_LOGI("output width = %{public}d, stride = %{public}d, height = %{public}d, format = %{public}d",
164 frame->width_, frame->stride_, frame->height_, outConfig_.colorFormat);
165 return result;
166 }
167
UninstallPipeline()168 void AVMetaFrameConverter::UninstallPipeline()
169 {
170 if (pipeline_ != nullptr) {
171 ChangeState(GST_STATE_NULL);
172 gst_object_unref(pipeline_);
173 pipeline_ = nullptr;
174 }
175
176 if (pipeline_ != nullptr) {
177 gst_object_unref(pipeline_);
178 pipeline_ = nullptr;
179 }
180
181 if (vidShMemSink_ != nullptr) {
182 gst_object_unref(vidShMemSink_);
183 vidShMemSink_ = nullptr;
184 }
185
186 if (appSrc_ != nullptr) {
187 gst_object_unref(appSrc_);
188 appSrc_ = nullptr;
189 }
190
191 currState_ = GST_STATE_NULL;
192 }
193
Reset()194 int32_t AVMetaFrameConverter::Reset()
195 {
196 std::unique_lock<std::mutex> lock(mutex_);
197
198 /**
199 * Must flush before change state and delete the msgProcessor, otherwise deadlock will
200 * happened when try to destroy the msgprocessor.
201 */
202 auto tempMsgProc = std::move(msgProcessor_);
203 lock.unlock();
204 tempMsgProc->FlushBegin();
205 tempMsgProc->Reset();
206 tempMsgProc = nullptr;
207 lock.lock();
208
209 UninstallPipeline();
210
211 if (lastResult_ != nullptr) {
212 gst_buffer_unref(lastResult_);
213 lastResult_ = nullptr;
214 }
215
216 if (lastCaps_ != nullptr) {
217 gst_caps_unref(lastCaps_);
218 lastCaps_ = nullptr;
219 }
220
221 for (auto &sample : allResults_) {
222 gst_buffer_unref(sample);
223 }
224 allResults_.clear();
225
226 startConverting_ = false;
227 cond_.notify_all();
228
229 return MSERR_OK;
230 }
231
SetupConvPipeline()232 int32_t AVMetaFrameConverter::SetupConvPipeline()
233 {
234 pipeline_ = GST_PIPELINE_CAST(gst_pipeline_new("conv_pipeline"));
235 CHECK_AND_RETURN_RET(pipeline_ != nullptr, MSERR_INVALID_OPERATION);
236 GstBin *bin = GST_BIN_CAST(pipeline_);
237
238 appSrc_ = GST_ELEMENT_CAST(gst_object_ref(gst_element_factory_make("appsrc", "conv_src")));
239 CHECK_AND_RETURN_RET(appSrc_ != nullptr, MSERR_INVALID_OPERATION);
240 CHECK_AND_RETURN_RET(gst_bin_add(bin, appSrc_), MSERR_INVALID_OPERATION);
241
242 GstElement *scale = gst_element_factory_make("videoscale", "conv_scale");
243 CHECK_AND_RETURN_RET(scale != nullptr, MSERR_INVALID_OPERATION);
244 CHECK_AND_RETURN_RET(gst_bin_add(bin, scale), MSERR_INVALID_OPERATION);
245
246 GstElement *conv = gst_element_factory_make("videoconvert", "conv_conv");
247 CHECK_AND_RETURN_RET(conv != nullptr, MSERR_INVALID_OPERATION);
248 CHECK_AND_RETURN_RET(gst_bin_add(bin, conv), MSERR_INVALID_OPERATION);
249
250 vidShMemSink_ = GST_ELEMENT_CAST(gst_object_ref(gst_element_factory_make("vidshmemsink", "conv_sink")));
251 CHECK_AND_RETURN_RET(vidShMemSink_ != nullptr, MSERR_INVALID_OPERATION);
252 CHECK_AND_RETURN_RET(gst_bin_add(bin, vidShMemSink_), MSERR_INVALID_OPERATION);
253
254 gboolean ret = gst_element_link_pads_full(appSrc_, "src", scale, "sink", GST_PAD_LINK_CHECK_NOTHING);
255 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
256 ret = gst_element_link_pads_full(scale, "src", conv, "sink", GST_PAD_LINK_CHECK_NOTHING);
257 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
258 ret = gst_element_link_pads_full(conv, "src", vidShMemSink_, "sink", GST_PAD_LINK_CHECK_NOTHING);
259 CHECK_AND_RETURN_RET(ret, MSERR_INVALID_OPERATION);
260
261 MEDIA_LOGI("setup converter pipeline success");
262 return MSERR_OK;
263 }
264
SetupConvSrc()265 int32_t AVMetaFrameConverter::SetupConvSrc()
266 {
267 g_object_set(appSrc_, "is-live", TRUE, nullptr);
268 g_object_set(appSrc_, "stream-type", GST_APP_STREAM_TYPE_STREAM, nullptr);
269 g_object_set(appSrc_, "do-timestamp", TRUE, nullptr);
270 return MSERR_OK;
271 }
272
SetupConvSink(const OutputConfiguration & outConfig)273 int32_t AVMetaFrameConverter::SetupConvSink(const OutputConfiguration &outConfig)
274 {
275 if (PIXELFORMAT_INFO.count(outConfig.colorFormat) == 0) {
276 MEDIA_LOGE("pixelformat unsupported: %{public}d", outConfig.colorFormat);
277 return MSERR_INVALID_VAL;
278 }
279
280 MEDIA_LOGI("target out config: width: %{public}d, height: %{public}d, format: %{public}d",
281 outConfig.dstWidth, outConfig.dstHeight, outConfig.colorFormat);
282
283 const char *formatStr = PIXELFORMAT_INFO.at(outConfig.colorFormat).gstVideoFormat.data();
284 GstStructure *struc = gst_structure_new("video/x-raw", "format", G_TYPE_STRING, formatStr, nullptr);
285 CHECK_AND_RETURN_RET(struc != nullptr, MSERR_NO_MEMORY);
286
287 if (outConfig.dstHeight != KEEP_ORIGINAL_WIDTH_OR_HEIGHT) {
288 gst_structure_set(struc, "height", G_TYPE_INT, outConfig.dstHeight, nullptr);
289 }
290
291 if (outConfig.dstWidth != KEEP_ORIGINAL_WIDTH_OR_HEIGHT) {
292 gst_structure_set(struc, "width", G_TYPE_INT, outConfig.dstWidth, nullptr);
293 }
294
295 GstCaps *caps = gst_caps_new_full(struc, nullptr);
296 CHECK_AND_RETURN_RET(caps != nullptr, MSERR_NO_MEMORY);
297
298 g_object_set(G_OBJECT(vidShMemSink_), "caps", caps, nullptr);
299 gst_caps_unref(caps);
300 caps = nullptr;
301
302 g_object_set(G_OBJECT(vidShMemSink_), "mem-prefix", sizeof(OutputFrame), nullptr);
303 (void)g_signal_connect(G_OBJECT(vidShMemSink_), "new-sample", G_CALLBACK(OnNotifyNewSample), this);
304
305 outConfig_ = outConfig;
306 return MSERR_OK;
307 }
308
SetupMsgProcessor()309 int32_t AVMetaFrameConverter::SetupMsgProcessor()
310 {
311 GstBus *bus = gst_pipeline_get_bus(pipeline_);
312 CHECK_AND_RETURN_RET_LOG(bus != nullptr, MSERR_UNKNOWN, "can not get bus");
313
314 auto msgNotifier = std::bind(&AVMetaFrameConverter::OnNotifyMessage, this, std::placeholders::_1);
315 msgProcessor_ = std::make_unique<GstMsgProcessor>(*bus, msgNotifier);
316 gst_object_unref(bus);
317 bus = nullptr;
318
319 int32_t ret = msgProcessor_->Init();
320 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
321
322 // only concern the msg from pipeline
323 msgProcessor_->AddMsgFilter(ELEM_NAME(GST_ELEMENT_CAST(pipeline_)));
324
325 return MSERR_OK;
326 }
327
ChangeState(GstState targetState)328 int32_t AVMetaFrameConverter::ChangeState(GstState targetState)
329 {
330 MEDIA_LOGI("begin change state to %{public}s", gst_element_state_get_name(targetState));
331
332 GstStateChangeReturn stateRet = gst_element_set_state(GST_ELEMENT_CAST(pipeline_), targetState);
333 if (stateRet == GST_STATE_CHANGE_FAILURE) {
334 MEDIA_LOGE("change conv pipeline to ready failed");
335 return MSERR_INVALID_OPERATION;
336 }
337
338 return MSERR_OK;
339 }
340
OnNotifyMessage(const InnerMessage & msg)341 void AVMetaFrameConverter::OnNotifyMessage(const InnerMessage &msg)
342 {
343 MEDIA_LOGD("msg: %{public}d", msg.type);
344 std::unique_lock<std::mutex> lock(mutex_);
345
346 switch (msg.type) {
347 case InnerMsgType::INNER_MSG_STATE_CHANGED: {
348 MEDIA_LOGD("state is %{public}s", gst_element_state_get_name(currState_));
349 currState_ = static_cast<GstState>(msg.detail2);
350 cond_.notify_all();
351 break;
352 }
353 case InnerMsgType::INNER_MSG_ERROR: {
354 startConverting_ = false;
355 MEDIA_LOGE("error happened");
356 cond_.notify_all();
357 break;
358 }
359 default:
360 break;
361 }
362 }
363
OnNotifyNewSample(GstElement * elem,AVMetaFrameConverter * thiz)364 GstFlowReturn AVMetaFrameConverter::OnNotifyNewSample(GstElement *elem, AVMetaFrameConverter *thiz)
365 {
366 CHECK_AND_RETURN_RET(thiz != nullptr, GST_FLOW_ERROR);
367 CHECK_AND_RETURN_RET(elem != nullptr, GST_FLOW_ERROR);
368
369 std::unique_lock<std::mutex> lock(thiz->mutex_);
370 if (thiz->lastResult_ != nullptr) {
371 gst_buffer_unref(thiz->lastResult_);
372 thiz->lastResult_ = nullptr;
373 }
374
375 GstSample *sample = nullptr;
376 g_signal_emit_by_name(elem, "pull-sample", &sample);
377 CHECK_AND_RETURN_RET(sample != nullptr, GST_FLOW_ERROR);
378
379 ON_SCOPE_EXIT(0) { gst_sample_unref(sample); };
380
381 GstBuffer *buffer = gst_sample_get_buffer(sample);
382 CHECK_AND_RETURN_RET(buffer != nullptr, GST_FLOW_ERROR);
383 MEDIA_LOGI("sample buffer arrived, pts: %{public}" PRIu64 "", GST_BUFFER_PTS(buffer));
384
385 thiz->lastResult_ = gst_buffer_ref(buffer);
386 CHECK_AND_RETURN_RET(thiz->lastResult_ != nullptr, GST_FLOW_ERROR);
387
388 // increase the refcount to avoid the buffer to be released to bufferpool.
389 thiz->allResults_.push_back(gst_buffer_ref(buffer));
390
391 thiz->cond_.notify_all();
392 return GST_FLOW_OK;
393 }
394 } // namespace Media
395 } // namespace OHOS
396