1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "config.h"
17 #include "gst_surface_video_src.h"
18 #include <gst/gst.h>
19 #include "media_errors.h"
20 #include "video_capture_factory.h"
21 #include "display_type.h"
22
23 static GstStaticPadTemplate gst_video_src_template =
24 GST_STATIC_PAD_TEMPLATE("src",
25 GST_PAD_SRC,
26 GST_PAD_ALWAYS,
27 GST_STATIC_CAPS_ANY);
28
29 namespace {
30 constexpr VideoStreamType DEFAULT_STREAM_TYPE = VIDEO_STREAM_TYPE_UNKNOWN;
31 constexpr gint DEFAULT_FRAME_RATE = 30;
32 }
33
34 enum {
35 PROP_0,
36 PROP_STREAM_TYPE,
37 PROP_SURFACE_WIDTH,
38 PROP_SURFACE_HEIGHT,
39 PROP_SURFACE,
40 PROP_FRAME_RATE,
41 };
42
43 using namespace OHOS::Media;
44
45 #define gst_surface_video_src_parent_class parent_class
46 G_DEFINE_TYPE(GstSurfaceVideoSrc, gst_surface_video_src, GST_TYPE_PUSH_SRC);
47
48 static void gst_surface_video_src_finalize(GObject *object);
49 static void gst_surface_video_src_set_property(GObject *object, guint prop_id,
50 const GValue *value, GParamSpec *pspec);
51 static void gst_surface_video_src_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec);
52 static GstStateChangeReturn gst_surface_video_src_change_state(GstElement *element, GstStateChange transition);
53 static GstFlowReturn gst_surface_video_src_create(GstPushSrc *psrc, GstBuffer **outbuf);
54 static void gst_surface_video_src_set_stream_type(GstSurfaceVideoSrc *src, gint stream_type);
55 static gboolean gst_surface_video_src_negotiate(GstBaseSrc *basesrc);
56 static gboolean gst_surface_video_src_is_seekable(GstBaseSrc *basesrc);
57 static gboolean gst_surface_video_src_send_event(GstElement *element, GstEvent *event);
58
59 #define GST_TYPE_SURFACE_VIDEO_SRC_STREAM_TYPE (gst_surface_video_src_stream_type_get_type())
gst_surface_video_src_stream_type_get_type(void)60 static GType gst_surface_video_src_stream_type_get_type(void)
61 {
62 static GType surface_video_src_stream_type = 0;
63 static const GEnumValue stream_types[] = {
64 {VIDEO_STREAM_TYPE_UNKNOWN, "UNKNOWN", "UNKNOWN"},
65 {VIDEO_STREAM_TYPE_ES_AVC, "ES_AVC", "ES_AVC"},
66 {VIDEO_STREAM_TYPE_YUV_420, "YUV_420", "YUV_420"},
67 {0, nullptr, nullptr}
68 };
69 if (!surface_video_src_stream_type) {
70 surface_video_src_stream_type = g_enum_register_static("VideoStreamType", stream_types);
71 }
72 return surface_video_src_stream_type;
73 }
74
gst_surface_video_src_class_init(GstSurfaceVideoSrcClass * klass)75 static void gst_surface_video_src_class_init(GstSurfaceVideoSrcClass *klass)
76 {
77 g_return_if_fail(klass != nullptr);
78 GObjectClass *gobject_class = reinterpret_cast<GObjectClass *>(klass);
79 GstElementClass *gstelement_class = reinterpret_cast<GstElementClass *>(klass);
80 GstBaseSrcClass *gstbasesrc_class = reinterpret_cast<GstBaseSrcClass *>(klass);
81 GstPushSrcClass *gstpushsrc_class = reinterpret_cast<GstPushSrcClass *>(klass);
82
83 gobject_class->finalize = gst_surface_video_src_finalize;
84 gobject_class->set_property = gst_surface_video_src_set_property;
85 gobject_class->get_property = gst_surface_video_src_get_property;
86
87 g_object_class_install_property(gobject_class, PROP_STREAM_TYPE,
88 g_param_spec_enum("stream-type", "Stream type",
89 "Stream type", GST_TYPE_SURFACE_VIDEO_SRC_STREAM_TYPE, DEFAULT_STREAM_TYPE,
90 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
91
92 g_object_class_install_property(gobject_class, PROP_SURFACE_WIDTH,
93 g_param_spec_uint("surface-width", "Surface width",
94 "Surface width", 0, G_MAXUINT32, 0,
95 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
96
97 g_object_class_install_property(gobject_class, PROP_SURFACE_HEIGHT,
98 g_param_spec_uint("surface-height", "Surface height",
99 "Surface width", 0, G_MAXUINT32, 0,
100 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
101
102 g_object_class_install_property(gobject_class, PROP_SURFACE,
103 g_param_spec_pointer("surface", "Surface", "Surface for recording",
104 (GParamFlags)(G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
105
106 g_object_class_install_property(gobject_class, PROP_FRAME_RATE,
107 g_param_spec_uint("frame-rate", "Frame rate",
108 "recorder frame rate", 0, G_MAXUINT32, DEFAULT_FRAME_RATE,
109 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
110
111 gst_element_class_set_static_metadata(gstelement_class,
112 "Surface video source", "Source/Video",
113 "Retrieve video frame from surface buffer queue", "OpenHarmony");
114
115 gst_element_class_add_static_pad_template(gstelement_class, &gst_video_src_template);
116
117 gstelement_class->change_state = gst_surface_video_src_change_state;
118 gstelement_class->send_event = gst_surface_video_src_send_event;
119 gstbasesrc_class->negotiate = gst_surface_video_src_negotiate;
120 gstbasesrc_class->is_seekable = gst_surface_video_src_is_seekable;
121 gstpushsrc_class->create = gst_surface_video_src_create;
122 }
123
gst_surface_video_src_init(GstSurfaceVideoSrc * src)124 static void gst_surface_video_src_init(GstSurfaceVideoSrc *src)
125 {
126 g_return_if_fail(src != nullptr);
127 gst_base_src_set_format(GST_BASE_SRC(src), GST_FORMAT_TIME);
128 gst_base_src_set_live(GST_BASE_SRC(src), TRUE);
129 src->stream_type = VIDEO_STREAM_TYPE_UNKNOWN;
130 src->capture = nullptr;
131 src->src_caps = nullptr;
132 src->video_width = 0;
133 src->video_height = 0;
134 src->video_frame_rate = DEFAULT_FRAME_RATE;
135 src->is_start = FALSE;
136 src->need_codec_data = TRUE;
137 src->is_eos = FALSE;
138 src->is_flushing = FALSE;
139 src->reset_caps = TRUE;
140 }
141
gst_surface_video_src_finalize(GObject * object)142 static void gst_surface_video_src_finalize(GObject *object)
143 {
144 g_return_if_fail(object != nullptr);
145 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(object);
146 g_return_if_fail(src != nullptr);
147
148 src->need_codec_data = FALSE;
149
150 if (src->src_caps != nullptr) {
151 gst_caps_unref(src->src_caps);
152 src->src_caps = nullptr;
153 }
154 }
155
gst_surface_video_src_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)156 static void gst_surface_video_src_set_property(GObject *object, guint prop_id,
157 const GValue *value, GParamSpec *pspec)
158 {
159 (void)pspec;
160 g_return_if_fail(object != nullptr);
161 g_return_if_fail(value != nullptr);
162 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(object);
163 g_return_if_fail(src != nullptr);
164 switch (prop_id) {
165 case PROP_STREAM_TYPE:
166 gst_surface_video_src_set_stream_type(src, g_value_get_enum(value));
167 break;
168 case PROP_SURFACE_WIDTH:
169 src->video_width = g_value_get_uint(value);
170 break;
171 case PROP_SURFACE_HEIGHT:
172 src->video_height = g_value_get_uint(value);
173 break;
174 case PROP_FRAME_RATE:
175 src->video_frame_rate = g_value_get_uint(value);
176 break;
177 default:
178 break;
179 }
180 }
181
gst_surface_video_src_set_stream_type(GstSurfaceVideoSrc * src,gint stream_type)182 static void gst_surface_video_src_set_stream_type(GstSurfaceVideoSrc *src, gint stream_type)
183 {
184 switch (stream_type) {
185 case VideoStreamType::VIDEO_STREAM_TYPE_ES_AVC:
186 src->stream_type = VIDEO_STREAM_TYPE_ES_AVC;
187 src->need_codec_data = TRUE;
188 break;
189 case VideoStreamType::VIDEO_STREAM_TYPE_YUV_420:
190 src->stream_type = VIDEO_STREAM_TYPE_YUV_420;
191 src->need_codec_data = FALSE;
192 break;
193 default:
194 return;
195 }
196 g_return_if_fail(src != nullptr);
197 }
198
gst_surface_video_src_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)199 static void gst_surface_video_src_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec)
200 {
201 g_return_if_fail(object != nullptr);
202 g_return_if_fail(value != nullptr);
203 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(object);
204 g_return_if_fail(src != nullptr);
205 (void)pspec;
206 switch (prop_id) {
207 case PROP_STREAM_TYPE:
208 g_value_set_enum(value, src->stream_type);
209 break;
210 case PROP_SURFACE_WIDTH:
211 g_value_set_uint(value, src->video_width);
212 break;
213 case PROP_SURFACE_HEIGHT:
214 g_value_set_uint(value, src->video_height);
215 break;
216 case PROP_FRAME_RATE:
217 g_value_set_uint(value, src->video_frame_rate);
218 break;
219 case PROP_SURFACE:
220 g_return_if_fail(src->capture != nullptr);
221 g_value_set_pointer(value, src->capture->GetSurface().GetRefPtr());
222 break;
223 default:
224 break;
225 }
226 }
227
process_codec_data(GstSurfaceVideoSrc * src)228 static gboolean process_codec_data(GstSurfaceVideoSrc *src)
229 {
230 g_return_val_if_fail(src != nullptr, FALSE);
231 g_return_val_if_fail(src->capture != nullptr, FALSE);
232
233 std::shared_ptr<EsAvcCodecBuffer> codec_buffer = src->capture->GetCodecBuffer();
234 g_return_val_if_fail(codec_buffer != nullptr, FALSE);
235
236 if (codec_buffer->gstCodecBuffer == nullptr) {
237 return FALSE;
238 }
239
240 if (src->src_caps != nullptr) {
241 gst_caps_unref(src->src_caps);
242 }
243
244 src->src_caps = gst_caps_new_simple("video/x-h264",
245 "width", G_TYPE_INT, codec_buffer->width,
246 "height", G_TYPE_INT, codec_buffer->height,
247 "framerate", GST_TYPE_FRACTION, src->video_frame_rate, 1,
248 "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
249 "level", G_TYPE_STRING, "2",
250 "profile", G_TYPE_STRING, "high",
251 "alignment", G_TYPE_STRING, "au",
252 "stream-format", G_TYPE_STRING, "avc", nullptr);
253 gst_caps_set_simple(src->src_caps, "codec_data", GST_TYPE_BUFFER, codec_buffer->gstCodecBuffer, nullptr);
254 GstBaseSrc *basesrc = GST_BASE_SRC_CAST(src);
255 basesrc->segment.start = codec_buffer->segmentStart;
256 gst_buffer_unref(codec_buffer->gstCodecBuffer);
257 codec_buffer->gstCodecBuffer = nullptr;
258 return TRUE;
259 }
260
set_fix_caps(GstSurfaceVideoSrc * src)261 static gboolean set_fix_caps(GstSurfaceVideoSrc *src)
262 {
263 g_return_val_if_fail(src != nullptr, FALSE);
264 g_return_val_if_fail(src->capture != nullptr, FALSE);
265
266 if (src->src_caps != nullptr) {
267 gst_caps_unref(src->src_caps);
268 }
269
270 src->src_caps = gst_caps_new_simple("video/x-raw",
271 "format", G_TYPE_STRING, "I420",
272 "framerate", GST_TYPE_FRACTION, src->video_frame_rate, 1,
273 "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
274 "width", G_TYPE_INT, src->video_width,
275 "height", G_TYPE_INT, src->video_height,
276 nullptr);
277
278 return TRUE;
279 }
280
start_video_capture(GstSurfaceVideoSrc * src)281 static gboolean start_video_capture(GstSurfaceVideoSrc *src)
282 {
283 g_return_val_if_fail(src != nullptr, FALSE);
284 g_return_val_if_fail(src->capture != nullptr, FALSE);
285
286 g_return_val_if_fail(src->capture->Start() == MSERR_OK, FALSE);
287 if (src->need_codec_data) {
288 gboolean ret = process_codec_data(src);
289 return ret;
290 }
291 return TRUE;
292 }
293
gst_state_change_forward_direction(GstSurfaceVideoSrc * src,GstStateChange transition)294 static GstStateChangeReturn gst_state_change_forward_direction(GstSurfaceVideoSrc *src, GstStateChange transition)
295 {
296 switch (transition) {
297 case GST_STATE_CHANGE_NULL_TO_READY:
298 src->capture = OHOS::Media::VideoCaptureFactory::CreateVideoCapture(src->stream_type);
299 g_return_val_if_fail(src->capture != nullptr, GST_STATE_CHANGE_FAILURE);
300 if (src->need_codec_data != TRUE) {
301 g_return_val_if_fail(set_fix_caps(src) == TRUE, GST_STATE_CHANGE_FAILURE);
302 }
303 break;
304 case GST_STATE_CHANGE_READY_TO_PAUSED:
305 g_return_val_if_fail(src->capture != nullptr, GST_STATE_CHANGE_FAILURE);
306 g_return_val_if_fail(src->capture->SetVideoHeight(src->video_height) == MSERR_OK,
307 GST_STATE_CHANGE_FAILURE);
308 g_return_val_if_fail(src->capture->SetVideoWidth(src->video_width) == MSERR_OK,
309 GST_STATE_CHANGE_FAILURE);
310 g_return_val_if_fail(src->capture->SetFrameRate(src->video_frame_rate) == MSERR_OK,
311 GST_STATE_CHANGE_FAILURE);
312 g_return_val_if_fail(src->capture->SetStreamType(src->stream_type) == MSERR_OK,
313 GST_STATE_CHANGE_FAILURE);
314 g_return_val_if_fail(src->capture->Prepare() == MSERR_OK, GST_STATE_CHANGE_FAILURE);
315 break;
316 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
317 g_return_val_if_fail(src->capture != nullptr, GST_STATE_CHANGE_FAILURE);
318 if (src->is_start == FALSE) {
319 g_return_val_if_fail(start_video_capture(src) == TRUE, GST_STATE_CHANGE_FAILURE);
320 src->is_start = TRUE;
321 } else {
322 g_return_val_if_fail(src->capture->Resume() == MSERR_OK, GST_STATE_CHANGE_FAILURE);
323 }
324 break;
325 default:
326 break;
327 }
328 return GST_STATE_CHANGE_SUCCESS;
329 }
330
gst_surface_video_src_change_state(GstElement * element,GstStateChange transition)331 static GstStateChangeReturn gst_surface_video_src_change_state(GstElement *element, GstStateChange transition)
332 {
333 g_return_val_if_fail(element != nullptr, GST_STATE_CHANGE_FAILURE);
334 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(element);
335 g_return_val_if_fail(src != nullptr, GST_STATE_CHANGE_FAILURE);
336
337 GstStateChangeReturn ret = gst_state_change_forward_direction(src, transition);
338 g_return_val_if_fail(ret == GST_STATE_CHANGE_SUCCESS, GST_STATE_CHANGE_FAILURE);
339
340 ret = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
341
342 switch (transition) {
343 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
344 g_return_val_if_fail(src->capture->Pause() == MSERR_OK, GST_STATE_CHANGE_FAILURE);
345 break;
346 case GST_STATE_CHANGE_PAUSED_TO_READY:
347 src->is_start = FALSE;
348 g_return_val_if_fail(src->capture != nullptr, GST_STATE_CHANGE_FAILURE);
349 g_return_val_if_fail(src->capture->Stop() == MSERR_OK, GST_STATE_CHANGE_FAILURE);
350 break;
351 case GST_STATE_CHANGE_READY_TO_NULL:
352 src->capture = nullptr;
353 break;
354 default:
355 break;
356 }
357
358 return ret;
359 }
360
gst_surface_video_src_is_seekable(GstBaseSrc * basesrc)361 static gboolean gst_surface_video_src_is_seekable(GstBaseSrc *basesrc)
362 {
363 (void)basesrc;
364 return FALSE;
365 }
366
gst_surface_video_src_negotiate(GstBaseSrc * basesrc)367 static gboolean gst_surface_video_src_negotiate(GstBaseSrc *basesrc)
368 {
369 g_return_val_if_fail(basesrc != nullptr, FALSE);
370 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(basesrc);
371 g_return_val_if_fail(src != nullptr, FALSE);
372
373 // no need to wait playing when yuv source
374 if (src->need_codec_data) {
375 (void)gst_base_src_wait_playing(basesrc);
376 }
377 g_return_val_if_fail(src->src_caps != nullptr, FALSE);
378 return gst_base_src_set_caps(basesrc, src->src_caps);
379 }
380
reset_src_caps(GstSurfaceVideoSrc * src,uint32_t pixelFormat)381 static gboolean reset_src_caps(GstSurfaceVideoSrc *src, uint32_t pixelFormat)
382 {
383 g_return_val_if_fail(src != nullptr, FALSE);
384
385 if (src->src_caps != nullptr) {
386 gst_caps_unref(src->src_caps);
387 }
388
389 std::string format = "NV21";
390
391 switch (pixelFormat) {
392 case PIXEL_FMT_YCRCB_420_SP:
393 GST_INFO("input pixel foramt is nv21");
394 format = "NV21";
395 break;
396 case PIXEL_FMT_YCBCR_420_P:
397 GST_INFO("input pixel foramt is I420");
398 format = "I420";
399 return TRUE;
400 case PIXEL_FMT_YCBCR_420_SP:
401 GST_INFO("input pixel foramt is nv12");
402 format = "NV12";
403 break;
404 default:
405 break;
406 }
407
408 src->src_caps = gst_caps_new_simple("video/x-raw",
409 "format", G_TYPE_STRING, format.c_str(),
410 "framerate", GST_TYPE_FRACTION, src->video_frame_rate, 1,
411 "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
412 "width", G_TYPE_INT, src->video_width,
413 "height", G_TYPE_INT, src->video_height,
414 nullptr);
415
416 gst_base_src_set_caps(GST_BASE_SRC(src), src->src_caps);
417 return TRUE;
418 }
419
gst_surface_video_src_create(GstPushSrc * psrc,GstBuffer ** outbuf)420 static GstFlowReturn gst_surface_video_src_create(GstPushSrc *psrc, GstBuffer **outbuf)
421 {
422 g_return_val_if_fail(psrc != nullptr, GST_FLOW_ERROR);
423 g_return_val_if_fail(outbuf != nullptr, GST_FLOW_ERROR);
424 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(psrc);
425 g_return_val_if_fail(src != nullptr, GST_FLOW_ERROR);
426
427 GST_DEBUG_OBJECT(src, "begin create...");
428
429 if (src->is_start == FALSE) {
430 return GST_FLOW_EOS;
431 }
432 g_return_val_if_fail(src->capture != nullptr, GST_FLOW_ERROR);
433
434 std::shared_ptr<VideoFrameBuffer> frame_buffer = src->capture->GetFrameBuffer();
435 if (src->is_eos) {
436 GST_INFO_OBJECT(src, "eos...");
437 return GST_FLOW_EOS;
438 }
439 if (src->is_flushing) {
440 GST_INFO_OBJECT(src, "flushing...");
441 return GST_FLOW_FLUSHING;
442 }
443 g_return_val_if_fail(frame_buffer != nullptr, GST_FLOW_ERROR);
444
445 if (src->reset_caps && !src->need_codec_data) {
446 (void)reset_src_caps(src, frame_buffer->pixelFormat);
447 src->reset_caps = FALSE;
448 }
449
450 gst_base_src_set_blocksize(GST_BASE_SRC_CAST(src), static_cast<guint>(frame_buffer->size));
451
452 *outbuf = frame_buffer->gstBuffer;
453 GST_BUFFER_PTS(*outbuf) = frame_buffer->timeStamp;
454
455 GST_DEBUG_OBJECT(src, "end create...");
456 return GST_FLOW_OK;
457 }
458
gst_surface_video_src_send_event(GstElement * element,GstEvent * event)459 static gboolean gst_surface_video_src_send_event(GstElement *element, GstEvent *event)
460 {
461 g_return_val_if_fail(element != nullptr, FALSE);
462 GstSurfaceVideoSrc *src = GST_SURFACE_VIDEO_SRC(element);
463 g_return_val_if_fail(src != nullptr, FALSE);
464
465 switch (GST_EVENT_TYPE (event)) {
466 case GST_EVENT_FLUSH_START:
467 g_return_val_if_fail(src->capture != nullptr, FALSE);
468 src->is_eos = FALSE;
469 src->is_flushing = TRUE;
470 src->capture->UnLock(true);
471 break;
472 case GST_EVENT_FLUSH_STOP:
473 g_return_val_if_fail(src->capture != nullptr, FALSE);
474 src->is_flushing = FALSE;
475 src->capture->UnLock(false);
476 break;
477 case GST_EVENT_EOS:
478 g_return_val_if_fail(src->capture != nullptr, FALSE);
479 src->is_eos = TRUE;
480 src->capture->UnLock(true);
481 break;
482 default:
483 break;
484 }
485
486 return GST_ELEMENT_CLASS(parent_class)->send_event(element, event);
487 }
488
plugin_init(GstPlugin * plugin)489 static gboolean plugin_init(GstPlugin *plugin)
490 {
491 g_return_val_if_fail(plugin != nullptr, FALSE);
492 return gst_element_register(plugin, "surfacevideosrc", GST_RANK_PRIMARY, GST_TYPE_SURFACE_VIDEO_SRC);
493 }
494
495 GST_PLUGIN_DEFINE(GST_VERSION_MAJOR,
496 GST_VERSION_MINOR,
497 _surface_video_src,
498 "GStreamer Surface Video Source",
499 plugin_init,
500 PACKAGE_VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)
501