1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "gst_consumer_surface_pool.h"
17 #include "gst_consumer_surface_allocator.h"
18 #include "gst_consumer_surface_memory.h"
19 #include "buffer_type_meta.h"
20 #include "scope_guard.h"
21 #include "media_dfx.h"
22 #include "media_log.h"
23 #include "watchdog.h"
24 #include "param_wrapper.h"
25 using namespace OHOS;
26
27 #define gst_consumer_surface_pool_parent_class parent_class
28
29 GST_DEBUG_CATEGORY_STATIC(gst_consumer_surface_pool_debug_category);
30 #define GST_CAT_DEFAULT gst_consumer_surface_pool_debug_category
31
32 class PoolManager : public OHOS::Media::WatchDog, public NoCopyable {
33 public:
PoolManager(GstConsumerSurfacePool & owner,uint32_t timeoutMs)34 explicit PoolManager(GstConsumerSurfacePool &owner, uint32_t timeoutMs) : WatchDog(timeoutMs), owner_(owner) {}
35 ~PoolManager() = default;
36
37 void Alarm() override;
38 private:
39 GstConsumerSurfacePool &owner_;
40 };
41
42 struct _GstConsumerSurfacePoolPrivate {
43 sptr<Surface> consumer_surface;
44 guint available_buf_count;
45 GMutex pool_lock;
46 GCond buffer_available_con;
47 gboolean flushing;
48 gboolean start;
49 gboolean suspend;
50 gboolean is_first_buffer;
51 guint32 repeat_interval;
52 guint32 max_frame_rate;
53 guint64 pre_timestamp;
54 GstBuffer *cache_buffer;
55 gboolean need_eos_buffer;
56 gboolean is_first_buffer_in_for_trace;
57 gboolean pause_data;
58 FILE *dump_file;
59 GstElement *src;
60 std::shared_ptr<PoolManager> poolMgr;
61 };
62
63 enum {
64 PROP_0,
65 PROP_SUSPEND,
66 PROP_REPEAT,
67 PROP_MAX_FRAME_RATE,
68 PROP_NOTIFY_EOS,
69 PROP_PAUSE_DATA,
70 PROP_SRC,
71 PROP_INPUT_DETECTION,
72 };
73
74 G_DEFINE_TYPE_WITH_PRIVATE(GstConsumerSurfacePool, gst_consumer_surface_pool, GST_TYPE_VIDEO_BUFFER_POOL);
75
76 class ConsumerListenerProxy : public IBufferConsumerListener, public NoCopyable {
77 public:
ConsumerListenerProxy(GstConsumerSurfacePool & owner)78 explicit ConsumerListenerProxy(GstConsumerSurfacePool &owner) : owner_(owner) {}
79 ~ConsumerListenerProxy() = default;
80 void OnBufferAvailable() override;
81 private:
82 GstConsumerSurfacePool &owner_;
83 };
84
85 static void gst_consumer_surface_pool_set_property(GObject *object, guint id, const GValue *value, GParamSpec *pspec);
86 static void gst_consumer_surface_pool_init(GstConsumerSurfacePool *pool);
87 static void gst_consumer_surface_pool_buffer_available(GstConsumerSurfacePool *pool);
88 static void gst_consumer_surface_pool_notify_timeout(GstConsumerSurfacePool *pool);
89 static void gst_consumer_surface_pool_set_input_detection(GObject *object, bool enable);
90 static GstFlowReturn gst_consumer_surface_pool_acquire_buffer(GstBufferPool *pool, GstBuffer **buffer,
91 GstBufferPoolAcquireParams *params);
92 static void gst_consumer_surface_pool_release_buffer(GstBufferPool *pool, GstBuffer *buffer);
93 static gboolean gst_consumer_surface_pool_stop(GstBufferPool *pool);
94 static gboolean gst_consumer_surface_pool_start(GstBufferPool *pool);
95 static void gst_consumer_surface_pool_flush_start(GstBufferPool *pool);
96 static void gst_consumer_surface_pool_flush_stop(GstBufferPool *pool);
97 static void add_buffer_info(GstConsumerSurfacePool *pool, GstConsumerSurfaceMemory *mem, GstBuffer *buffer);
98 static void cache_frame_if_necessary(GstConsumerSurfacePool *pool, GstConsumerSurfaceMemory *mem, GstBuffer *buffer);
99 static gboolean drop_this_frame(GstConsumerSurfacePool *pool, guint64 new_timestamp,
100 guint64 old_timestamp, guint32 frame_rate);
101 static GstFlowReturn gst_consumer_surface_pool_get_surface_buffer(GstConsumerSurfacePool *pool,
102 sptr<SurfaceBuffer> &surface_buffer, gint32 &fencefd);
103 static void gst_consumer_surface_pool_release_surface_buffer(GstConsumerSurfacePool *pool,
104 sptr<SurfaceBuffer> &surface_buffer, gint32 &fencefd);
105 static void gst_consumer_surface_pool_get_dump_file(GstConsumerSurfacePool *pool);
106 static void gst_consumer_surface_pool_dump_surfacebuffer(GstConsumerSurfacePool *pool, sptr<SurfaceBuffer> &buffer);
107 static void gst_consumer_surface_pool_dump_gstbuffer(GstConsumerSurfacePool *pool, GstBuffer *buf);
108
OnBufferAvailable()109 void ConsumerListenerProxy::OnBufferAvailable()
110 {
111 gst_consumer_surface_pool_buffer_available(&owner_);
112 }
113
Alarm()114 void PoolManager::Alarm()
115 {
116 gst_consumer_surface_pool_notify_timeout(&owner_);
117 }
118
gst_consumer_surface_pool_get_options(GstBufferPool * pool)119 static const gchar **gst_consumer_surface_pool_get_options(GstBufferPool *pool)
120 {
121 (void)pool;
122 static const gchar *options[] = { GST_BUFFER_POOL_OPTION_VIDEO_META, nullptr };
123 return options;
124 }
125
gst_consumer_surface_pool_set_config(GstBufferPool * pool,GstStructure * config)126 static gboolean gst_consumer_surface_pool_set_config(GstBufferPool *pool, GstStructure *config)
127 {
128 g_return_val_if_fail(pool != nullptr, FALSE);
129 g_return_val_if_fail(config != nullptr, FALSE);
130
131 GstAllocator *allocator = nullptr;
132 (void)gst_buffer_pool_config_get_allocator(config, &allocator, nullptr);
133 if (!(allocator && GST_IS_CONSUMER_SURFACE_ALLOCATOR(allocator))) {
134 GST_WARNING_OBJECT(pool, "no valid allocator in pool");
135 return FALSE;
136 }
137
138 return GST_BUFFER_POOL_CLASS(parent_class)->set_config(pool, config);
139 }
140
141 // before unref must stop(deactive)
gst_consumer_surface_pool_finalize(GObject * obj)142 static void gst_consumer_surface_pool_finalize(GObject *obj)
143 {
144 g_return_if_fail(obj != nullptr);
145 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL_CAST(obj);
146 g_return_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr);
147 auto priv = surfacepool->priv;
148 priv->poolMgr = nullptr;
149 if (priv->consumer_surface != nullptr) {
150 if (priv->consumer_surface->UnregisterConsumerListener() != SURFACE_ERROR_OK) {
151 GST_WARNING_OBJECT(surfacepool, "deregister consumer listener fail");
152 }
153 priv->consumer_surface = nullptr;
154 }
155 g_mutex_clear(&priv->pool_lock);
156 g_cond_clear(&priv->buffer_available_con);
157 if (priv->dump_file) {
158 (void)fclose(priv->dump_file);
159 }
160 G_OBJECT_CLASS(parent_class)->finalize(obj);
161 }
162
gst_consumer_surface_pool_class_init(GstConsumerSurfacePoolClass * klass)163 static void gst_consumer_surface_pool_class_init(GstConsumerSurfacePoolClass *klass)
164 {
165 g_return_if_fail(klass != nullptr);
166 GstBufferPoolClass *poolClass = GST_BUFFER_POOL_CLASS(klass);
167 GObjectClass *gobjectClass = G_OBJECT_CLASS(klass);
168 GST_DEBUG_CATEGORY_INIT(gst_consumer_surface_pool_debug_category, "surfacepool", 0, "surface pool");
169 gobjectClass->set_property = gst_consumer_surface_pool_set_property;
170 gobjectClass->finalize = gst_consumer_surface_pool_finalize;
171
172 g_object_class_install_property(gobjectClass, PROP_SUSPEND,
173 g_param_spec_boolean("suspend", "Suspend surface", "Suspend surface",
174 FALSE, (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
175
176 g_object_class_install_property(gobjectClass, PROP_REPEAT,
177 g_param_spec_uint("repeat", "Repeat frame", "Repeat previous frame after given milliseconds",
178 0, G_MAXUINT32, 0, (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
179
180 g_object_class_install_property(gobjectClass, PROP_MAX_FRAME_RATE,
181 g_param_spec_uint("max-framerate", "Max frame rate", "Max frame rate",
182 0, G_MAXUINT32, 0, (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
183
184 g_object_class_install_property(gobjectClass, PROP_NOTIFY_EOS,
185 g_param_spec_boolean("notify-eos", "notify eos", "Need notify eos",
186 FALSE, (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
187
188 g_object_class_install_property(gobjectClass, PROP_PAUSE_DATA,
189 g_param_spec_boolean("pause-data", "pause data", "pause data",
190 FALSE, (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
191
192 g_object_class_install_property(gobjectClass, PROP_SRC,
193 g_param_spec_pointer("src", "Src plugin-in", "Src plugin-in",
194 (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
195
196 g_object_class_install_property(gobjectClass, PROP_INPUT_DETECTION,
197 g_param_spec_boolean("input-detection", "input-detection", "input-detection",
198 FALSE, (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
199
200 poolClass->get_options = gst_consumer_surface_pool_get_options;
201 poolClass->set_config = gst_consumer_surface_pool_set_config;
202 poolClass->release_buffer = gst_consumer_surface_pool_release_buffer;
203 poolClass->acquire_buffer = gst_consumer_surface_pool_acquire_buffer;
204 poolClass->start = gst_consumer_surface_pool_start;
205 poolClass->stop = gst_consumer_surface_pool_stop;
206 poolClass->flush_start = gst_consumer_surface_pool_flush_start;
207 poolClass->flush_stop = gst_consumer_surface_pool_flush_stop;
208 }
209
gst_consumer_surface_pool_set_property(GObject * object,guint id,const GValue * value,GParamSpec * pspec)210 static void gst_consumer_surface_pool_set_property(GObject *object, guint id, const GValue *value, GParamSpec *pspec)
211 {
212 (void)pspec;
213 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(object);
214 g_return_if_fail(surfacepool != nullptr && value != nullptr);
215 auto priv = surfacepool->priv;
216
217 g_mutex_lock(&priv->pool_lock);
218 ON_SCOPE_EXIT(0) { g_mutex_unlock(&priv->pool_lock); };
219
220 switch (id) {
221 case PROP_SUSPEND:
222 priv->suspend = g_value_get_boolean(value);
223 break;
224 case PROP_REPEAT:
225 if (g_value_get_uint(value) == 0 && priv->cache_buffer != nullptr) {
226 gst_buffer_unref(priv->cache_buffer);
227 priv->cache_buffer = nullptr;
228 }
229 priv->repeat_interval = g_value_get_uint(value) * 1000; // ms * 1000 = us
230 break;
231 case PROP_MAX_FRAME_RATE:
232 priv->max_frame_rate = g_value_get_uint(value);
233 break;
234 case PROP_NOTIFY_EOS:
235 priv->need_eos_buffer = g_value_get_boolean(value);
236 g_cond_signal(&priv->buffer_available_con);
237 break;
238 case PROP_PAUSE_DATA:
239 priv->pause_data = g_value_get_boolean(value);
240 g_cond_signal(&priv->buffer_available_con);
241 break;
242 case PROP_SRC:
243 priv->src = static_cast<GstElement *>(g_value_get_pointer(value));
244 break;
245 case PROP_INPUT_DETECTION:
246 gst_consumer_surface_pool_set_input_detection(object, g_value_get_boolean(value));
247 break;
248 default:
249 break;
250 }
251 }
252
gst_consumer_surface_pool_flush_start(GstBufferPool * pool)253 static void gst_consumer_surface_pool_flush_start(GstBufferPool *pool)
254 {
255 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
256 g_return_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr);
257 auto priv = surfacepool->priv;
258 g_mutex_lock(&priv->pool_lock);
259 if (priv->cache_buffer != nullptr) {
260 gst_buffer_unref(priv->cache_buffer);
261 priv->cache_buffer = nullptr;
262 }
263
264 // clear cache buffer
265 while (priv->available_buf_count > 0) {
266 sptr<SurfaceBuffer> buffer = nullptr;
267 gint32 fencefd = -1;
268 gint64 timestamp = 0;
269 Rect damage = {0, 0, 0, 0};
270 if (priv->consumer_surface->AcquireBuffer(buffer, fencefd, timestamp, damage) == SURFACE_ERROR_OK) {
271 gst_consumer_surface_pool_dump_surfacebuffer(surfacepool, buffer);
272 (void)priv->consumer_surface->ReleaseBuffer(buffer, fencefd);
273 }
274 priv->available_buf_count--;
275 GST_DEBUG_OBJECT(pool, "Release buffer on flush. Available buffer count %u", priv->available_buf_count);
276 }
277
278 surfacepool->priv->flushing = TRUE;
279 g_cond_signal(&priv->buffer_available_con);
280 g_mutex_unlock(&priv->pool_lock);
281 }
282
gst_consumer_surface_pool_flush_stop(GstBufferPool * pool)283 static void gst_consumer_surface_pool_flush_stop(GstBufferPool *pool)
284 {
285 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
286 g_return_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr);
287 auto priv = surfacepool->priv;
288 g_mutex_lock(&priv->pool_lock);
289 surfacepool->priv->flushing = FALSE;
290 surfacepool->priv->is_first_buffer = TRUE;
291 surfacepool->priv->is_first_buffer_in_for_trace = TRUE;
292 g_mutex_unlock(&priv->pool_lock);
293 }
294
295 // Disable pre-caching
gst_consumer_surface_pool_start(GstBufferPool * pool)296 static gboolean gst_consumer_surface_pool_start(GstBufferPool *pool)
297 {
298 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
299 g_return_val_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr, FALSE);
300 auto priv = surfacepool->priv;
301 g_mutex_lock(&priv->pool_lock);
302 surfacepool->priv->start = TRUE;
303 g_mutex_unlock(&priv->pool_lock);
304 return TRUE;
305 }
306
307 // Disable release buffers
gst_consumer_surface_pool_stop(GstBufferPool * pool)308 static gboolean gst_consumer_surface_pool_stop(GstBufferPool *pool)
309 {
310 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
311 g_return_val_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr, FALSE);
312 auto priv = surfacepool->priv;
313 g_mutex_lock(&priv->pool_lock);
314 surfacepool->priv->poolMgr = nullptr;
315 surfacepool->priv->start = FALSE;
316 g_cond_signal(&priv->buffer_available_con);
317 g_mutex_unlock(&priv->pool_lock);
318 return TRUE;
319 }
320
gst_consumer_surface_pool_release_buffer(GstBufferPool * pool,GstBuffer * buffer)321 static void gst_consumer_surface_pool_release_buffer(GstBufferPool *pool, GstBuffer *buffer)
322 {
323 g_return_if_fail(pool != nullptr && buffer != nullptr);
324 GstMemory *mem = gst_buffer_peek_memory(buffer, 0);
325 g_return_if_fail(mem != nullptr);
326 if (gst_is_consumer_surface_memory(mem)) {
327 GstBufferTypeMeta *meta = gst_buffer_get_buffer_type_meta(buffer);
328 if (meta != nullptr) {
329 GstConsumerSurfaceMemory *surfacemem = reinterpret_cast<GstConsumerSurfaceMemory *>(mem);
330 surfacemem->fencefd = meta->fenceFd;
331 }
332 }
333 // the buffer's pool is remove, the buffer will free by allocator.
334 gst_buffer_unref(buffer);
335 }
336
gst_consumer_surface_pool_get_eos_buffer(GstConsumerSurfacePool * surfacepool,GstBuffer ** buffer)337 static GstFlowReturn gst_consumer_surface_pool_get_eos_buffer(GstConsumerSurfacePool *surfacepool, GstBuffer **buffer)
338 {
339 g_return_val_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr, GST_FLOW_ERROR);
340 g_return_val_if_fail(buffer != nullptr, GST_FLOW_ERROR);
341 *buffer = gst_buffer_new();
342 g_return_val_if_fail(*buffer != nullptr, GST_FLOW_ERROR);
343 uint32_t bufferFlag = BUFFER_FLAG_EOS;
344 GstBufferHandleConfig config = { 0, -1, bufferFlag, 0, 0, 0, 0 };
345 gst_buffer_add_buffer_handle_meta(*buffer, 0, config);
346
347 surfacepool->priv->need_eos_buffer = FALSE;
348 return GST_FLOW_OK;
349 }
350
gst_consumer_surface_pool_get_repeat_buffer(GstConsumerSurfacePool * surfacepool,GstBuffer ** buffer)351 static void gst_consumer_surface_pool_get_repeat_buffer(GstConsumerSurfacePool *surfacepool, GstBuffer **buffer)
352 {
353 auto priv = surfacepool->priv;
354 *buffer = priv->cache_buffer;
355 gst_buffer_ref(priv->cache_buffer);
356 GST_BUFFER_PTS(*buffer) = priv->pre_timestamp + priv->repeat_interval;
357 priv->pre_timestamp = GST_BUFFER_PTS(*buffer);
358 }
359
gst_consumer_surface_pool_alloc_buffer(GstBufferPool * pool,GstBuffer ** buffer,GstBufferPoolAcquireParams * params,GstConsumerSurfaceMemory ** surfacemem)360 static GstFlowReturn gst_consumer_surface_pool_alloc_buffer(GstBufferPool *pool, GstBuffer **buffer,
361 GstBufferPoolAcquireParams *params, GstConsumerSurfaceMemory **surfacemem)
362 {
363 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
364 g_return_val_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr, GST_FLOW_ERROR);
365 GstBufferPoolClass *pclass = GST_BUFFER_POOL_GET_CLASS(pool);
366 g_return_val_if_fail(pclass != nullptr && pclass->alloc_buffer != nullptr, GST_FLOW_NOT_SUPPORTED);
367 g_return_val_if_fail(surfacemem != nullptr, GST_FLOW_ERROR);
368
369 if (surfacepool->find_buffer) {
370 bool found = false;
371 g_return_val_if_fail(surfacepool->find_buffer(pool, buffer, &found) == GST_FLOW_OK,
372 GST_FLOW_ERROR);
373 if (found) {
374 gst_consumer_surface_pool_dump_gstbuffer(surfacepool, *buffer);
375 return GST_FLOW_OK;
376 }
377 }
378
379 GstFlowReturn result = pclass->alloc_buffer(pool, buffer, params);
380 g_return_val_if_fail(result == GST_FLOW_OK && *buffer != nullptr, GST_FLOW_ERROR);
381 GstMemory *mem = gst_buffer_peek_memory(*buffer, 0);
382 g_return_val_if_fail(mem != nullptr, GST_FLOW_ERROR);
383 if (gst_is_consumer_surface_memory(mem)) {
384 *surfacemem = reinterpret_cast<GstConsumerSurfaceMemory*>(mem);
385 add_buffer_info(surfacepool, *surfacemem, *buffer);
386 }
387 gst_consumer_surface_pool_dump_gstbuffer(surfacepool, *buffer);
388 return GST_FLOW_OK;
389 }
390
gst_consumer_surface_pool_acquire_buffer(GstBufferPool * pool,GstBuffer ** buffer,GstBufferPoolAcquireParams * params)391 static GstFlowReturn gst_consumer_surface_pool_acquire_buffer(GstBufferPool *pool, GstBuffer **buffer,
392 GstBufferPoolAcquireParams *params)
393 {
394 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
395 g_return_val_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr, GST_FLOW_ERROR);
396 GstBufferPoolClass *pclass = GST_BUFFER_POOL_GET_CLASS(pool);
397 g_return_val_if_fail(pclass != nullptr && pclass->alloc_buffer != nullptr, GST_FLOW_NOT_SUPPORTED);
398 auto priv = surfacepool->priv;
399 g_mutex_lock(&priv->pool_lock);
400 ON_SCOPE_EXIT(0) { g_mutex_unlock(&priv->pool_lock); };
401
402 while (true) {
403 gboolean repeat = FALSE;
404 while (priv->available_buf_count == 0 && !priv->flushing && priv->start && !priv->need_eos_buffer) {
405 if (priv->repeat_interval == 0 || priv->cache_buffer == nullptr) {
406 g_cond_wait(&priv->buffer_available_con, &priv->pool_lock);
407 } else if (g_cond_wait_until(&priv->buffer_available_con, &priv->pool_lock, priv->repeat_interval)) {
408 repeat = TRUE;
409 break;
410 }
411 }
412
413 if (priv->pause_data && priv->start) {
414 g_cond_wait(&priv->buffer_available_con, &priv->pool_lock);
415 continue;
416 }
417
418 if (priv->flushing || !priv->start) {
419 return GST_FLOW_FLUSHING;
420 }
421 if (priv->available_buf_count == 0 && priv->need_eos_buffer) {
422 return gst_consumer_surface_pool_get_eos_buffer(surfacepool, buffer);
423 }
424
425 if (repeat && priv->cache_buffer != nullptr) {
426 gst_consumer_surface_pool_get_repeat_buffer(surfacepool, buffer);
427 break;
428 }
429
430 GstConsumerSurfaceMemory *surfacemem = nullptr;
431 GstFlowReturn result = gst_consumer_surface_pool_alloc_buffer(pool, buffer, params, &surfacemem);
432 g_return_val_if_fail(result == GST_FLOW_OK, result);
433
434 if (!repeat) {
435 priv->available_buf_count--;
436 }
437
438 // check whether needs to drop frame to ensure the maximum frame rate
439 if (surfacemem != nullptr && priv->max_frame_rate > 0 && !priv->is_first_buffer &&
440 drop_this_frame(surfacepool, surfacemem->timestamp, priv->pre_timestamp, priv->max_frame_rate)) {
441 (void)priv->consumer_surface->ReleaseBuffer(surfacemem->surface_buffer, surfacemem->fencefd);
442 if (!priv->flushing && priv->start) {
443 continue;
444 }
445 }
446 cache_frame_if_necessary(surfacepool, surfacemem, *buffer);
447 break;
448 };
449
450 return GST_FLOW_OK;
451 }
452
gst_consumer_surface_pool_init(GstConsumerSurfacePool * pool)453 static void gst_consumer_surface_pool_init(GstConsumerSurfacePool *pool)
454 {
455 g_return_if_fail(pool != nullptr);
456 auto priv = reinterpret_cast<GstConsumerSurfacePoolPrivate *>
457 (gst_consumer_surface_pool_get_instance_private(pool));
458 g_return_if_fail(priv != nullptr);
459 pool->priv = priv;
460 pool->buffer_available = nullptr;
461 pool->find_buffer = nullptr;
462 pool->get_surface_buffer = gst_consumer_surface_pool_get_surface_buffer;
463 pool->release_surface_buffer = gst_consumer_surface_pool_release_surface_buffer;
464 priv->available_buf_count = 0;
465 priv->flushing = FALSE;
466 priv->start = FALSE;
467 priv->suspend = FALSE;
468 priv->pause_data = FALSE;
469 priv->is_first_buffer = TRUE;
470 priv->is_first_buffer_in_for_trace = TRUE;
471 priv->repeat_interval = 0;
472 priv->max_frame_rate = 0;
473 priv->pre_timestamp = 0;
474 priv->cache_buffer = nullptr;
475 priv->need_eos_buffer = FALSE;
476 g_mutex_init(&priv->pool_lock);
477 g_cond_init(&priv->buffer_available_con);
478 priv->src = nullptr;
479 priv->poolMgr = nullptr;
480 priv->dump_file = nullptr;
481 gst_consumer_surface_pool_get_dump_file(pool);
482 }
483
gst_consumer_surface_pool_buffer_available(GstConsumerSurfacePool * pool)484 static void gst_consumer_surface_pool_buffer_available(GstConsumerSurfacePool *pool)
485 {
486 g_return_if_fail(pool != nullptr && pool->priv != nullptr);
487 auto priv = pool->priv;
488 g_mutex_lock(&priv->pool_lock);
489 ON_SCOPE_EXIT(0) { g_mutex_unlock(&priv->pool_lock); };
490
491 if (priv->poolMgr) {
492 priv->poolMgr->Notify();
493 }
494
495 if (priv->suspend) {
496 if (pool->buffer_available) {
497 bool releasebuffer = false;
498 if (pool->buffer_available(pool, &releasebuffer) != GST_FLOW_OK) {
499 GST_WARNING_OBJECT(pool, "Cache buffer failed.");
500 }
501
502 if (releasebuffer) {
503 GST_INFO_OBJECT(pool, "release buffer. Available buffer count %u", priv->available_buf_count);
504 return;
505 }
506 } else {
507 sptr<SurfaceBuffer> buffer = nullptr;
508 gint32 fencefd = -1;
509 gint64 timestamp = 0;
510 Rect damage = {0, 0, 0, 0};
511 if (priv->consumer_surface->AcquireBuffer(buffer, fencefd, timestamp, damage) == SURFACE_ERROR_OK) {
512 GST_INFO_OBJECT(pool, "Surface is suspended, release buffer. Available buffer count %u",
513 priv->available_buf_count);
514 gst_consumer_surface_pool_dump_surfacebuffer(pool, buffer);
515 (void)priv->consumer_surface->ReleaseBuffer(buffer, fencefd);
516 return;
517 }
518 }
519 }
520
521 if (priv->available_buf_count == 0) {
522 g_cond_signal(&priv->buffer_available_con);
523 }
524
525 if (priv->is_first_buffer_in_for_trace) {
526 OHOS::Media::MediaTrace::TraceBegin("AVCodecServer::FirstFrame",
527 FAKE_POINTER(priv->consumer_surface.GetRefPtr()));
528 priv->is_first_buffer_in_for_trace = FALSE;
529 }
530
531 pool->priv->available_buf_count++;
532 GST_DEBUG_OBJECT(pool, "Available buffer count %u", pool->priv->available_buf_count);
533 }
534
gst_consumer_surface_pool_set_surface(GstBufferPool * pool,sptr<Surface> & consumer_surface)535 void gst_consumer_surface_pool_set_surface(GstBufferPool *pool, sptr<Surface> &consumer_surface)
536 {
537 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool);
538 g_return_if_fail(surfacepool != nullptr && surfacepool->priv != nullptr);
539 g_return_if_fail(consumer_surface != nullptr && surfacepool->priv->consumer_surface == nullptr);
540 surfacepool->priv->consumer_surface = consumer_surface;
541 sptr<IBufferConsumerListener> listenerProxy = new (std::nothrow) ConsumerListenerProxy(*surfacepool);
542 g_return_if_fail(listenerProxy != nullptr);
543
544 if (consumer_surface->RegisterConsumerListener(listenerProxy) != SURFACE_ERROR_OK) {
545 GST_WARNING_OBJECT(surfacepool, "register consumer listener fail");
546 }
547 }
548
add_buffer_info(GstConsumerSurfacePool * pool,GstConsumerSurfaceMemory * mem,GstBuffer * buffer)549 static void add_buffer_info(GstConsumerSurfacePool *pool, GstConsumerSurfaceMemory *mem, GstBuffer *buffer)
550 {
551 g_return_if_fail(pool != nullptr && mem != nullptr && buffer != nullptr);
552 uint32_t bufferFlag = 0;
553 if (mem->is_eos_frame) {
554 bufferFlag = BUFFER_FLAG_EOS;
555 }
556 GstBufferHandleConfig config = { sizeof(*(mem->buffer_handle)), mem->fencefd,
557 bufferFlag, mem->data_size, mem->pixel_format, mem->width, mem->height };
558 gst_buffer_add_buffer_handle_meta(buffer, reinterpret_cast<intptr_t>(mem->buffer_handle), config);
559
560 if (mem->timestamp < 0) {
561 GST_WARNING_OBJECT(pool, "Invalid timestamp: < 0");
562 GST_BUFFER_PTS(buffer) = 0;
563 } else {
564 GST_BUFFER_PTS(buffer) = static_cast<uint64_t>(mem->timestamp);
565 }
566 }
567
cache_frame_if_necessary(GstConsumerSurfacePool * pool,GstConsumerSurfaceMemory * mem,GstBuffer * buffer)568 static void cache_frame_if_necessary(GstConsumerSurfacePool *pool, GstConsumerSurfaceMemory *mem, GstBuffer *buffer)
569 {
570 g_return_if_fail(pool != nullptr && pool->priv != nullptr && mem != nullptr && buffer != nullptr);
571 auto priv = pool->priv;
572 priv->pre_timestamp = static_cast<uint64_t>(mem->timestamp);
573 if (priv->is_first_buffer) {
574 priv->is_first_buffer = FALSE;
575 } else if (priv->repeat_interval > 0) {
576 if (priv->cache_buffer != nullptr) {
577 gst_buffer_unref(priv->cache_buffer);
578 }
579 priv->cache_buffer = buffer;
580 gst_buffer_ref(priv->cache_buffer);
581 }
582 }
583
drop_this_frame(GstConsumerSurfacePool * pool,guint64 new_timestamp,guint64 old_timestamp,guint32 frame_rate)584 static gboolean drop_this_frame(GstConsumerSurfacePool *pool, guint64 new_timestamp,
585 guint64 old_timestamp, guint32 frame_rate)
586 {
587 if (new_timestamp <= old_timestamp) {
588 GST_WARNING_OBJECT(pool, "Invalid timestamp: not increased");
589 return TRUE;
590 }
591
592 if (frame_rate == 0) {
593 GST_WARNING_OBJECT(pool, "Invalid frame rate: 0");
594 return FALSE;
595 }
596 guint64 min_interval = 1000000000 / frame_rate; // 1s = 1000000000ns
597 if ((UINT64_MAX - min_interval) < old_timestamp) {
598 GST_WARNING_OBJECT(pool, "Invalid timestamp: too big");
599 return TRUE;
600 }
601
602 const guint64 deviations = 3000000; // 3ms
603 if (new_timestamp < (old_timestamp - deviations + min_interval)) {
604 GST_INFO_OBJECT(pool, "Drop this frame to make sure maximum frame rate");
605 return TRUE;
606 }
607 return FALSE;
608 }
609
gst_consumer_surface_pool_new()610 GstBufferPool *gst_consumer_surface_pool_new()
611 {
612 GstBufferPool *pool = GST_BUFFER_POOL_CAST(g_object_new(
613 GST_TYPE_CONSUMER_SURFACE_POOL, "name", "consumer_surfacepool", nullptr));
614 (void)gst_object_ref_sink(pool);
615
616 return pool;
617 }
618
gst_consumer_surface_pool_set_input_detection(GObject * object,bool enable)619 static void gst_consumer_surface_pool_set_input_detection(GObject *object, bool enable)
620 {
621 GST_DEBUG_OBJECT(object, "set_input_detection enable = %d.", enable);
622 GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(object);
623 g_return_if_fail(surfacepool != nullptr);
624 auto priv = surfacepool->priv;
625 g_return_if_fail(priv != nullptr);
626
627 if (enable) {
628 if (priv->poolMgr == nullptr) {
629 const guint32 timeoutMs = 3000; // Error will be reported if there is no data input in 3000ms by default.
630 priv->poolMgr = std::make_shared<PoolManager>(*surfacepool, timeoutMs);
631 g_return_if_fail(priv->poolMgr != nullptr);
632 }
633
634 priv->poolMgr->EnableWatchDog();
635 } else {
636 if (priv->poolMgr) {
637 priv->poolMgr->DisableWatchDog();
638 }
639 }
640 }
641
gst_consumer_surface_pool_notify_timeout(GstConsumerSurfacePool * pool)642 static void gst_consumer_surface_pool_notify_timeout(GstConsumerSurfacePool *pool)
643 {
644 GST_DEBUG_OBJECT(pool, "Input stream timeout.");
645 g_return_if_fail(pool != nullptr && pool->priv != nullptr);
646 auto priv = pool->priv;
647
648 if (priv->src) {
649 GST_ELEMENT_ERROR (priv->src, RESOURCE, NOT_FOUND,
650 ("Input stream timeout, please confirm whether the input is normal."),
651 ("Input stream timeout, please confirm whether the input is normal."));
652 }
653 }
654
gst_consumer_surface_pool_get_surface_buffer(GstConsumerSurfacePool * pool,sptr<SurfaceBuffer> & surface_buffer,gint32 & fencefd)655 static GstFlowReturn gst_consumer_surface_pool_get_surface_buffer(GstConsumerSurfacePool *pool,
656 sptr<SurfaceBuffer> &surface_buffer, gint32 &fencefd)
657 {
658 g_return_val_if_fail(pool != nullptr && pool->priv != nullptr, GST_FLOW_ERROR);
659 auto priv = pool->priv;
660
661 gint64 timestamp = 0;
662 Rect damage = {0, 0, 0, 0};
663 if (priv->consumer_surface->AcquireBuffer(surface_buffer, fencefd, timestamp, damage) == SURFACE_ERROR_OK) {
664 return GST_FLOW_OK;
665 }
666
667 return GST_FLOW_ERROR;
668 }
669
gst_consumer_surface_pool_release_surface_buffer(GstConsumerSurfacePool * pool,sptr<SurfaceBuffer> & surface_buffer,gint32 & fencefd)670 static void gst_consumer_surface_pool_release_surface_buffer(GstConsumerSurfacePool *pool,
671 sptr<SurfaceBuffer> &surface_buffer, gint32 &fencefd)
672 {
673 g_return_if_fail(pool != nullptr && pool->priv != nullptr);
674 auto priv = pool->priv;
675
676 (void)priv->consumer_surface->ReleaseBuffer(surface_buffer, fencefd);
677 }
678
gst_consumer_surface_pool_get_dump_file(GstConsumerSurfacePool * pool)679 static void gst_consumer_surface_pool_get_dump_file(GstConsumerSurfacePool *pool)
680 {
681 g_return_if_fail(pool != nullptr && pool->priv != nullptr);
682 auto priv = pool->priv;
683
684 std::string dump_enable;
685 int32_t res = OHOS::system::GetStringParameter("sys.media.dump.surfacesrc.enable", dump_enable, "");
686 if (res != 0 || dump_enable.empty()) {
687 GST_DEBUG_OBJECT(pool, "sys.media.dump.surfacesrc.enable");
688 return;
689 }
690 GST_DEBUG_OBJECT(pool, "sys.media.dump.surfacesrc.enable=%s", dump_enable.c_str());
691
692 if (dump_enable == "true") {
693 std::string input_dump_file = "/data/media/surface_in" +
694 std::to_string(static_cast<int32_t>(FAKE_POINTER(pool))) + ".es_yuv";
695
696 priv->dump_file = fopen(input_dump_file.c_str(), "ab+");
697 if (priv->dump_file == nullptr) {
698 GST_ERROR_OBJECT(pool, "open file failed");
699 return;
700 }
701 }
702 }
703
gst_consumer_surface_pool_dump_data(FILE * dump_file,const void * addr,gint32 size,gint32 width,gint32 height)704 static void gst_consumer_surface_pool_dump_data(FILE *dump_file, const void *addr,
705 gint32 size, gint32 width, gint32 height)
706 {
707 g_return_if_fail(dump_file != nullptr && addr != nullptr);
708 gint32 data_size = size;
709
710 if (width != 0 && height != 0) {
711 // The size of non-es streams needs to be adjusted, only dump video data
712 gint32 rgbaSize = width * height * 4; // rgba = w * h * 4
713 gint32 yuvSize = width * height * 3 / 2; // yuv = w * h * 3 / 2
714 if (size > rgbaSize) {
715 data_size = rgbaSize;
716 } else if (size > yuvSize) {
717 data_size = yuvSize;
718 }
719 }
720
721 (void)fwrite(addr, data_size, 1, dump_file);
722 (void)fflush(dump_file);
723 return;
724 }
725
gst_consumer_surface_pool_dump_surfacebuffer(GstConsumerSurfacePool * pool,sptr<SurfaceBuffer> & buffer)726 static void gst_consumer_surface_pool_dump_surfacebuffer(GstConsumerSurfacePool *pool, sptr<SurfaceBuffer> &buffer)
727 {
728 g_return_if_fail(pool != nullptr && pool->priv != nullptr && buffer != nullptr);
729
730 if (pool->priv->dump_file == nullptr) {
731 return;
732 }
733
734 gint32 data_size = 0;
735 const sptr<OHOS::BufferExtraData>& extraData = buffer->GetExtraData();
736 if (extraData != nullptr) {
737 (void)extraData->ExtraGet("dataSize", data_size);
738 }
739
740 gst_consumer_surface_pool_dump_data(pool->priv->dump_file, buffer->GetVirAddr(),
741 data_size, buffer->GetWidth(), buffer->GetHeight());
742 return;
743 }
744
gst_consumer_surface_pool_dump_gstbuffer(GstConsumerSurfacePool * pool,GstBuffer * buf)745 static void gst_consumer_surface_pool_dump_gstbuffer(GstConsumerSurfacePool *pool, GstBuffer *buf)
746 {
747 g_return_if_fail(pool != nullptr && pool->priv != nullptr && buf != nullptr);
748
749 if (pool->priv->dump_file == nullptr) {
750 return;
751 }
752
753 GstBufferTypeMeta *meta = gst_buffer_get_buffer_type_meta(buf);
754 g_return_if_fail(meta != nullptr);
755 GstMapInfo info = GST_MAP_INFO_INIT;
756 gst_buffer_map(buf, &info, GST_MAP_READ);
757 gst_consumer_surface_pool_dump_data(pool->priv->dump_file, info.data,
758 static_cast<gint32>(meta->length), static_cast<gint32>(meta->width), static_cast<gint32>(meta->height));
759 gst_buffer_unmap(buf, &info);
760 return;
761 }