1 /* GStreamer
2 * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 *
19 * NOTE: some of implementations are copied/modified from Chromium code
20 *
21 * Copyright 2015 The Chromium Authors. All rights reserved.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions are
25 * met:
26 *
27 * * Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * * Redistributions in binary form must reproduce the above
30 * copyright notice, this list of conditions and the following disclaimer
31 * in the documentation and/or other materials provided with the
32 * distribution.
33 * * Neither the name of Google Inc. nor the names of its
34 * contributors may be used to endorse or promote products derived from
35 * this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 */
49
50 /**
51 * SECTION:element-d3d11h264dec
52 * @title: d3d11h264dec
53 *
54 * A Direct3D11/DXVA based H.264 video decoder
55 *
56 * ## Example launch line
57 * ```
58 * gst-launch-1.0 filesrc location=/path/to/h264/file ! parsebin ! d3d11h264dec ! d3d11videosink
59 * ```
60 *
61 * Since: 1.18
62 *
63 */
64
65 #ifdef HAVE_CONFIG_H
66 #include <config.h>
67 #endif
68
69 #include "gstd3d11h264dec.h"
70
71 #include <gst/codecs/gsth264decoder.h>
72 #include <string.h>
73 #include <vector>
74
75 /* HACK: to expose dxva data structure on UWP */
76 #ifdef WINAPI_PARTITION_DESKTOP
77 #undef WINAPI_PARTITION_DESKTOP
78 #endif
79 #define WINAPI_PARTITION_DESKTOP 1
80 #include <d3d9.h>
81 #include <dxva.h>
82
83 GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_h264_dec_debug);
84 #define GST_CAT_DEFAULT gst_d3d11_h264_dec_debug
85
86 /* *INDENT-OFF* */
87 typedef struct _GstD3D11H264DecInner
88 {
89 GstD3D11Device *device = nullptr;
90 GstD3D11Decoder *d3d11_decoder = nullptr;
91
92 DXVA_PicParams_H264 pic_params;
93 DXVA_Qmatrix_H264 iq_matrix;
94
95 std::vector<DXVA_Slice_H264_Short> slice_list;
96 std::vector<guint8> bitstream_buffer;
97
98 gint width = 0;
99 gint height = 0;
100 gint coded_width = 0;
101 gint coded_height = 0;
102 gint bitdepth = 0;
103 guint8 chroma_format_idc = 0;
104 GstVideoFormat out_format = GST_VIDEO_FORMAT_UNKNOWN;
105 gboolean interlaced = FALSE;
106 gint max_dpb_size = 0;
107 } GstD3D11H264DecInner;
108
109 /* *INDENT-ON* */
110 typedef struct _GstD3D11H264Dec
111 {
112 GstH264Decoder parent;
113 GstD3D11H264DecInner *inner;
114 } GstD3D11H264Dec;
115
116 typedef struct _GstD3D11H264DecClass
117 {
118 GstH264DecoderClass parent_class;
119 GstD3D11DecoderSubClassData class_data;
120 } GstD3D11H264DecClass;
121
122 static GstElementClass *parent_class = NULL;
123
124 #define GST_D3D11_H264_DEC(object) ((GstD3D11H264Dec *) (object))
125 #define GST_D3D11_H264_DEC_GET_CLASS(object) \
126 (G_TYPE_INSTANCE_GET_CLASS ((object),G_TYPE_FROM_INSTANCE (object),GstD3D11H264DecClass))
127
128 static void gst_d3d11_h264_dec_get_property (GObject * object,
129 guint prop_id, GValue * value, GParamSpec * pspec);
130 static void gst_d3d11_h264_dec_finalize (GObject * object);
131 static void gst_d3d11_h264_dec_set_context (GstElement * element,
132 GstContext * context);
133
134 static gboolean gst_d3d11_h264_dec_open (GstVideoDecoder * decoder);
135 static gboolean gst_d3d11_h264_dec_close (GstVideoDecoder * decoder);
136 static gboolean gst_d3d11_h264_dec_negotiate (GstVideoDecoder * decoder);
137 static gboolean gst_d3d11_h264_dec_decide_allocation (GstVideoDecoder *
138 decoder, GstQuery * query);
139 static gboolean gst_d3d11_h264_dec_src_query (GstVideoDecoder * decoder,
140 GstQuery * query);
141 static gboolean gst_d3d11_h264_dec_sink_event (GstVideoDecoder * decoder,
142 GstEvent * event);
143
144 /* GstH264Decoder */
145 static GstFlowReturn gst_d3d11_h264_dec_new_sequence (GstH264Decoder * decoder,
146 const GstH264SPS * sps, gint max_dpb_size);
147 static GstFlowReturn gst_d3d11_h264_dec_new_picture (GstH264Decoder * decoder,
148 GstVideoCodecFrame * frame, GstH264Picture * picture);
149 static GstFlowReturn gst_d3d11_h264_dec_new_field_picture (GstH264Decoder *
150 decoder, const GstH264Picture * first_field, GstH264Picture * second_field);
151 static GstFlowReturn gst_d3d11_h264_dec_start_picture (GstH264Decoder * decoder,
152 GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb);
153 static GstFlowReturn gst_d3d11_h264_dec_decode_slice (GstH264Decoder * decoder,
154 GstH264Picture * picture, GstH264Slice * slice, GArray * ref_pic_list0,
155 GArray * ref_pic_list1);
156 static GstFlowReturn gst_d3d11_h264_dec_end_picture (GstH264Decoder * decoder,
157 GstH264Picture * picture);
158 static GstFlowReturn gst_d3d11_h264_dec_output_picture (GstH264Decoder *
159 decoder, GstVideoCodecFrame * frame, GstH264Picture * picture);
160
161 static void
gst_d3d11_h264_dec_class_init(GstD3D11H264DecClass * klass,gpointer data)162 gst_d3d11_h264_dec_class_init (GstD3D11H264DecClass * klass, gpointer data)
163 {
164 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
165 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
166 GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
167 GstH264DecoderClass *h264decoder_class = GST_H264_DECODER_CLASS (klass);
168 GstD3D11DecoderClassData *cdata = (GstD3D11DecoderClassData *) data;
169
170 gobject_class->get_property = gst_d3d11_h264_dec_get_property;
171 gobject_class->finalize = gst_d3d11_h264_dec_finalize;
172
173 element_class->set_context =
174 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_set_context);
175
176 parent_class = (GstElementClass *) g_type_class_peek_parent (klass);
177 gst_d3d11_decoder_class_data_fill_subclass_data (cdata, &klass->class_data);
178
179 /**
180 * GstD3D11H264Dec:adapter-luid:
181 *
182 * DXGI Adapter LUID for this element
183 *
184 * Since: 1.20
185 */
186 gst_d3d11_decoder_proxy_class_init (element_class, cdata,
187 "Seungha Yang <seungha.yang@navercorp.com>");
188
189 decoder_class->open = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_open);
190 decoder_class->close = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_close);
191 decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_negotiate);
192 decoder_class->decide_allocation =
193 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_decide_allocation);
194 decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_src_query);
195 decoder_class->sink_event = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_sink_event);
196
197 h264decoder_class->new_sequence =
198 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_new_sequence);
199 h264decoder_class->new_picture =
200 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_new_picture);
201 h264decoder_class->new_field_picture =
202 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_new_field_picture);
203 h264decoder_class->start_picture =
204 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_start_picture);
205 h264decoder_class->decode_slice =
206 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_decode_slice);
207 h264decoder_class->end_picture =
208 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_end_picture);
209 h264decoder_class->output_picture =
210 GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_output_picture);
211 }
212
213 static void
gst_d3d11_h264_dec_init(GstD3D11H264Dec * self)214 gst_d3d11_h264_dec_init (GstD3D11H264Dec * self)
215 {
216 self->inner = new GstD3D11H264DecInner ();
217 }
218
219 static void
gst_d3d11_h264_dec_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)220 gst_d3d11_h264_dec_get_property (GObject * object, guint prop_id,
221 GValue * value, GParamSpec * pspec)
222 {
223 GstD3D11H264DecClass *klass = GST_D3D11_H264_DEC_GET_CLASS (object);
224 GstD3D11DecoderSubClassData *cdata = &klass->class_data;
225
226 gst_d3d11_decoder_proxy_get_property (object, prop_id, value, pspec, cdata);
227 }
228
229 static void
gst_d3d11_h264_dec_finalize(GObject * object)230 gst_d3d11_h264_dec_finalize (GObject * object)
231 {
232 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (object);
233
234 delete self->inner;
235
236 G_OBJECT_CLASS (parent_class)->finalize (object);
237 }
238
239 static void
gst_d3d11_h264_dec_set_context(GstElement * element,GstContext * context)240 gst_d3d11_h264_dec_set_context (GstElement * element, GstContext * context)
241 {
242 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (element);
243 GstD3D11H264DecInner *inner = self->inner;
244 GstD3D11H264DecClass *klass = GST_D3D11_H264_DEC_GET_CLASS (self);
245 GstD3D11DecoderSubClassData *cdata = &klass->class_data;
246
247 gst_d3d11_handle_set_context_for_adapter_luid (element,
248 context, cdata->adapter_luid, &inner->device);
249
250 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
251 }
252
253 /* Clear all codec specific (e.g., SPS) data */
254 static void
gst_d3d11_h264_dec_reset(GstD3D11H264Dec * self)255 gst_d3d11_h264_dec_reset (GstD3D11H264Dec * self)
256 {
257 GstD3D11H264DecInner *inner = self->inner;
258
259 inner->width = 0;
260 inner->height = 0;
261 inner->coded_width = 0;
262 inner->coded_height = 0;
263 inner->bitdepth = 0;
264 inner->chroma_format_idc = 0;
265 inner->out_format = GST_VIDEO_FORMAT_UNKNOWN;
266 inner->interlaced = FALSE;
267 inner->max_dpb_size = 0;
268 }
269
270 static gboolean
gst_d3d11_h264_dec_open(GstVideoDecoder * decoder)271 gst_d3d11_h264_dec_open (GstVideoDecoder * decoder)
272 {
273 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
274 GstD3D11H264DecInner *inner = self->inner;
275 GstD3D11H264DecClass *klass = GST_D3D11_H264_DEC_GET_CLASS (self);
276 GstD3D11DecoderSubClassData *cdata = &klass->class_data;
277
278 if (!gst_d3d11_decoder_proxy_open (decoder,
279 cdata, &inner->device, &inner->d3d11_decoder)) {
280 GST_ERROR_OBJECT (self, "Failed to open decoder");
281 return FALSE;
282 }
283
284 gst_d3d11_h264_dec_reset (self);
285
286 return TRUE;
287 }
288
289 static gboolean
gst_d3d11_h264_dec_close(GstVideoDecoder * decoder)290 gst_d3d11_h264_dec_close (GstVideoDecoder * decoder)
291 {
292 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
293 GstD3D11H264DecInner *inner = self->inner;
294
295 gst_clear_object (&inner->d3d11_decoder);
296 gst_clear_object (&inner->device);
297
298 return TRUE;
299 }
300
301 static gboolean
gst_d3d11_h264_dec_negotiate(GstVideoDecoder * decoder)302 gst_d3d11_h264_dec_negotiate (GstVideoDecoder * decoder)
303 {
304 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
305 GstD3D11H264DecInner *inner = self->inner;
306
307 if (!gst_d3d11_decoder_negotiate (inner->d3d11_decoder, decoder))
308 return FALSE;
309
310 return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder);
311 }
312
313 static gboolean
gst_d3d11_h264_dec_decide_allocation(GstVideoDecoder * decoder,GstQuery * query)314 gst_d3d11_h264_dec_decide_allocation (GstVideoDecoder * decoder,
315 GstQuery * query)
316 {
317 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
318 GstD3D11H264DecInner *inner = self->inner;
319
320 if (!gst_d3d11_decoder_decide_allocation (inner->d3d11_decoder,
321 decoder, query)) {
322 return FALSE;
323 }
324
325 return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
326 (decoder, query);
327 }
328
329 static gboolean
gst_d3d11_h264_dec_src_query(GstVideoDecoder * decoder,GstQuery * query)330 gst_d3d11_h264_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
331 {
332 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
333 GstD3D11H264DecInner *inner = self->inner;
334
335 switch (GST_QUERY_TYPE (query)) {
336 case GST_QUERY_CONTEXT:
337 if (gst_d3d11_handle_context_query (GST_ELEMENT (decoder),
338 query, inner->device)) {
339 return TRUE;
340 }
341 break;
342 default:
343 break;
344 }
345
346 return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
347 }
348
349 static gboolean
gst_d3d11_h264_dec_sink_event(GstVideoDecoder * decoder,GstEvent * event)350 gst_d3d11_h264_dec_sink_event (GstVideoDecoder * decoder, GstEvent * event)
351 {
352 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
353 GstD3D11H264DecInner *inner = self->inner;
354
355 switch (GST_EVENT_TYPE (event)) {
356 case GST_EVENT_FLUSH_START:
357 if (inner->d3d11_decoder)
358 gst_d3d11_decoder_set_flushing (inner->d3d11_decoder, decoder, TRUE);
359 break;
360 case GST_EVENT_FLUSH_STOP:
361 if (inner->d3d11_decoder)
362 gst_d3d11_decoder_set_flushing (inner->d3d11_decoder, decoder, FALSE);
363 default:
364 break;
365 }
366
367 return GST_VIDEO_DECODER_CLASS (parent_class)->sink_event (decoder, event);
368 }
369
370 static GstFlowReturn
gst_d3d11_h264_dec_new_sequence(GstH264Decoder * decoder,const GstH264SPS * sps,gint max_dpb_size)371 gst_d3d11_h264_dec_new_sequence (GstH264Decoder * decoder,
372 const GstH264SPS * sps, gint max_dpb_size)
373 {
374 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
375 GstD3D11H264DecInner *inner = self->inner;
376 gint crop_width, crop_height;
377 gboolean interlaced;
378 gboolean modified = FALSE;
379
380 GST_LOG_OBJECT (self, "new sequence");
381
382 if (sps->frame_cropping_flag) {
383 crop_width = sps->crop_rect_width;
384 crop_height = sps->crop_rect_height;
385 } else {
386 crop_width = sps->width;
387 crop_height = sps->height;
388 }
389
390 if (inner->width != crop_width || inner->height != crop_height ||
391 inner->coded_width != sps->width || inner->coded_height != sps->height) {
392 GST_INFO_OBJECT (self, "resolution changed %dx%d (%dx%d)",
393 crop_width, crop_height, sps->width, sps->height);
394 inner->width = crop_width;
395 inner->height = crop_height;
396 inner->coded_width = sps->width;
397 inner->coded_height = sps->height;
398 modified = TRUE;
399 }
400
401 if (inner->bitdepth != sps->bit_depth_luma_minus8 + 8) {
402 GST_INFO_OBJECT (self, "bitdepth changed");
403 inner->bitdepth = (guint) sps->bit_depth_luma_minus8 + 8;
404 modified = TRUE;
405 }
406
407 if (inner->chroma_format_idc != sps->chroma_format_idc) {
408 GST_INFO_OBJECT (self, "chroma format changed");
409 inner->chroma_format_idc = sps->chroma_format_idc;
410 modified = TRUE;
411 }
412
413 interlaced = !sps->frame_mbs_only_flag;
414 if (inner->interlaced != interlaced) {
415 GST_INFO_OBJECT (self, "interlaced sequence changed");
416 inner->interlaced = interlaced;
417 modified = TRUE;
418 }
419
420 if (inner->max_dpb_size < max_dpb_size) {
421 GST_INFO_OBJECT (self, "Requires larger DPB size (%d -> %d)",
422 inner->max_dpb_size, max_dpb_size);
423 modified = TRUE;
424 }
425
426 if (modified || !gst_d3d11_decoder_is_configured (inner->d3d11_decoder)) {
427 GstVideoInfo info;
428
429 inner->out_format = GST_VIDEO_FORMAT_UNKNOWN;
430
431 if (inner->bitdepth == 8) {
432 if (inner->chroma_format_idc == 1)
433 inner->out_format = GST_VIDEO_FORMAT_NV12;
434 else {
435 GST_FIXME_OBJECT (self, "Could not support 8bits non-4:2:0 format");
436 }
437 }
438
439 if (inner->out_format == GST_VIDEO_FORMAT_UNKNOWN) {
440 GST_ERROR_OBJECT (self, "Could not support bitdepth/chroma format");
441 return GST_FLOW_NOT_NEGOTIATED;
442 }
443
444 gst_video_info_set_format (&info,
445 inner->out_format, inner->width, inner->height);
446 if (inner->interlaced)
447 GST_VIDEO_INFO_INTERLACE_MODE (&info) = GST_VIDEO_INTERLACE_MODE_MIXED;
448
449 /* Store configured DPB size here. Then, it will be referenced later
450 * to decide whether we need to re-open decoder object or not.
451 * For instance, if every configuration is same apart from DPB size and
452 * new DPB size is decreased, we can reuse existing decoder object.
453 */
454 inner->max_dpb_size = max_dpb_size;
455 if (!gst_d3d11_decoder_configure (inner->d3d11_decoder,
456 decoder->input_state, &info,
457 inner->coded_width, inner->coded_height,
458 /* Additional 4 views margin for zero-copy rendering */
459 max_dpb_size + 4)) {
460 GST_ERROR_OBJECT (self, "Failed to create decoder");
461 return GST_FLOW_NOT_NEGOTIATED;
462 }
463
464 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
465 GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
466 return GST_FLOW_NOT_NEGOTIATED;
467 }
468 }
469
470 return GST_FLOW_OK;
471 }
472
473 static GstFlowReturn
gst_d3d11_h264_dec_new_picture(GstH264Decoder * decoder,GstVideoCodecFrame * frame,GstH264Picture * picture)474 gst_d3d11_h264_dec_new_picture (GstH264Decoder * decoder,
475 GstVideoCodecFrame * frame, GstH264Picture * picture)
476 {
477 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
478 GstD3D11H264DecInner *inner = self->inner;
479 GstBuffer *view_buffer;
480
481 view_buffer = gst_d3d11_decoder_get_output_view_buffer (inner->d3d11_decoder,
482 GST_VIDEO_DECODER (decoder));
483 if (!view_buffer) {
484 GST_DEBUG_OBJECT (self, "No available output view buffer");
485 return GST_FLOW_FLUSHING;
486 }
487
488 GST_LOG_OBJECT (self, "New output view buffer %" GST_PTR_FORMAT, view_buffer);
489
490 gst_h264_picture_set_user_data (picture,
491 view_buffer, (GDestroyNotify) gst_buffer_unref);
492
493 GST_LOG_OBJECT (self, "New h264picture %p", picture);
494
495 return GST_FLOW_OK;
496 }
497
498 static GstFlowReturn
gst_d3d11_h264_dec_new_field_picture(GstH264Decoder * decoder,const GstH264Picture * first_field,GstH264Picture * second_field)499 gst_d3d11_h264_dec_new_field_picture (GstH264Decoder * decoder,
500 const GstH264Picture * first_field, GstH264Picture * second_field)
501 {
502 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
503 GstBuffer *view_buffer;
504
505 view_buffer = (GstBuffer *) gst_h264_picture_get_user_data ((GstH264Picture *)
506 first_field);
507
508 if (!view_buffer) {
509 GST_WARNING_OBJECT (self, "First picture does not have output view buffer");
510 return GST_FLOW_OK;
511 }
512
513 GST_LOG_OBJECT (self, "New field picture with buffer %" GST_PTR_FORMAT,
514 view_buffer);
515
516 gst_h264_picture_set_user_data (second_field,
517 gst_buffer_ref (view_buffer), (GDestroyNotify) gst_buffer_unref);
518
519 return GST_FLOW_OK;
520 }
521
522 static ID3D11VideoDecoderOutputView *
gst_d3d11_h264_dec_get_output_view_from_picture(GstD3D11H264Dec * self,GstH264Picture * picture,guint8 * view_id)523 gst_d3d11_h264_dec_get_output_view_from_picture (GstD3D11H264Dec * self,
524 GstH264Picture * picture, guint8 * view_id)
525 {
526 GstD3D11H264DecInner *inner = self->inner;
527 GstBuffer *view_buffer;
528 ID3D11VideoDecoderOutputView *view;
529
530 view_buffer = (GstBuffer *) gst_h264_picture_get_user_data (picture);
531 if (!view_buffer) {
532 GST_DEBUG_OBJECT (self, "current picture does not have output view buffer");
533 return NULL;
534 }
535
536 view = gst_d3d11_decoder_get_output_view_from_buffer (inner->d3d11_decoder,
537 view_buffer, view_id);
538 if (!view) {
539 GST_DEBUG_OBJECT (self, "current picture does not have output view handle");
540 return NULL;
541 }
542
543 return view;
544 }
545
546 static void
gst_d3d11_h264_dec_picture_params_from_sps(GstD3D11H264Dec * self,const GstH264SPS * sps,gboolean field_pic,DXVA_PicParams_H264 * params)547 gst_d3d11_h264_dec_picture_params_from_sps (GstD3D11H264Dec * self,
548 const GstH264SPS * sps, gboolean field_pic, DXVA_PicParams_H264 * params)
549 {
550 #define COPY_FIELD(f) \
551 (params)->f = (sps)->f
552
553 params->wFrameWidthInMbsMinus1 = sps->pic_width_in_mbs_minus1;
554 if (!sps->frame_mbs_only_flag) {
555 params->wFrameHeightInMbsMinus1 =
556 ((sps->pic_height_in_map_units_minus1 + 1) << 1) - 1;
557 } else {
558 params->wFrameHeightInMbsMinus1 = sps->pic_height_in_map_units_minus1;
559 }
560 params->residual_colour_transform_flag = sps->separate_colour_plane_flag;
561 params->MbaffFrameFlag = (sps->mb_adaptive_frame_field_flag && !field_pic);
562 params->field_pic_flag = field_pic;
563 params->MinLumaBipredSize8x8Flag = sps->level_idc >= 31;
564
565 COPY_FIELD (num_ref_frames);
566 COPY_FIELD (chroma_format_idc);
567 COPY_FIELD (frame_mbs_only_flag);
568 COPY_FIELD (bit_depth_luma_minus8);
569 COPY_FIELD (bit_depth_chroma_minus8);
570 COPY_FIELD (log2_max_frame_num_minus4);
571 COPY_FIELD (pic_order_cnt_type);
572 COPY_FIELD (log2_max_pic_order_cnt_lsb_minus4);
573 COPY_FIELD (delta_pic_order_always_zero_flag);
574 COPY_FIELD (direct_8x8_inference_flag);
575
576 #undef COPY_FIELD
577 }
578
579 static void
gst_d3d11_h264_dec_picture_params_from_pps(GstD3D11H264Dec * self,const GstH264PPS * pps,DXVA_PicParams_H264 * params)580 gst_d3d11_h264_dec_picture_params_from_pps (GstD3D11H264Dec * self,
581 const GstH264PPS * pps, DXVA_PicParams_H264 * params)
582 {
583 #define COPY_FIELD(f) \
584 (params)->f = (pps)->f
585
586 COPY_FIELD (constrained_intra_pred_flag);
587 COPY_FIELD (weighted_pred_flag);
588 COPY_FIELD (weighted_bipred_idc);
589 COPY_FIELD (transform_8x8_mode_flag);
590 COPY_FIELD (pic_init_qs_minus26);
591 COPY_FIELD (chroma_qp_index_offset);
592 COPY_FIELD (second_chroma_qp_index_offset);
593 COPY_FIELD (pic_init_qp_minus26);
594 COPY_FIELD (num_ref_idx_l0_active_minus1);
595 COPY_FIELD (num_ref_idx_l1_active_minus1);
596 COPY_FIELD (entropy_coding_mode_flag);
597 COPY_FIELD (pic_order_present_flag);
598 COPY_FIELD (deblocking_filter_control_present_flag);
599 COPY_FIELD (redundant_pic_cnt_present_flag);
600 COPY_FIELD (num_slice_groups_minus1);
601 COPY_FIELD (slice_group_map_type);
602
603 #undef COPY_FIELD
604 }
605
606 static void
gst_d3d11_h264_dec_picture_params_from_slice_header(GstD3D11H264Dec * self,const GstH264SliceHdr * slice_header,DXVA_PicParams_H264 * params)607 gst_d3d11_h264_dec_picture_params_from_slice_header (GstD3D11H264Dec *
608 self, const GstH264SliceHdr * slice_header, DXVA_PicParams_H264 * params)
609 {
610 params->sp_for_switch_flag = slice_header->sp_for_switch_flag;
611 params->field_pic_flag = slice_header->field_pic_flag;
612 params->CurrPic.AssociatedFlag = slice_header->bottom_field_flag;
613 params->IntraPicFlag =
614 GST_H264_IS_I_SLICE (slice_header) || GST_H264_IS_SI_SLICE (slice_header);
615 }
616
617 static gboolean
gst_d3d11_h264_dec_fill_picture_params(GstD3D11H264Dec * self,const GstH264SliceHdr * slice_header,DXVA_PicParams_H264 * params)618 gst_d3d11_h264_dec_fill_picture_params (GstD3D11H264Dec * self,
619 const GstH264SliceHdr * slice_header, DXVA_PicParams_H264 * params)
620 {
621 const GstH264SPS *sps;
622 const GstH264PPS *pps;
623
624 g_return_val_if_fail (slice_header->pps != NULL, FALSE);
625 g_return_val_if_fail (slice_header->pps->sequence != NULL, FALSE);
626
627 pps = slice_header->pps;
628 sps = pps->sequence;
629
630 params->MbsConsecutiveFlag = 1;
631 params->Reserved16Bits = 3;
632 params->ContinuationFlag = 1;
633 params->Reserved8BitsA = 0;
634 params->Reserved8BitsB = 0;
635 params->StatusReportFeedbackNumber = 1;
636
637 gst_d3d11_h264_dec_picture_params_from_sps (self,
638 sps, slice_header->field_pic_flag, params);
639 gst_d3d11_h264_dec_picture_params_from_pps (self, pps, params);
640 gst_d3d11_h264_dec_picture_params_from_slice_header (self,
641 slice_header, params);
642
643 return TRUE;
644 }
645
646 static inline void
init_pic_params(DXVA_PicParams_H264 * params)647 init_pic_params (DXVA_PicParams_H264 * params)
648 {
649 memset (params, 0, sizeof (DXVA_PicParams_H264));
650 for (guint i = 0; i < G_N_ELEMENTS (params->RefFrameList); i++)
651 params->RefFrameList[i].bPicEntry = 0xff;
652 }
653
654 static GstFlowReturn
gst_d3d11_h264_dec_start_picture(GstH264Decoder * decoder,GstH264Picture * picture,GstH264Slice * slice,GstH264Dpb * dpb)655 gst_d3d11_h264_dec_start_picture (GstH264Decoder * decoder,
656 GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb)
657 {
658 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
659 GstD3D11H264DecInner *inner = self->inner;
660 DXVA_PicParams_H264 *pic_params = &inner->pic_params;
661 DXVA_Qmatrix_H264 *iq_matrix = &inner->iq_matrix;
662 ID3D11VideoDecoderOutputView *view;
663 guint8 view_id = 0xff;
664 GArray *dpb_array;
665 GstH264PPS *pps;
666 guint i, j;
667
668 pps = slice->header.pps;
669
670 view = gst_d3d11_h264_dec_get_output_view_from_picture (self, picture,
671 &view_id);
672 if (!view) {
673 GST_ERROR_OBJECT (self, "current picture does not have output view handle");
674 return GST_FLOW_ERROR;
675 }
676
677 init_pic_params (pic_params);
678 gst_d3d11_h264_dec_fill_picture_params (self, &slice->header, pic_params);
679
680 pic_params->CurrPic.Index7Bits = view_id;
681 pic_params->RefPicFlag = GST_H264_PICTURE_IS_REF (picture);
682 pic_params->frame_num = picture->frame_num;
683
684 if (picture->field == GST_H264_PICTURE_FIELD_TOP_FIELD) {
685 pic_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt;
686 pic_params->CurrFieldOrderCnt[1] = 0;
687 } else if (picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD) {
688 pic_params->CurrFieldOrderCnt[0] = 0;
689 pic_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt;
690 } else {
691 pic_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt;
692 pic_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt;
693 }
694
695 dpb_array = gst_h264_dpb_get_pictures_all (dpb);
696 for (i = 0, j = 0; i < dpb_array->len && j < 16; i++) {
697 GstH264Picture *other = g_array_index (dpb_array, GstH264Picture *, i);
698 guint8 id = 0xff;
699
700 if (!GST_H264_PICTURE_IS_REF (other))
701 continue;
702
703 /* The second field picture will be handled differently */
704 if (other->second_field)
705 continue;
706
707 gst_d3d11_h264_dec_get_output_view_from_picture (self, other, &id);
708 pic_params->RefFrameList[j].Index7Bits = id;
709
710 if (GST_H264_PICTURE_IS_LONG_TERM_REF (other)) {
711 pic_params->RefFrameList[j].AssociatedFlag = 1;
712 pic_params->FrameNumList[j] = other->long_term_frame_idx;
713 } else {
714 pic_params->RefFrameList[j].AssociatedFlag = 0;
715 pic_params->FrameNumList[j] = other->frame_num;
716 }
717
718 switch (other->field) {
719 case GST_H264_PICTURE_FIELD_TOP_FIELD:
720 pic_params->FieldOrderCntList[j][0] = other->top_field_order_cnt;
721 pic_params->UsedForReferenceFlags |= 0x1 << (2 * j);
722 break;
723 case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
724 pic_params->FieldOrderCntList[j][1] = other->bottom_field_order_cnt;
725 pic_params->UsedForReferenceFlags |= 0x1 << (2 * j + 1);
726 break;
727 default:
728 pic_params->FieldOrderCntList[j][0] = other->top_field_order_cnt;
729 pic_params->FieldOrderCntList[j][1] = other->bottom_field_order_cnt;
730 pic_params->UsedForReferenceFlags |= 0x3 << (2 * j);
731 break;
732 }
733
734 if (other->other_field) {
735 GstH264Picture *other_field = other->other_field;
736
737 switch (other_field->field) {
738 case GST_H264_PICTURE_FIELD_TOP_FIELD:
739 pic_params->FieldOrderCntList[j][0] =
740 other_field->top_field_order_cnt;
741 pic_params->UsedForReferenceFlags |= 0x1 << (2 * j);
742 break;
743 case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
744 pic_params->FieldOrderCntList[j][1] =
745 other_field->bottom_field_order_cnt;
746 pic_params->UsedForReferenceFlags |= 0x1 << (2 * j + 1);
747 break;
748 default:
749 break;
750 }
751 }
752
753 pic_params->NonExistingFrameFlags |= (other->nonexisting) << j;
754 j++;
755 }
756 g_array_unref (dpb_array);
757
758 G_STATIC_ASSERT (sizeof (iq_matrix->bScalingLists4x4) ==
759 sizeof (pps->scaling_lists_4x4));
760 memcpy (iq_matrix->bScalingLists4x4, pps->scaling_lists_4x4,
761 sizeof (pps->scaling_lists_4x4));
762
763 G_STATIC_ASSERT (sizeof (iq_matrix->bScalingLists8x8[0]) ==
764 sizeof (pps->scaling_lists_8x8[0]));
765 memcpy (iq_matrix->bScalingLists8x8[0], pps->scaling_lists_8x8[0],
766 sizeof (pps->scaling_lists_8x8[0]));
767 memcpy (iq_matrix->bScalingLists8x8[1], pps->scaling_lists_8x8[1],
768 sizeof (pps->scaling_lists_8x8[1]));
769
770 inner->slice_list.resize (0);
771 inner->bitstream_buffer.resize (0);
772
773 return GST_FLOW_OK;
774 }
775
776 static GstFlowReturn
gst_d3d11_h264_dec_decode_slice(GstH264Decoder * decoder,GstH264Picture * picture,GstH264Slice * slice,GArray * ref_pic_list0,GArray * ref_pic_list1)777 gst_d3d11_h264_dec_decode_slice (GstH264Decoder * decoder,
778 GstH264Picture * picture, GstH264Slice * slice, GArray * ref_pic_list0,
779 GArray * ref_pic_list1)
780 {
781 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
782 GstD3D11H264DecInner *inner = self->inner;
783 DXVA_Slice_H264_Short dxva_slice;
784 static const guint8 start_code[] = { 0, 0, 1 };
785 const size_t start_code_size = sizeof (start_code);
786
787 dxva_slice.BSNALunitDataLocation = inner->bitstream_buffer.size ();
788 /* Includes 3 bytes start code prefix */
789 dxva_slice.SliceBytesInBuffer = slice->nalu.size + start_code_size;
790 dxva_slice.wBadSliceChopping = 0;
791
792 inner->slice_list.push_back (dxva_slice);
793
794 size_t pos = inner->bitstream_buffer.size ();
795 inner->bitstream_buffer.resize (pos + start_code_size + slice->nalu.size);
796
797 /* Fill start code prefix */
798 memcpy (&inner->bitstream_buffer[0] + pos, start_code, start_code_size);
799
800 /* Copy bitstream */
801 memcpy (&inner->bitstream_buffer[0] + pos + start_code_size,
802 slice->nalu.data + slice->nalu.offset, slice->nalu.size);
803
804 return GST_FLOW_OK;
805 }
806
807 static GstFlowReturn
gst_d3d11_h264_dec_end_picture(GstH264Decoder * decoder,GstH264Picture * picture)808 gst_d3d11_h264_dec_end_picture (GstH264Decoder * decoder,
809 GstH264Picture * picture)
810 {
811 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
812 GstD3D11H264DecInner *inner = self->inner;
813 ID3D11VideoDecoderOutputView *view;
814 guint8 view_id = 0xff;
815 size_t bitstream_buffer_size;
816 size_t bitstream_pos;
817 GstD3D11DecodeInputStreamArgs input_args;
818
819 GST_LOG_OBJECT (self, "end picture %p, (poc %d)",
820 picture, picture->pic_order_cnt);
821
822 if (inner->bitstream_buffer.empty () || inner->slice_list.empty ()) {
823 GST_ERROR_OBJECT (self, "No bitstream buffer to submit");
824 return GST_FLOW_ERROR;
825 }
826
827 view = gst_d3d11_h264_dec_get_output_view_from_picture (self, picture,
828 &view_id);
829 if (!view) {
830 GST_ERROR_OBJECT (self, "current picture does not have output view handle");
831 return GST_FLOW_ERROR;
832 }
833
834 memset (&input_args, 0, sizeof (GstD3D11DecodeInputStreamArgs));
835
836 bitstream_pos = inner->bitstream_buffer.size ();
837 bitstream_buffer_size = GST_ROUND_UP_128 (bitstream_pos);
838
839 if (bitstream_buffer_size > bitstream_pos) {
840 size_t padding = bitstream_buffer_size - bitstream_pos;
841
842 /* As per DXVA spec, total amount of bitstream buffer size should be
843 * 128 bytes aligned. If actual data is not multiple of 128 bytes,
844 * the last slice data needs to be zero-padded */
845 inner->bitstream_buffer.resize (bitstream_buffer_size, 0);
846
847 DXVA_Slice_H264_Short & slice = inner->slice_list.back ();
848 slice.SliceBytesInBuffer += padding;
849 }
850
851 input_args.picture_params = &inner->pic_params;
852 input_args.picture_params_size = sizeof (DXVA_PicParams_H264);
853 input_args.slice_control = &inner->slice_list[0];
854 input_args.slice_control_size =
855 sizeof (DXVA_Slice_H264_Short) * inner->slice_list.size ();
856 input_args.bitstream = &inner->bitstream_buffer[0];
857 input_args.bitstream_size = inner->bitstream_buffer.size ();
858 input_args.inverse_quantization_matrix = &inner->iq_matrix;
859 input_args.inverse_quantization_matrix_size = sizeof (DXVA_Qmatrix_H264);
860
861 if (!gst_d3d11_decoder_decode_frame (inner->d3d11_decoder, view, &input_args))
862 return GST_FLOW_ERROR;
863
864 return GST_FLOW_OK;
865 }
866
867 static GstFlowReturn
gst_d3d11_h264_dec_output_picture(GstH264Decoder * decoder,GstVideoCodecFrame * frame,GstH264Picture * picture)868 gst_d3d11_h264_dec_output_picture (GstH264Decoder * decoder,
869 GstVideoCodecFrame * frame, GstH264Picture * picture)
870 {
871 GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder);
872 GstD3D11H264DecInner *inner = self->inner;
873 GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
874 GstBuffer *view_buffer;
875
876 GST_LOG_OBJECT (self,
877 "Outputting picture %p (poc %d)", picture, picture->pic_order_cnt);
878
879 view_buffer = (GstBuffer *) gst_h264_picture_get_user_data (picture);
880
881 if (!view_buffer) {
882 GST_ERROR_OBJECT (self, "Could not get output view");
883 goto error;
884 }
885
886 if (!gst_d3d11_decoder_process_output (inner->d3d11_decoder, vdec,
887 inner->width, inner->height, view_buffer, &frame->output_buffer)) {
888 GST_ERROR_OBJECT (self, "Failed to copy buffer");
889 goto error;
890 }
891
892 if (picture->buffer_flags != 0) {
893 gboolean interlaced =
894 (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_INTERLACED) != 0;
895 gboolean tff = (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_TFF) != 0;
896
897 GST_TRACE_OBJECT (self,
898 "apply buffer flags 0x%x (interlaced %d, top-field-first %d)",
899 picture->buffer_flags, interlaced, tff);
900 GST_BUFFER_FLAG_SET (frame->output_buffer, picture->buffer_flags);
901 }
902
903 gst_h264_picture_unref (picture);
904
905 return gst_video_decoder_finish_frame (vdec, frame);
906
907 error:
908 gst_h264_picture_unref (picture);
909 gst_video_decoder_release_frame (vdec, frame);
910
911 return GST_FLOW_ERROR;
912 }
913
914 void
gst_d3d11_h264_dec_register(GstPlugin * plugin,GstD3D11Device * device,guint rank,gboolean legacy)915 gst_d3d11_h264_dec_register (GstPlugin * plugin, GstD3D11Device * device,
916 guint rank, gboolean legacy)
917 {
918 GType type;
919 gchar *type_name;
920 gchar *feature_name;
921 guint index = 0;
922 guint i;
923 gboolean ret;
924 GTypeInfo type_info = {
925 sizeof (GstD3D11H264DecClass),
926 NULL,
927 NULL,
928 (GClassInitFunc) gst_d3d11_h264_dec_class_init,
929 NULL,
930 NULL,
931 sizeof (GstD3D11H264Dec),
932 0,
933 (GInstanceInitFunc) gst_d3d11_h264_dec_init,
934 };
935 const GUID *supported_profile = NULL;
936 GstCaps *sink_caps = NULL;
937 GstCaps *src_caps = NULL;
938 guint max_width = 0;
939 guint max_height = 0;
940 guint resolution;
941
942 ret = gst_d3d11_decoder_get_supported_decoder_profile (device,
943 GST_DXVA_CODEC_H264, GST_VIDEO_FORMAT_NV12, &supported_profile);
944
945 if (!ret) {
946 GST_WARNING_OBJECT (device, "decoder profile unavailable");
947 return;
948 }
949
950 ret =
951 gst_d3d11_decoder_supports_format (device, supported_profile,
952 DXGI_FORMAT_NV12);
953 if (!ret) {
954 GST_FIXME_OBJECT (device, "device does not support NV12 format");
955 return;
956 }
957
958 /* we will not check the maximum resolution for legacy devices.
959 * it might cause crash */
960 if (legacy) {
961 max_width = gst_dxva_resolutions[0].width;
962 max_height = gst_dxva_resolutions[0].height;
963 } else {
964 for (i = 0; i < G_N_ELEMENTS (gst_dxva_resolutions); i++) {
965 if (gst_d3d11_decoder_supports_resolution (device, supported_profile,
966 DXGI_FORMAT_NV12, gst_dxva_resolutions[i].width,
967 gst_dxva_resolutions[i].height)) {
968 max_width = gst_dxva_resolutions[i].width;
969 max_height = gst_dxva_resolutions[i].height;
970
971 GST_DEBUG_OBJECT (device,
972 "device support resolution %dx%d", max_width, max_height);
973 } else {
974 break;
975 }
976 }
977 }
978
979 if (max_width == 0 || max_height == 0) {
980 GST_WARNING_OBJECT (device, "Couldn't query supported resolution");
981 return;
982 }
983
984 sink_caps = gst_caps_from_string ("video/x-h264, "
985 "stream-format= (string) { avc, avc3, byte-stream }, "
986 "alignment= (string) au, "
987 "profile = (string) { high, progressive-high, constrained-high, main, constrained-baseline, baseline }");
988 src_caps = gst_caps_from_string ("video/x-raw("
989 GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "), format = (string) NV12; "
990 "video/x-raw, format = (string) NV12");
991
992 /* To cover both landscape and portrait, select max value */
993 resolution = MAX (max_width, max_height);
994 gst_caps_set_simple (sink_caps,
995 "width", GST_TYPE_INT_RANGE, 1, resolution,
996 "height", GST_TYPE_INT_RANGE, 1, resolution, NULL);
997 gst_caps_set_simple (src_caps,
998 "width", GST_TYPE_INT_RANGE, 1, resolution,
999 "height", GST_TYPE_INT_RANGE, 1, resolution, NULL);
1000
1001 type_info.class_data =
1002 gst_d3d11_decoder_class_data_new (device, GST_DXVA_CODEC_H264,
1003 sink_caps, src_caps);
1004
1005 type_name = g_strdup ("GstD3D11H264Dec");
1006 feature_name = g_strdup ("d3d11h264dec");
1007
1008 while (g_type_from_name (type_name)) {
1009 index++;
1010 g_free (type_name);
1011 g_free (feature_name);
1012 type_name = g_strdup_printf ("GstD3D11H264Device%dDec", index);
1013 feature_name = g_strdup_printf ("d3d11h264device%ddec", index);
1014 }
1015
1016 type = g_type_register_static (GST_TYPE_H264_DECODER,
1017 type_name, &type_info, (GTypeFlags) 0);
1018
1019 /* make lower rank than default device */
1020 if (rank > 0 && index != 0)
1021 rank--;
1022
1023 if (index != 0)
1024 gst_element_type_set_skip_documentation (type);
1025
1026 if (!gst_element_register (plugin, feature_name, rank, type))
1027 GST_WARNING ("Failed to register plugin '%s'", type_name);
1028
1029 g_free (type_name);
1030 g_free (feature_name);
1031 }
1032