1 /* GStreamer
2 * Copyright (C) 2020 Seungha Yang <seungha.yang@navercorp.com>
3 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 #ifdef HAVE_CONFIG_H
22 #include "config.h"
23 #endif
24
25 #include <gst/gst.h>
26 #include "gstmfvideoenc.h"
27 #include "gstmfvideobuffer.h"
28 #include "gstmfplatloader.h"
29 #include <wrl.h>
30 #include <string.h>
31 #include <cmath>
32
33 #if GST_MF_HAVE_D3D11
34 #include <d3d10.h>
35 #endif
36
37 /* *INDENT-OFF* */
38 using namespace Microsoft::WRL;
39
40 G_BEGIN_DECLS
41
42 GST_DEBUG_CATEGORY_EXTERN (gst_mf_video_enc_debug);
43 #define GST_CAT_DEFAULT gst_mf_video_enc_debug
44
45 G_END_DECLS
46 /* *INDENT-ON* */
47
48 #define gst_mf_video_enc_parent_class parent_class
49 G_DEFINE_ABSTRACT_TYPE (GstMFVideoEnc, gst_mf_video_enc,
50 GST_TYPE_VIDEO_ENCODER);
51
52 static void gst_mf_video_enc_dispose (GObject * object);
53 static void gst_mf_video_enc_set_context (GstElement * element,
54 GstContext * context);
55 static gboolean gst_mf_video_enc_open (GstVideoEncoder * enc);
56 static gboolean gst_mf_video_enc_close (GstVideoEncoder * enc);
57 static gboolean gst_mf_video_enc_start (GstVideoEncoder * enc);
58 static gboolean gst_mf_video_enc_set_format (GstVideoEncoder * enc,
59 GstVideoCodecState * state);
60 static GstFlowReturn gst_mf_video_enc_handle_frame (GstVideoEncoder * enc,
61 GstVideoCodecFrame * frame);
62 static GstFlowReturn gst_mf_video_enc_finish (GstVideoEncoder * enc);
63 static gboolean gst_mf_video_enc_flush (GstVideoEncoder * enc);
64 static gboolean gst_mf_video_enc_propose_allocation (GstVideoEncoder * enc,
65 GstQuery * query);
66 static gboolean gst_mf_video_enc_sink_query (GstVideoEncoder * enc,
67 GstQuery * query);
68 static gboolean gst_mf_video_enc_src_query (GstVideoEncoder * enc,
69 GstQuery * query);
70
71 static HRESULT gst_mf_video_on_new_sample (GstMFTransform * object,
72 IMFSample * sample, GstMFVideoEnc * self);
73
74 static void
gst_mf_video_enc_class_init(GstMFVideoEncClass * klass)75 gst_mf_video_enc_class_init (GstMFVideoEncClass * klass)
76 {
77 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
78 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
79 GstVideoEncoderClass *videoenc_class = GST_VIDEO_ENCODER_CLASS (klass);
80
81 gobject_class->dispose = gst_mf_video_enc_dispose;
82
83 element_class->set_context = GST_DEBUG_FUNCPTR (gst_mf_video_enc_set_context);
84
85 videoenc_class->open = GST_DEBUG_FUNCPTR (gst_mf_video_enc_open);
86 videoenc_class->close = GST_DEBUG_FUNCPTR (gst_mf_video_enc_close);
87 videoenc_class->start = GST_DEBUG_FUNCPTR (gst_mf_video_enc_start);
88 videoenc_class->set_format = GST_DEBUG_FUNCPTR (gst_mf_video_enc_set_format);
89 videoenc_class->handle_frame =
90 GST_DEBUG_FUNCPTR (gst_mf_video_enc_handle_frame);
91 videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_mf_video_enc_finish);
92 videoenc_class->flush = GST_DEBUG_FUNCPTR (gst_mf_video_enc_flush);
93 videoenc_class->propose_allocation =
94 GST_DEBUG_FUNCPTR (gst_mf_video_enc_propose_allocation);
95 videoenc_class->sink_query = GST_DEBUG_FUNCPTR (gst_mf_video_enc_sink_query);
96 videoenc_class->src_query = GST_DEBUG_FUNCPTR (gst_mf_video_enc_src_query);
97
98 gst_type_mark_as_plugin_api (GST_TYPE_MF_VIDEO_ENC, (GstPluginAPIFlags) 0);
99 }
100
101 static void
gst_mf_video_enc_init(GstMFVideoEnc * self)102 gst_mf_video_enc_init (GstMFVideoEnc * self)
103 {
104 }
105
106 static void
gst_mf_video_enc_dispose(GObject * object)107 gst_mf_video_enc_dispose (GObject * object)
108 {
109 #if GST_MF_HAVE_D3D11
110 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (object);
111
112 gst_clear_object (&self->d3d11_device);
113 gst_clear_object (&self->other_d3d11_device);
114 #endif
115
116 G_OBJECT_CLASS (parent_class)->dispose (object);
117 }
118
119 static void
gst_mf_video_enc_set_context(GstElement * element,GstContext * context)120 gst_mf_video_enc_set_context (GstElement * element, GstContext * context)
121 {
122 #if GST_MF_HAVE_D3D11
123 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (element);
124 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (self);
125 GstMFVideoEncDeviceCaps *device_caps = &klass->device_caps;
126
127 if (device_caps->d3d11_aware) {
128 gst_d3d11_handle_set_context_for_adapter_luid (element, context,
129 device_caps->adapter_luid, &self->other_d3d11_device);
130 }
131 #endif
132
133 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
134 }
135
136 static gboolean
gst_mf_video_enc_open(GstVideoEncoder * enc)137 gst_mf_video_enc_open (GstVideoEncoder * enc)
138 {
139 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
140 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (enc);
141 GstMFVideoEncDeviceCaps *device_caps = &klass->device_caps;
142 GstMFTransformEnumParams enum_params = { 0, };
143 MFT_REGISTER_TYPE_INFO output_type;
144 gboolean ret;
145
146 #if GST_MF_HAVE_D3D11
147 if (device_caps->d3d11_aware) {
148 HRESULT hr;
149 ID3D11Device *device_handle;
150 ComPtr < ID3D10Multithread > multi_thread;
151 GstD3D11Device *device;
152
153 if (!gst_d3d11_ensure_element_data_for_adapter_luid (GST_ELEMENT (self),
154 device_caps->adapter_luid, &self->other_d3d11_device)) {
155 GST_ERROR_OBJECT (self, "Other d3d11 device is unavailable");
156 return FALSE;
157 }
158
159 /* Create our own device with D3D11_CREATE_DEVICE_VIDEO_SUPPORT */
160 self->d3d11_device =
161 gst_d3d11_device_new_for_adapter_luid (device_caps->adapter_luid,
162 D3D11_CREATE_DEVICE_VIDEO_SUPPORT);
163 if (!self->d3d11_device) {
164 GST_ERROR_OBJECT (self, "Couldn't create internal d3d11 device");
165 gst_clear_object (&self->other_d3d11_device);
166 return FALSE;
167 }
168
169 device = self->d3d11_device;
170
171 hr = GstMFCreateDXGIDeviceManager (&self->reset_token, &self->device_manager);
172 if (!gst_mf_result (hr)) {
173 GST_ERROR_OBJECT (self, "Couldn't create DXGI device manager");
174 gst_clear_object (&self->other_d3d11_device);
175 gst_clear_object (&self->d3d11_device);
176 return FALSE;
177 }
178
179 device_handle = gst_d3d11_device_get_device_handle (device);
180 /* Enable multi thread protection as this device will be shared with
181 * MFT */
182 hr = device_handle->QueryInterface (IID_PPV_ARGS (&multi_thread));
183 if (!gst_d3d11_result (hr, device)) {
184 GST_WARNING_OBJECT (self,
185 "device doesn't suport ID3D10Multithread interface");
186 gst_clear_object (&self->other_d3d11_device);
187 gst_clear_object (&self->d3d11_device);
188 }
189
190 multi_thread->SetMultithreadProtected (TRUE);
191
192 hr = self->device_manager->ResetDevice ((IUnknown *) device_handle,
193 self->reset_token);
194 if (!gst_mf_result (hr)) {
195 GST_ERROR_OBJECT (self, "Couldn't reset device with given d3d11 device");
196 gst_clear_object (&self->other_d3d11_device);
197 gst_clear_object (&self->d3d11_device);
198 return FALSE;
199 }
200 }
201 #endif
202
203 output_type.guidMajorType = MFMediaType_Video;
204 output_type.guidSubtype = klass->codec_id;
205
206 enum_params.category = MFT_CATEGORY_VIDEO_ENCODER;
207 enum_params.enum_flags = klass->enum_flags;
208 enum_params.output_typeinfo = &output_type;
209 enum_params.device_index = klass->device_index;
210
211 if (device_caps->d3d11_aware)
212 enum_params.adapter_luid = device_caps->adapter_luid;
213
214 GST_DEBUG_OBJECT (self,
215 "Create MFT with enum flags: 0x%x, device index: %d, d3d11 aware: %d, "
216 "adapter-luid %" G_GINT64_FORMAT, klass->enum_flags, klass->device_index,
217 device_caps->d3d11_aware, device_caps->adapter_luid);
218
219 self->transform = gst_mf_transform_new (&enum_params);
220 ret = !!self->transform;
221
222 if (!ret) {
223 GST_ERROR_OBJECT (self, "Cannot create MFT object");
224 return FALSE;
225 }
226
227 /* In case of hardware MFT, it will be running on async mode.
228 * And new output sample callback will be called from Media Foundation's
229 * internal worker queue thread */
230 if (self->transform &&
231 (enum_params.enum_flags & MFT_ENUM_FLAG_HARDWARE) ==
232 MFT_ENUM_FLAG_HARDWARE) {
233 self->async_mft = TRUE;
234 gst_mf_transform_set_new_sample_callback (self->transform,
235 (GstMFTransformNewSampleCallback) gst_mf_video_on_new_sample, self);
236 } else {
237 self->async_mft = FALSE;
238 }
239
240 return ret;
241 }
242
243 static gboolean
gst_mf_video_enc_close(GstVideoEncoder * enc)244 gst_mf_video_enc_close (GstVideoEncoder * enc)
245 {
246 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
247
248 gst_clear_object (&self->transform);
249
250 if (self->input_state) {
251 gst_video_codec_state_unref (self->input_state);
252 self->input_state = NULL;
253 }
254 #if GST_MF_HAVE_D3D11
255 if (self->device_manager) {
256 self->device_manager->Release ();
257 self->device_manager = nullptr;
258 }
259
260 if (self->mf_allocator) {
261 self->mf_allocator->UninitializeSampleAllocator ();
262 self->mf_allocator->Release ();
263 self->mf_allocator = NULL;
264 }
265
266 gst_clear_object (&self->other_d3d11_device);
267 gst_clear_object (&self->d3d11_device);
268 #endif
269
270 return TRUE;
271 }
272
273 static gboolean
gst_mf_video_enc_start(GstVideoEncoder * enc)274 gst_mf_video_enc_start (GstVideoEncoder * enc)
275 {
276 /* Media Foundation Transform will shift PTS in case that B-frame is enabled.
277 * We need to adjust DTS correspondingly */
278 gst_video_encoder_set_min_pts (enc, GST_SECOND * 60 * 60 * 1000);
279
280 return TRUE;
281 }
282
283 static gboolean
gst_mf_video_enc_set_format(GstVideoEncoder * enc,GstVideoCodecState * state)284 gst_mf_video_enc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state)
285 {
286 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
287 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (enc);
288 GstVideoInfo *info = &state->info;
289 ComPtr < IMFMediaType > in_type;
290 ComPtr < IMFMediaType > out_type;
291 GList *input_types = NULL;
292 GList *iter;
293 HRESULT hr;
294 gint fps_n, fps_d;
295
296 GST_DEBUG_OBJECT (self, "Set format");
297
298 gst_mf_video_enc_finish (enc);
299
300 self->mf_pts_offset = 0;
301 self->has_reorder_frame = FALSE;
302 self->last_ret = GST_FLOW_OK;
303
304 if (self->input_state)
305 gst_video_codec_state_unref (self->input_state);
306 self->input_state = gst_video_codec_state_ref (state);
307
308 if (!gst_mf_transform_open (self->transform)) {
309 GST_ERROR_OBJECT (self, "Failed to open MFT");
310 return FALSE;
311 }
312 #if GST_MF_HAVE_D3D11
313 if (self->device_manager) {
314 if (!gst_mf_transform_set_device_manager (self->transform,
315 self->device_manager)) {
316 GST_ERROR_OBJECT (self, "Couldn't set device manager");
317 return FALSE;
318 } else {
319 GST_DEBUG_OBJECT (self, "set device manager done");
320 }
321 }
322 #endif
323
324 hr = MFCreateMediaType (out_type.GetAddressOf ());
325 if (!gst_mf_result (hr))
326 return FALSE;
327
328 hr = out_type->SetGUID (MF_MT_MAJOR_TYPE, MFMediaType_Video);
329 if (!gst_mf_result (hr))
330 return FALSE;
331
332 if (klass->set_option) {
333 if (!klass->set_option (self, self->input_state, out_type.Get ())) {
334 GST_ERROR_OBJECT (self, "subclass failed to set option");
335 return FALSE;
336 }
337 }
338
339 fps_n = GST_VIDEO_INFO_FPS_N (info);
340 fps_d = GST_VIDEO_INFO_FPS_D (info);
341 if (fps_n <= 0 || fps_d <= 0) {
342 /* XXX: not sure why. NVIDIA MFT accepts 0/1 framerate, but Intel or
343 * Microsoft's software MFT doesn't accept 0/1 framerate.
344 * Need to set something meaningful value here therefore */
345 fps_n = 25;
346 fps_d = 1;
347 }
348
349 hr = MFSetAttributeRatio (out_type.Get (), MF_MT_FRAME_RATE, fps_n, fps_d);
350 if (!gst_mf_result (hr)) {
351 GST_ERROR_OBJECT (self,
352 "Couldn't set framerate %d/%d, hr: 0x%x", (guint) hr);
353 return FALSE;
354 }
355
356 hr = MFSetAttributeSize (out_type.Get (), MF_MT_FRAME_SIZE,
357 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
358 if (!gst_mf_result (hr)) {
359 GST_ERROR_OBJECT (self,
360 "Couldn't set resolution %dx%d, hr: 0x%x", GST_VIDEO_INFO_WIDTH (info),
361 GST_VIDEO_INFO_HEIGHT (info), (guint) hr);
362 return FALSE;
363 }
364
365 hr = MFSetAttributeRatio (out_type.Get (), MF_MT_PIXEL_ASPECT_RATIO,
366 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
367 if (!gst_mf_result (hr)) {
368 GST_ERROR_OBJECT (self, "Couldn't set par %d/%d",
369 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
370 return FALSE;
371 }
372
373 hr = out_type->SetUINT32 (MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
374 if (!gst_mf_result (hr)) {
375 GST_ERROR_OBJECT (self,
376 "Couldn't set interlace mode, hr: 0x%x", (guint) hr);
377 return FALSE;
378 }
379
380 if (!gst_mf_transform_set_output_type (self->transform, out_type.Get ())) {
381 GST_ERROR_OBJECT (self, "Couldn't set output type");
382 return FALSE;
383 }
384
385 if (!gst_mf_transform_get_input_available_types (self->transform,
386 &input_types)) {
387 GST_ERROR_OBJECT (self, "Couldn't get available input types");
388 return FALSE;
389 }
390
391 for (iter = input_types; iter; iter = g_list_next (iter)) {
392 GstVideoFormat format;
393 GUID subtype;
394 IMFMediaType *type = (IMFMediaType *) iter->data;
395
396 hr = type->GetGUID (MF_MT_SUBTYPE, &subtype);
397 if (!gst_mf_result (hr))
398 continue;
399
400 format = gst_mf_video_subtype_to_video_format (&subtype);
401 if (format != GST_VIDEO_INFO_FORMAT (info))
402 continue;
403
404 in_type = type;
405 }
406
407 g_list_free_full (input_types, (GDestroyNotify) gst_mf_media_type_release);
408
409 if (!in_type) {
410 GST_ERROR_OBJECT (self,
411 "Couldn't convert input caps %" GST_PTR_FORMAT " to media type",
412 state->caps);
413 return FALSE;
414 }
415
416 hr = MFSetAttributeSize (in_type.Get (), MF_MT_FRAME_SIZE,
417 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
418 if (!gst_mf_result (hr)) {
419 GST_ERROR_OBJECT (self, "Couldn't set frame size %dx%d",
420 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
421 return FALSE;
422 }
423
424 hr = in_type->SetUINT32 (MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
425 if (!gst_mf_result (hr)) {
426 GST_ERROR_OBJECT (self,
427 "Couldn't set interlace mode, hr: 0x%x", (guint) hr);
428 return FALSE;
429 }
430
431 hr = MFSetAttributeRatio (in_type.Get (), MF_MT_PIXEL_ASPECT_RATIO,
432 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
433 if (!gst_mf_result (hr)) {
434 GST_ERROR_OBJECT (self, "Couldn't set par %d/%d",
435 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
436 return FALSE;
437 }
438
439 hr = MFSetAttributeRatio (in_type.Get (), MF_MT_FRAME_RATE, fps_n, fps_d);
440 if (!gst_mf_result (hr)) {
441 GST_ERROR_OBJECT (self, "Couldn't set framerate ratio %d/%d", fps_n, fps_d);
442 return FALSE;
443 }
444
445 hr = in_type->SetUINT32 (MF_MT_DEFAULT_STRIDE,
446 GST_VIDEO_INFO_PLANE_STRIDE (info, 0));
447 if (!gst_mf_result (hr)) {
448 GST_ERROR_OBJECT (self, "Couldn't set default stride");
449 return FALSE;
450 }
451
452 if (!gst_mf_transform_set_input_type (self->transform, in_type.Get ())) {
453 GST_ERROR_OBJECT (self, "Couldn't set input media type");
454 return FALSE;
455 }
456
457 g_assert (klass->set_src_caps != NULL);
458 if (!klass->set_src_caps (self, self->input_state, out_type.Get ())) {
459 GST_ERROR_OBJECT (self, "subclass couldn't set src caps");
460 return FALSE;
461 }
462 #if GST_MF_HAVE_D3D11
463 if (self->mf_allocator) {
464 self->mf_allocator->UninitializeSampleAllocator ();
465 self->mf_allocator->Release ();
466 self->mf_allocator = NULL;
467 }
468
469 /* Check whether upstream is d3d11 element */
470 if (state->caps) {
471 GstCapsFeatures *features;
472 ComPtr < IMFVideoSampleAllocatorEx > allocator;
473
474 features = gst_caps_get_features (state->caps, 0);
475
476 if (features &&
477 gst_caps_features_contains (features,
478 GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
479 GST_DEBUG_OBJECT (self, "found D3D11 memory feature");
480
481 hr = GstMFCreateVideoSampleAllocatorEx (IID_PPV_ARGS (&allocator));
482 if (!gst_mf_result (hr))
483 GST_WARNING_OBJECT (self,
484 "IMFVideoSampleAllocatorEx interface is unavailable");
485 }
486
487 if (allocator) {
488 do {
489 ComPtr < IMFAttributes > attr;
490
491 hr = MFCreateAttributes (&attr, 4);
492 if (!gst_mf_result (hr))
493 break;
494
495 /* Only one buffer per sample
496 * (multiple sample is usually for multi-view things) */
497 hr = attr->SetUINT32 (GST_GUID_MF_SA_BUFFERS_PER_SAMPLE, 1);
498 if (!gst_mf_result (hr))
499 break;
500
501 hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_USAGE, D3D11_USAGE_DEFAULT);
502 if (!gst_mf_result (hr))
503 break;
504
505 /* TODO: Check if we need to use keyed-mutex */
506 hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_SHARED_WITHOUT_MUTEX, TRUE);
507 if (!gst_mf_result (hr))
508 break;
509
510 hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_BINDFLAGS,
511 D3D11_BIND_VIDEO_ENCODER);
512 if (!gst_mf_result (hr))
513 break;
514
515 hr = allocator->SetDirectXManager (self->device_manager);
516 if (!gst_mf_result (hr))
517 break;
518
519 hr = allocator->InitializeSampleAllocatorEx (
520 /* min samples, since we are running on async mode,
521 * at least 2 samples would be required */
522 2,
523 /* max samples, why 16 + 2? it's just magic number
524 * (H264 max dpb size 16 + our min sample size 2) */
525 16 + 2, attr.Get (), in_type.Get ()
526 );
527
528 if (!gst_mf_result (hr))
529 break;
530
531 GST_DEBUG_OBJECT (self, "IMFVideoSampleAllocatorEx is initialized");
532
533 self->mf_allocator = allocator.Detach ();
534 } while (0);
535 }
536 }
537 #endif
538
539 return TRUE;
540 }
541
542 static void
gst_mf_video_buffer_free(GstVideoFrame * frame)543 gst_mf_video_buffer_free (GstVideoFrame * frame)
544 {
545 if (!frame)
546 return;
547
548 gst_video_frame_unmap (frame);
549 g_free (frame);
550 }
551
552 static gboolean
gst_mf_video_enc_frame_needs_copy(GstVideoFrame * vframe)553 gst_mf_video_enc_frame_needs_copy (GstVideoFrame * vframe)
554 {
555 /* Single plane data can be used without copy */
556 if (GST_VIDEO_FRAME_N_PLANES (vframe) == 1)
557 return FALSE;
558
559 switch (GST_VIDEO_FRAME_FORMAT (vframe)) {
560 case GST_VIDEO_FORMAT_I420:
561 {
562 guint8 *data, *other_data;
563 guint size;
564
565 /* Unexpected stride size, Media Foundation doesn't provide API for
566 * per plane stride information */
567 if (GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) !=
568 2 * GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1) ||
569 GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1) !=
570 GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 2)) {
571 return TRUE;
572 }
573
574 size = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) *
575 GST_VIDEO_FRAME_HEIGHT (vframe);
576 if (size + GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 0) !=
577 GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 1))
578 return TRUE;
579
580 data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0);
581 other_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 1);
582 if (data + size != other_data)
583 return TRUE;
584
585 size = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1) *
586 GST_VIDEO_FRAME_COMP_HEIGHT (vframe, 1);
587 if (size + GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 1) !=
588 GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 2))
589 return TRUE;
590
591 data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 1);
592 other_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 2);
593 if (data + size != other_data)
594 return TRUE;
595
596 return FALSE;
597 }
598 case GST_VIDEO_FORMAT_NV12:
599 case GST_VIDEO_FORMAT_P010_10LE:
600 case GST_VIDEO_FORMAT_P016_LE:
601 {
602 guint8 *data, *other_data;
603 guint size;
604
605 /* Unexpected stride size, Media Foundation doesn't provide API for
606 * per plane stride information */
607 if (GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) !=
608 GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1)) {
609 return TRUE;
610 }
611
612 size = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) *
613 GST_VIDEO_FRAME_HEIGHT (vframe);
614
615 /* Unexpected padding */
616 if (size + GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 0) !=
617 GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 1))
618 return TRUE;
619
620 data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0);
621 other_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 1);
622 if (data + size != other_data)
623 return TRUE;
624
625 return FALSE;
626 }
627 default:
628 g_assert_not_reached ();
629 return TRUE;
630 }
631
632 return TRUE;
633 }
634
635 typedef struct
636 {
637 LONGLONG mf_pts;
638 } GstMFVideoEncFrameData;
639
640 static gboolean
gst_mf_video_enc_process_input(GstMFVideoEnc * self,GstVideoCodecFrame * frame,IMFSample * sample)641 gst_mf_video_enc_process_input (GstMFVideoEnc * self,
642 GstVideoCodecFrame * frame, IMFSample * sample)
643 {
644 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (self);
645 HRESULT hr;
646 gboolean unset_force_keyframe = FALSE;
647 GstMFVideoEncFrameData *frame_data = NULL;
648 gboolean res;
649
650 frame_data = g_new0 (GstMFVideoEncFrameData, 1);
651 frame_data->mf_pts = frame->pts / 100;
652
653 gst_video_codec_frame_set_user_data (frame,
654 frame_data, (GDestroyNotify) g_free);
655
656 hr = sample->SetSampleTime (frame_data->mf_pts);
657 if (!gst_mf_result (hr))
658 return FALSE;
659
660 hr = sample->
661 SetSampleDuration (GST_CLOCK_TIME_IS_VALID (frame->duration) ? frame->
662 duration / 100 : 0);
663 if (!gst_mf_result (hr))
664 return FALSE;
665
666 if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) {
667 if (klass->device_caps.force_keyframe) {
668 unset_force_keyframe =
669 gst_mf_transform_set_codec_api_uint32 (self->transform,
670 &CODECAPI_AVEncVideoForceKeyFrame, TRUE);
671 } else {
672 GST_WARNING_OBJECT (self, "encoder does not support force keyframe");
673 }
674 }
675
676 /* Unlock temporary so that we can output frame from Media Foundation's
677 * worker thread.
678 * While we are processing input, MFT might notify
679 * METransformHaveOutput event from Media Foundation's internal worker queue
680 * thread. Then we will output encoded data from the thread synchroniously,
681 * not from streaming (this) thread */
682 if (self->async_mft)
683 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
684 res = gst_mf_transform_process_input (self->transform, sample);
685 if (self->async_mft)
686 GST_VIDEO_ENCODER_STREAM_LOCK (self);
687
688 if (unset_force_keyframe) {
689 gst_mf_transform_set_codec_api_uint32 (self->transform,
690 &CODECAPI_AVEncVideoForceKeyFrame, FALSE);
691 }
692
693 if (!res) {
694 GST_ERROR_OBJECT (self, "Failed to process input");
695 return FALSE;
696 }
697
698 return TRUE;
699 }
700
701 static GstVideoCodecFrame *
gst_mf_video_enc_find_output_frame(GstMFVideoEnc * self,LONGLONG mf_pts)702 gst_mf_video_enc_find_output_frame (GstMFVideoEnc * self, LONGLONG mf_pts)
703 {
704 GList *l, *walk = gst_video_encoder_get_frames (GST_VIDEO_ENCODER (self));
705 GstVideoCodecFrame *ret = NULL;
706 GstVideoCodecFrame *closest = NULL;
707 LONGLONG min_pts_abs_diff = 0;
708
709 for (l = walk; l; l = l->next) {
710 GstVideoCodecFrame *frame = (GstVideoCodecFrame *) l->data;
711 GstMFVideoEncFrameData *data = (GstMFVideoEncFrameData *)
712 gst_video_codec_frame_get_user_data (frame);
713 LONGLONG abs_diff;
714
715 if (!data)
716 continue;
717
718 if (mf_pts == data->mf_pts) {
719 ret = frame;
720 break;
721 }
722
723 abs_diff = std::abs (mf_pts - data->mf_pts);
724
725 if (!closest || abs_diff < min_pts_abs_diff) {
726 closest = frame;
727 min_pts_abs_diff = abs_diff;
728 }
729 }
730
731 if (!ret && closest)
732 ret = closest;
733
734 if (ret) {
735 gst_video_codec_frame_ref (ret);
736 } else {
737 /* XXX: Shouldn't happen, but possible if no GstVideoCodecFrame holds
738 * user data for some reasons */
739 GST_WARNING_OBJECT (self,
740 "Failed to find closest GstVideoCodecFrame with MF pts %"
741 G_GINT64_FORMAT, mf_pts);
742 ret = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER (self));
743 }
744
745 if (walk)
746 g_list_free_full (walk, (GDestroyNotify) gst_video_codec_frame_unref);
747
748 return ret;
749 }
750
751 static HRESULT
gst_mf_video_enc_finish_sample(GstMFVideoEnc * self,IMFSample * sample)752 gst_mf_video_enc_finish_sample (GstMFVideoEnc * self, IMFSample * sample)
753 {
754 HRESULT hr = S_OK;
755 BYTE *data;
756 ComPtr < IMFMediaBuffer > media_buffer;
757 GstBuffer *buffer;
758 GstFlowReturn res = GST_FLOW_ERROR;
759 GstVideoCodecFrame *frame;
760 LONGLONG sample_timestamp;
761 LONGLONG sample_duration;
762 LONGLONG target_mf_pts;
763 UINT64 mf_dts;
764 UINT32 keyframe = FALSE;
765 DWORD buffer_len;
766 GstClockTime pts, dts, duration;
767
768 hr = sample->GetBufferByIndex (0, media_buffer.GetAddressOf ());
769 if (!gst_mf_result (hr))
770 goto done;
771
772 hr = media_buffer->Lock (&data, NULL, &buffer_len);
773 if (!gst_mf_result (hr))
774 goto done;
775
776 buffer = gst_buffer_new_allocate (NULL, buffer_len, NULL);
777 gst_buffer_fill (buffer, 0, data, buffer_len);
778 media_buffer->Unlock ();
779
780 sample->GetSampleTime (&sample_timestamp);
781 target_mf_pts = sample_timestamp;
782 sample->GetSampleDuration (&sample_duration);
783 sample->GetUINT32 (MFSampleExtension_CleanPoint, &keyframe);
784
785 hr = sample->GetUINT64 (MFSampleExtension_DecodeTimestamp, &mf_dts);
786 if (FAILED (hr)) {
787 mf_dts = sample_timestamp;
788 hr = S_OK;
789 }
790
791 pts = sample_timestamp * 100;
792 dts = mf_dts * 100;
793 duration = sample_duration * 100;
794
795 GST_LOG_OBJECT (self, "Finish sample, MF pts %" GST_TIME_FORMAT " MF dts %"
796 GST_TIME_FORMAT ", MF duration %" GST_TIME_FORMAT,
797 GST_TIME_ARGS (pts), GST_TIME_ARGS (dts), GST_TIME_ARGS (duration));
798
799 /* NOTE: When B-frame is enabled, MFT shows following pattern
800 * (input timestamp starts from 1000:00:00.000000000, and 30fps)
801 *
802 * Frame-1: MF pts 0:00.033333300 MF dts 0:00.000000000
803 * Frame-2: MF pts 0:00.133333300 MF dts 0:00.033333300
804 * Frame-3: MF pts 0:00.066666600 MF dts 0:00.066666600
805 * Frame-4: MF pts 0:00.099999900 MF dts 0:00.100000000
806 *
807 * - Sounds MFT doesn't support negative timestamp, so PTS of each frame seems
808 * to be shifthed
809 * - DTS is likely based on timestamp we've set to input sample,
810 * but some frames has (especially Frame-4 case) unexpected PTS and
811 * even PTS < DTS. That would be the result of PTS shifting
812 *
813 * To handle this case,
814 * - Calculate timestamp offset "Frame-1 PTS" - "Frame-1 DTS" (== duration),
815 * and compensate PTS/DTS of each frame
816 * - Needs additional offset for DTS to compenstate GST/MF timescale difference
817 * (MF uses 100ns timescale). So DTS offset should be "PTS offset + 100ns"
818 * - Find corresponding GstVideoCodecFrame by using compensated PTS.
819 * Note that MFT doesn't support user-data for tracing input/output sample
820 * pair. So, timestamp based lookup is the only way to map MF sample
821 * and our GstVideoCodecFrame
822 */
823 if (self->has_reorder_frame) {
824 /* This would be the first frame */
825 if (self->mf_pts_offset == 0) {
826 LONGLONG mf_pts_offset = -1;
827 if (sample_timestamp > mf_dts) {
828 mf_pts_offset = sample_timestamp - mf_dts;
829 GST_DEBUG_OBJECT (self, "Calculates PTS offset using \"PTS - DTS\": %"
830 G_GINT64_FORMAT, mf_pts_offset);
831 } else if (sample_duration > 0) {
832 mf_pts_offset = sample_duration;
833 GST_DEBUG_OBJECT (self, "Calculates PTS offset using duration: %"
834 G_GINT64_FORMAT, mf_pts_offset);
835 } else {
836 GST_WARNING_OBJECT (self, "Cannot calculate PTS offset");
837 }
838
839 self->mf_pts_offset = mf_pts_offset;
840 }
841
842 if (self->mf_pts_offset > 0) {
843 target_mf_pts -= self->mf_pts_offset;
844
845 pts -= (self->mf_pts_offset * 100);
846 /* +1 to compensate timescale difference */
847 dts -= ((self->mf_pts_offset + 1) * 100);
848 }
849 }
850
851 frame = gst_mf_video_enc_find_output_frame (self, target_mf_pts);
852
853 if (frame) {
854 if (keyframe) {
855 GST_DEBUG_OBJECT (self, "Keyframe pts %" GST_TIME_FORMAT,
856 GST_TIME_ARGS (frame->pts));
857 GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame);
858 }
859
860 frame->output_buffer = buffer;
861
862 /* Update DTS only if B-frame was enabled, but use input frame pts as-is.
863 * Otherwise we will lost at most 100ns precision */
864 if (self->has_reorder_frame) {
865 frame->dts = dts;
866 } else {
867 frame->dts = frame->pts;
868 }
869
870 /* make sure PTS > DTS */
871 if (GST_CLOCK_TIME_IS_VALID (frame->pts) &&
872 GST_CLOCK_TIME_IS_VALID (frame->dts) && frame->pts < frame->dts) {
873 GST_WARNING_OBJECT (self, "Calculated DTS %" GST_TIME_FORMAT
874 " is larger than PTS %" GST_TIME_FORMAT, GST_TIME_ARGS (frame->pts),
875 GST_TIME_ARGS (frame->dts));
876
877 /* XXX: just set clock-time-none? */
878 frame->dts = frame->pts;
879 }
880
881 GST_LOG_OBJECT (self, "Frame pts %" GST_TIME_FORMAT ", Frame DTS %"
882 GST_TIME_FORMAT, GST_TIME_ARGS (frame->pts),
883 GST_TIME_ARGS (frame->dts));
884
885 res = gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (self), frame);
886 } else {
887 GST_BUFFER_PTS (buffer) = pts;
888 GST_BUFFER_DTS (buffer) = dts;
889 GST_BUFFER_DURATION (buffer) = duration;
890
891 if (keyframe) {
892 GST_DEBUG_OBJECT (self, "Keyframe pts %" GST_TIME_FORMAT,
893 GST_BUFFER_PTS (buffer));
894 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
895 } else {
896 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
897 }
898
899 GST_LOG_OBJECT (self, "Buffer pts %" GST_TIME_FORMAT ", Buffer DTS %"
900 GST_TIME_FORMAT, GST_TIME_ARGS (pts), GST_TIME_ARGS (dts));
901
902 res = gst_pad_push (GST_VIDEO_ENCODER_SRC_PAD (self), buffer);
903 }
904
905 done:
906 self->last_ret = res;
907
908 return hr;
909 }
910
911 static GstFlowReturn
gst_mf_video_enc_process_output(GstMFVideoEnc * self)912 gst_mf_video_enc_process_output (GstMFVideoEnc * self)
913 {
914 ComPtr < IMFSample > sample;
915 GstFlowReturn res = GST_FLOW_ERROR;
916
917 res = gst_mf_transform_get_output (self->transform, &sample);
918
919 if (res != GST_FLOW_OK)
920 return res;
921
922 gst_mf_video_enc_finish_sample (self, sample.Get ());
923
924 return self->last_ret;
925 }
926
927 static gboolean
gst_mf_video_enc_create_input_sample(GstMFVideoEnc * self,GstVideoCodecFrame * frame,IMFSample ** sample)928 gst_mf_video_enc_create_input_sample (GstMFVideoEnc * self,
929 GstVideoCodecFrame * frame, IMFSample ** sample)
930 {
931 HRESULT hr;
932 ComPtr < IMFSample > new_sample;
933 ComPtr < IMFMediaBuffer > media_buffer;
934 ComPtr < IGstMFVideoBuffer > video_buffer;
935 GstVideoInfo *info = &self->input_state->info;
936 gint i, j;
937 GstVideoFrame *vframe = NULL;
938 BYTE *data = NULL;
939 gboolean need_copy;
940
941 vframe = g_new0 (GstVideoFrame, 1);
942
943 if (!gst_video_frame_map (vframe, info, frame->input_buffer, GST_MAP_READ)) {
944 GST_ERROR_OBJECT (self, "Couldn't map input frame");
945 g_free (vframe);
946 return FALSE;
947 }
948
949 hr = MFCreateSample (&new_sample);
950 if (!gst_mf_result (hr))
951 goto error;
952
953 /* Check if we can forward this memory to Media Foundation without copy */
954 need_copy = gst_mf_video_enc_frame_needs_copy (vframe);
955 if (need_copy) {
956 GST_TRACE_OBJECT (self, "Copy input buffer into Media Foundation memory");
957 hr = MFCreateMemoryBuffer (GST_VIDEO_INFO_SIZE (info), &media_buffer);
958 } else {
959 GST_TRACE_OBJECT (self, "Can use input buffer without copy");
960 hr = IGstMFVideoBuffer::CreateInstanceWrapped (&vframe->info,
961 (BYTE *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0),
962 GST_VIDEO_INFO_SIZE (&vframe->info), &media_buffer);
963 }
964
965 if (!gst_mf_result (hr))
966 goto error;
967
968 if (!need_copy) {
969 hr = media_buffer.As (&video_buffer);
970 if (!gst_mf_result (hr))
971 goto error;
972 } else {
973 hr = media_buffer->Lock (&data, NULL, NULL);
974 if (!gst_mf_result (hr))
975 goto error;
976
977 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
978 guint8 *src, *dst;
979 gint src_stride, dst_stride;
980 gint width;
981
982 src = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, i);
983 dst = data + GST_VIDEO_INFO_PLANE_OFFSET (info, i);
984
985 src_stride = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, i);
986 dst_stride = GST_VIDEO_INFO_PLANE_STRIDE (info, i);
987
988 width = GST_VIDEO_INFO_COMP_WIDTH (info, i)
989 * GST_VIDEO_INFO_COMP_PSTRIDE (info, i);
990
991 for (j = 0; j < GST_VIDEO_INFO_COMP_HEIGHT (info, i); j++) {
992 memcpy (dst, src, width);
993 src += src_stride;
994 dst += dst_stride;
995 }
996 }
997
998 media_buffer->Unlock ();
999 }
1000
1001 hr = media_buffer->SetCurrentLength (GST_VIDEO_INFO_SIZE (info));
1002 if (!gst_mf_result (hr))
1003 goto error;
1004
1005 hr = new_sample->AddBuffer (media_buffer.Get ());
1006 if (!gst_mf_result (hr))
1007 goto error;
1008
1009 if (!need_copy) {
1010 /* IGstMFVideoBuffer will hold GstVideoFrame (+ GstBuffer), then it will be
1011 * cleared when it's no more referenced by Media Foundation internals */
1012 hr = video_buffer->SetUserData ((gpointer) vframe,
1013 (GDestroyNotify) gst_mf_video_buffer_free);
1014 if (!gst_mf_result (hr))
1015 goto error;
1016 } else {
1017 gst_video_frame_unmap (vframe);
1018 g_free (vframe);
1019 vframe = NULL;
1020 }
1021
1022 *sample = new_sample.Detach ();
1023
1024 return TRUE;
1025
1026 error:
1027 if (vframe) {
1028 gst_video_frame_unmap (vframe);
1029 g_free (vframe);
1030 }
1031
1032 return FALSE;
1033 }
1034
1035 #if GST_MF_HAVE_D3D11
1036 static gboolean
gst_mf_video_enc_create_input_sample_d3d11(GstMFVideoEnc * self,GstVideoCodecFrame * frame,IMFSample ** sample)1037 gst_mf_video_enc_create_input_sample_d3d11 (GstMFVideoEnc * self,
1038 GstVideoCodecFrame * frame, IMFSample ** sample)
1039 {
1040 HRESULT hr;
1041 ComPtr < IMFSample > new_sample;
1042 ComPtr < IMFMediaBuffer > mf_buffer;
1043 ComPtr < IMFDXGIBuffer > dxgi_buffer;
1044 ComPtr < ID3D11Texture2D > mf_texture;
1045 ComPtr < IDXGIResource > dxgi_resource;
1046 ComPtr < ID3D11Texture2D > shared_texture;
1047 ComPtr < ID3D11Query > query;
1048 D3D11_QUERY_DESC query_desc;
1049 BOOL sync_done = FALSE;
1050 HANDLE shared_handle;
1051 GstMemory *mem;
1052 GstD3D11Memory *dmem;
1053 ID3D11Texture2D *texture;
1054 ID3D11Device *device_handle;
1055 ID3D11DeviceContext *context_handle;
1056 GstMapInfo info;
1057 D3D11_BOX src_box = { 0, };
1058 D3D11_TEXTURE2D_DESC dst_desc, src_desc;
1059 guint subidx;
1060
1061 if (!self->mf_allocator) {
1062 GST_WARNING_OBJECT (self, "IMFVideoSampleAllocatorEx was configured");
1063 return FALSE;
1064 }
1065
1066 mem = gst_buffer_peek_memory (frame->input_buffer, 0);
1067 if (!gst_is_d3d11_memory (mem)) {
1068 GST_WARNING_OBJECT (self, "Non-d3d11 memory");
1069 return FALSE;
1070 }
1071
1072 dmem = (GstD3D11Memory *) mem;
1073 device_handle = gst_d3d11_device_get_device_handle (dmem->device);
1074 context_handle = gst_d3d11_device_get_device_context_handle (dmem->device);
1075
1076 /* 1) Allocate new encoding surface */
1077 hr = self->mf_allocator->AllocateSample (&new_sample);
1078 if (!gst_mf_result (hr)) {
1079 GST_WARNING_OBJECT (self,
1080 "Couldn't allocate new sample via IMFVideoSampleAllocatorEx");
1081 return FALSE;
1082 }
1083
1084 hr = new_sample->GetBufferByIndex (0, &mf_buffer);
1085 if (!gst_mf_result (hr)) {
1086 GST_WARNING_OBJECT (self, "Couldn't get IMFMediaBuffer from sample");
1087 return FALSE;
1088 }
1089
1090 hr = mf_buffer.As (&dxgi_buffer);
1091 if (!gst_mf_result (hr)) {
1092 GST_WARNING_OBJECT (self, "Couldn't get IMFDXGIBuffer from IMFMediaBuffer");
1093 return FALSE;
1094 }
1095
1096 hr = dxgi_buffer->GetResource (IID_PPV_ARGS (&mf_texture));
1097 if (!gst_mf_result (hr)) {
1098 GST_WARNING_OBJECT (self,
1099 "Couldn't get ID3D11Texture2D from IMFDXGIBuffer");
1100 return FALSE;
1101 }
1102
1103 hr = mf_texture.As (&dxgi_resource);
1104 if (!gst_mf_result (hr)) {
1105 GST_WARNING_OBJECT (self,
1106 "Couldn't get IDXGIResource from ID3D11Texture2D");
1107 return FALSE;
1108 }
1109
1110 hr = dxgi_resource->GetSharedHandle (&shared_handle);
1111 if (!gst_mf_result (hr)) {
1112 GST_WARNING_OBJECT (self, "Couldn't get shared handle from IDXGIResource");
1113 return FALSE;
1114 }
1115
1116 /* Allocation succeeded. Now open shared texture to access it from
1117 * other device */
1118 hr = device_handle->OpenSharedResource (shared_handle,
1119 IID_PPV_ARGS (&shared_texture));
1120 if (!gst_mf_result (hr)) {
1121 GST_WARNING_OBJECT (self, "Couldn't open shared resource");
1122 return FALSE;
1123 }
1124
1125 /* 2) Copy upstream texture to mf's texture */
1126 /* Map memory so that ensure pending upload from staging texture */
1127 if (!gst_memory_map (mem, &info,
1128 (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) {
1129 GST_ERROR_OBJECT (self, "Couldn't map d3d11 memory");
1130 return FALSE;
1131 }
1132
1133 texture = (ID3D11Texture2D *) info.data;
1134 texture->GetDesc (&src_desc);
1135 shared_texture->GetDesc (&dst_desc);
1136 subidx = gst_d3d11_memory_get_subresource_index (dmem);
1137
1138 /* src/dst texture size might be different if padding was used.
1139 * select smaller size */
1140 src_box.left = 0;
1141 src_box.top = 0;
1142 src_box.front = 0;
1143 src_box.back = 1;
1144 src_box.right = MIN (src_desc.Width, dst_desc.Width);
1145 src_box.bottom = MIN (src_desc.Height, dst_desc.Height);
1146
1147 /* CopySubresourceRegion() might not be able to guarantee
1148 * copying. To ensure it, we will make use of d3d11 query */
1149 query_desc.Query = D3D11_QUERY_EVENT;
1150 query_desc.MiscFlags = 0;
1151
1152 hr = device_handle->CreateQuery (&query_desc, &query);
1153 if (!gst_d3d11_result (hr, dmem->device)) {
1154 GST_ERROR_OBJECT (self, "Couldn't Create event query");
1155 return FALSE;
1156 }
1157
1158 gst_d3d11_device_lock (dmem->device);
1159 context_handle->CopySubresourceRegion (shared_texture.Get (), 0, 0, 0, 0,
1160 texture, subidx, &src_box);
1161 context_handle->End (query.Get ());
1162
1163 /* Wait until all issued GPU commands are finished */
1164 do {
1165 context_handle->GetData (query.Get (), &sync_done, sizeof (BOOL), 0);
1166 } while (!sync_done && (hr == S_OK || hr == S_FALSE));
1167
1168 if (!gst_d3d11_result (hr, dmem->device)) {
1169 GST_ERROR_OBJECT (self, "Couldn't sync GPU operation");
1170 gst_d3d11_device_unlock (dmem->device);
1171 gst_memory_unmap (mem, &info);
1172
1173 return FALSE;
1174 }
1175
1176 gst_d3d11_device_unlock (dmem->device);
1177 gst_memory_unmap (mem, &info);
1178
1179 *sample = new_sample.Detach ();
1180
1181 return TRUE;
1182 }
1183 #endif
1184
1185 static GstFlowReturn
gst_mf_video_enc_handle_frame(GstVideoEncoder * enc,GstVideoCodecFrame * frame)1186 gst_mf_video_enc_handle_frame (GstVideoEncoder * enc,
1187 GstVideoCodecFrame * frame)
1188 {
1189 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1190 GstFlowReturn ret = GST_FLOW_OK;
1191 ComPtr < IMFSample > sample;
1192
1193 if (self->last_ret != GST_FLOW_OK) {
1194 GST_DEBUG_OBJECT (self, "Last return was %s", gst_flow_get_name (ret));
1195 ret = self->last_ret;
1196 goto done;
1197 }
1198 #if GST_MF_HAVE_D3D11
1199 if (self->mf_allocator &&
1200 !gst_mf_video_enc_create_input_sample_d3d11 (self, frame, &sample)) {
1201 GST_WARNING_OBJECT (self, "Failed to create IMFSample for D3D11");
1202 sample = nullptr;
1203 }
1204 #endif
1205
1206 if (!sample && !gst_mf_video_enc_create_input_sample (self, frame, &sample)) {
1207 GST_ERROR_OBJECT (self, "Failed to create IMFSample");
1208 ret = GST_FLOW_ERROR;
1209 goto done;
1210 }
1211
1212 if (!gst_mf_video_enc_process_input (self, frame, sample.Get ())) {
1213 GST_ERROR_OBJECT (self, "Failed to process input");
1214 ret = GST_FLOW_ERROR;
1215 goto done;
1216 }
1217
1218 /* Don't call process_output for async (hardware) MFT. We will output
1219 * encoded data from gst_mf_video_on_new_sample() callback which is called
1220 * from Media Foundation's internal worker queue thread */
1221 if (!self->async_mft) {
1222 do {
1223 ret = gst_mf_video_enc_process_output (self);
1224 } while (ret == GST_FLOW_OK);
1225 }
1226
1227 if (ret == GST_MF_TRANSFORM_FLOW_NEED_DATA)
1228 ret = GST_FLOW_OK;
1229
1230 done:
1231 gst_video_codec_frame_unref (frame);
1232
1233 return ret;
1234 }
1235
1236 static GstFlowReturn
gst_mf_video_enc_finish(GstVideoEncoder * enc)1237 gst_mf_video_enc_finish (GstVideoEncoder * enc)
1238 {
1239 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1240 GstFlowReturn ret = GST_FLOW_OK;
1241
1242 if (!self->transform)
1243 return GST_FLOW_OK;
1244
1245 /* Unlock temporary so that we can output frame from Media Foundation's
1246 * worker thread */
1247 if (self->async_mft)
1248 GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
1249
1250 gst_mf_transform_drain (self->transform);
1251
1252 if (self->async_mft)
1253 GST_VIDEO_ENCODER_STREAM_LOCK (enc);
1254
1255 if (!self->async_mft) {
1256 do {
1257 ret = gst_mf_video_enc_process_output (self);
1258 } while (ret == GST_FLOW_OK);
1259 }
1260
1261 if (ret == GST_MF_TRANSFORM_FLOW_NEED_DATA)
1262 ret = GST_FLOW_OK;
1263
1264 return ret;
1265 }
1266
1267 static gboolean
gst_mf_video_enc_flush(GstVideoEncoder * enc)1268 gst_mf_video_enc_flush (GstVideoEncoder * enc)
1269 {
1270 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1271
1272 if (!self->transform)
1273 goto out;
1274
1275 /* Unlock while flushing, while flushing, new sample callback might happen */
1276 if (self->async_mft)
1277 GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
1278
1279 gst_mf_transform_flush (self->transform);
1280
1281 if (self->async_mft)
1282 GST_VIDEO_ENCODER_STREAM_LOCK (enc);
1283
1284 out:
1285 self->last_ret = GST_FLOW_OK;
1286
1287 return TRUE;
1288 }
1289
1290 static gboolean
gst_mf_video_enc_propose_allocation(GstVideoEncoder * enc,GstQuery * query)1291 gst_mf_video_enc_propose_allocation (GstVideoEncoder * enc, GstQuery * query)
1292 {
1293 #if GST_MF_HAVE_D3D11
1294 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1295 GstVideoInfo info;
1296 GstBufferPool *pool = NULL;
1297 GstCaps *caps;
1298 guint size;
1299 GstD3D11Device *device = self->other_d3d11_device;
1300
1301 gst_query_parse_allocation (query, &caps, NULL);
1302
1303 if (caps == NULL)
1304 return FALSE;
1305
1306 if (!gst_video_info_from_caps (&info, caps))
1307 return FALSE;
1308
1309 if (gst_query_get_n_allocation_pools (query) == 0) {
1310 GstCapsFeatures *features;
1311 GstStructure *config;
1312 gboolean is_d3d11 = FALSE;
1313
1314 features = gst_caps_get_features (caps, 0);
1315
1316 if (features && gst_caps_features_contains (features,
1317 GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
1318 GST_DEBUG_OBJECT (self, "Allocation caps supports d3d11 memory");
1319 pool = gst_d3d11_buffer_pool_new (device);
1320 is_d3d11 = TRUE;
1321 } else {
1322 pool = gst_video_buffer_pool_new ();
1323 }
1324
1325 config = gst_buffer_pool_get_config (pool);
1326
1327 gst_buffer_pool_config_add_option (config,
1328 GST_BUFFER_POOL_OPTION_VIDEO_META);
1329
1330 /* d3d11 pool does not support video alignment */
1331 if (!is_d3d11) {
1332 gst_buffer_pool_config_add_option (config,
1333 GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
1334 } else {
1335 GstD3D11AllocationParams *d3d11_params;
1336 guint misc_flags = 0;
1337 gboolean is_hardware = FALSE;
1338 gint i;
1339
1340 g_object_get (device, "hardware", &is_hardware, NULL);
1341
1342 /* In case of hardware, set D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag
1343 * so that it can be shared with other d3d11 devices */
1344 if (is_hardware)
1345 misc_flags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
1346
1347 d3d11_params =
1348 gst_buffer_pool_config_get_d3d11_allocation_params (config);
1349 if (!d3d11_params) {
1350 d3d11_params = gst_d3d11_allocation_params_new (device, &info,
1351 (GstD3D11AllocationFlags) 0, 0);
1352 } else {
1353 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++)
1354 d3d11_params->desc[i].MiscFlags |= misc_flags;
1355 }
1356
1357 gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
1358 gst_d3d11_allocation_params_free (d3d11_params);
1359 }
1360
1361 size = GST_VIDEO_INFO_SIZE (&info);
1362 gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
1363
1364 if (!gst_buffer_pool_set_config (pool, config))
1365 goto config_failed;
1366
1367 /* d3d11 buffer pool will update buffer size based on allocated texture,
1368 * get size from config again */
1369 if (is_d3d11) {
1370 config = gst_buffer_pool_get_config (pool);
1371 gst_buffer_pool_config_get_params (config,
1372 nullptr, &size, nullptr, nullptr);
1373 gst_structure_free (config);
1374 }
1375
1376 gst_query_add_allocation_pool (query, pool, size, 0, 0);
1377 gst_object_unref (pool);
1378 }
1379
1380 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
1381
1382 return TRUE;
1383
1384 /* ERRORS */
1385 config_failed:
1386 {
1387 GST_ERROR_OBJECT (self, "failed to set config");
1388 gst_object_unref (pool);
1389 return FALSE;
1390 }
1391
1392 #else
1393 return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (enc,
1394 query);
1395 #endif
1396 }
1397
1398 static gboolean
gst_mf_video_enc_sink_query(GstVideoEncoder * enc,GstQuery * query)1399 gst_mf_video_enc_sink_query (GstVideoEncoder * enc, GstQuery * query)
1400 {
1401 #if GST_MF_HAVE_D3D11
1402 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1403
1404 switch (GST_QUERY_TYPE (query)) {
1405 case GST_QUERY_CONTEXT:
1406 if (gst_d3d11_handle_context_query (GST_ELEMENT (self),
1407 query, self->other_d3d11_device)) {
1408 return TRUE;
1409 }
1410 break;
1411 default:
1412 break;
1413 }
1414 #endif
1415
1416 return GST_VIDEO_ENCODER_CLASS (parent_class)->sink_query (enc, query);
1417 }
1418
1419 static gboolean
gst_mf_video_enc_src_query(GstVideoEncoder * enc,GstQuery * query)1420 gst_mf_video_enc_src_query (GstVideoEncoder * enc, GstQuery * query)
1421 {
1422 #if GST_MF_HAVE_D3D11
1423 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1424
1425 switch (GST_QUERY_TYPE (query)) {
1426 case GST_QUERY_CONTEXT:
1427 if (gst_d3d11_handle_context_query (GST_ELEMENT (self),
1428 query, self->other_d3d11_device)) {
1429 return TRUE;
1430 }
1431 break;
1432 default:
1433 break;
1434 }
1435 #endif
1436
1437 return GST_VIDEO_ENCODER_CLASS (parent_class)->src_query (enc, query);
1438 }
1439
1440 static HRESULT
gst_mf_video_on_new_sample(GstMFTransform * object,IMFSample * sample,GstMFVideoEnc * self)1441 gst_mf_video_on_new_sample (GstMFTransform * object,
1442 IMFSample * sample, GstMFVideoEnc * self)
1443 {
1444 GST_LOG_OBJECT (self, "New Sample callback");
1445
1446 /* NOTE: this callback will be called from Media Foundation's internal
1447 * worker queue thread */
1448 GST_VIDEO_ENCODER_STREAM_LOCK (self);
1449 gst_mf_video_enc_finish_sample (self, sample);
1450 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
1451
1452 return S_OK;
1453 }
1454
1455 typedef struct
1456 {
1457 guint profile;
1458 const gchar *profile_str;
1459 } GstMFVideoEncProfileMap;
1460
1461 static void
gst_mf_video_enc_enum_internal(GstMFTransform * transform,GUID & subtype,GstObject * d3d11_device,GstMFVideoEncDeviceCaps * device_caps,GstCaps ** sink_template,GstCaps ** src_template)1462 gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID & subtype,
1463 GstObject * d3d11_device, GstMFVideoEncDeviceCaps * device_caps,
1464 GstCaps ** sink_template, GstCaps ** src_template)
1465 {
1466 HRESULT hr;
1467 MFT_REGISTER_TYPE_INFO *infos;
1468 UINT32 info_size;
1469 gint i;
1470 GstCaps *src_caps = NULL;
1471 GstCaps *sink_caps = NULL;
1472 GstCaps *d3d11_caps = NULL;
1473 GValue *supported_formats = NULL;
1474 GValue *profiles = NULL;
1475 gboolean have_I420 = FALSE;
1476 gboolean have_NV12 = FALSE;
1477 gboolean have_P010 = FALSE;
1478 gboolean d3d11_aware = FALSE;
1479 gchar *device_name = NULL;
1480 IMFActivate *activate;
1481 IMFTransform *encoder;
1482 ICodecAPI *codec_api;
1483 ComPtr < IMFMediaType > out_type;
1484 GstMFVideoEncProfileMap h264_profile_map[] = {
1485 {eAVEncH264VProfile_High, "high"},
1486 {eAVEncH264VProfile_Main, "main"},
1487 {eAVEncH264VProfile_Base, "baseline"},
1488 {0, NULL},
1489 };
1490 GstMFVideoEncProfileMap hevc_profile_map[] = {
1491 {eAVEncH265VProfile_Main_420_8, "main"},
1492 {eAVEncH265VProfile_Main_420_10, "main-10"},
1493 {0, NULL},
1494 };
1495 GstMFVideoEncProfileMap *profile_to_check = NULL;
1496 static const gchar *h264_caps_str =
1497 "video/x-h264, stream-format=(string) byte-stream, alignment=(string) au";
1498 static const gchar *hevc_caps_str =
1499 "video/x-h265, stream-format=(string) byte-stream, alignment=(string) au";
1500 static const gchar *vp9_caps_str = "video/x-vp9";
1501 const gchar *codec_caps_str = NULL;
1502
1503 /* NOTE: depending on environment,
1504 * some enumerated h/w MFT might not be usable (e.g., multiple GPU case) */
1505 if (!gst_mf_transform_open (transform))
1506 return;
1507
1508 activate = gst_mf_transform_get_activate_handle (transform);
1509 if (!activate) {
1510 GST_WARNING_OBJECT (transform, "No IMFActivate interface available");
1511 return;
1512 }
1513
1514 encoder = gst_mf_transform_get_transform_handle (transform);
1515 if (!encoder) {
1516 GST_WARNING_OBJECT (transform, "No IMFTransform interface available");
1517 return;
1518 }
1519
1520 codec_api = gst_mf_transform_get_codec_api_handle (transform);
1521 if (!codec_api) {
1522 GST_WARNING_OBJECT (transform, "No ICodecAPI interface available");
1523 return;
1524 }
1525
1526 g_object_get (transform, "device-name", &device_name, NULL);
1527 if (!device_name) {
1528 GST_WARNING_OBJECT (transform, "Unknown device name");
1529 return;
1530 }
1531 g_free (device_name);
1532
1533 hr = activate->GetAllocatedBlob (MFT_INPUT_TYPES_Attributes,
1534 (UINT8 **) & infos, &info_size);
1535 if (!gst_mf_result (hr))
1536 return;
1537
1538 for (i = 0; i < info_size / sizeof (MFT_REGISTER_TYPE_INFO); i++) {
1539 GstVideoFormat format;
1540 const GstVideoFormatInfo *format_info;
1541 GValue val = G_VALUE_INIT;
1542
1543 format = gst_mf_video_subtype_to_video_format (&infos[i].guidSubtype);
1544 if (format == GST_VIDEO_FORMAT_UNKNOWN)
1545 continue;
1546
1547 format_info = gst_video_format_get_info (format);
1548 if (GST_VIDEO_FORMAT_INFO_IS_RGB (format_info)) {
1549 GST_DEBUG_OBJECT (transform, "Skip %s format",
1550 GST_VIDEO_FORMAT_INFO_NAME (format_info));
1551 continue;
1552 }
1553
1554 if (!supported_formats) {
1555 supported_formats = g_new0 (GValue, 1);
1556 g_value_init (supported_formats, GST_TYPE_LIST);
1557 }
1558
1559 switch (format) {
1560 /* media foundation has duplicated formats IYUV and I420 */
1561 case GST_VIDEO_FORMAT_I420:
1562 if (have_I420)
1563 continue;
1564
1565 have_I420 = TRUE;
1566 break;
1567 case GST_VIDEO_FORMAT_NV12:
1568 have_NV12 = TRUE;
1569 break;
1570 case GST_VIDEO_FORMAT_P010_10LE:
1571 have_P010 = TRUE;
1572 break;
1573 default:
1574 break;
1575 }
1576
1577 g_value_init (&val, G_TYPE_STRING);
1578 g_value_set_static_string (&val, gst_video_format_to_string (format));
1579 gst_value_list_append_and_take_value (supported_formats, &val);
1580 }
1581 CoTaskMemFree (infos);
1582
1583 if (!supported_formats) {
1584 GST_WARNING_OBJECT (transform, "Couldn't figure out supported format");
1585 return;
1586 }
1587
1588 if (IsEqualGUID (MFVideoFormat_H264, subtype)) {
1589 profile_to_check = h264_profile_map;
1590 codec_caps_str = h264_caps_str;
1591 } else if (IsEqualGUID (MFVideoFormat_HEVC, subtype)) {
1592 profile_to_check = hevc_profile_map;
1593 codec_caps_str = hevc_caps_str;
1594 } else if (IsEqualGUID (MFVideoFormat_VP90, subtype)) {
1595 codec_caps_str = vp9_caps_str;
1596 } else {
1597 g_assert_not_reached ();
1598 return;
1599 }
1600
1601 if (profile_to_check) {
1602 hr = MFCreateMediaType (&out_type);
1603 if (!gst_mf_result (hr))
1604 return;
1605
1606 hr = out_type->SetGUID (MF_MT_MAJOR_TYPE, MFMediaType_Video);
1607 if (!gst_mf_result (hr))
1608 return;
1609
1610 hr = out_type->SetGUID (MF_MT_SUBTYPE, subtype);
1611 if (!gst_mf_result (hr))
1612 return;
1613
1614 hr = out_type->SetUINT32 (MF_MT_AVG_BITRATE, 2048000);
1615 if (!gst_mf_result (hr))
1616 return;
1617
1618 hr = MFSetAttributeRatio (out_type.Get (), MF_MT_FRAME_RATE, 30, 1);
1619 if (!gst_mf_result (hr))
1620 return;
1621
1622 hr = out_type->SetUINT32 (MF_MT_INTERLACE_MODE,
1623 MFVideoInterlace_Progressive);
1624 if (!gst_mf_result (hr))
1625 return;
1626
1627 hr = MFSetAttributeSize (out_type.Get (), MF_MT_FRAME_SIZE, 1920, 1080);
1628 if (!gst_mf_result (hr))
1629 return;
1630
1631 i = 0;
1632 do {
1633 GValue profile_val = G_VALUE_INIT;
1634 guint mf_profile = profile_to_check[i].profile;
1635 const gchar *profile_str = profile_to_check[i].profile_str;
1636
1637 i++;
1638
1639 if (mf_profile == 0)
1640 break;
1641
1642 g_assert (profile_str != NULL);
1643
1644 hr = out_type->SetUINT32 (MF_MT_MPEG2_PROFILE, mf_profile);
1645 if (!gst_mf_result (hr))
1646 return;
1647
1648 if (!gst_mf_transform_set_output_type (transform, out_type.Get ()))
1649 continue;
1650
1651 if (!profiles) {
1652 profiles = g_new0 (GValue, 1);
1653 g_value_init (profiles, GST_TYPE_LIST);
1654 }
1655
1656 /* Add "constrained-baseline" in addition to "baseline" */
1657 if (profile_str == "baseline") {
1658 g_value_init (&profile_val, G_TYPE_STRING);
1659 g_value_set_static_string (&profile_val, "constrained-baseline");
1660 gst_value_list_append_and_take_value (profiles, &profile_val);
1661 }
1662
1663 g_value_init (&profile_val, G_TYPE_STRING);
1664 g_value_set_static_string (&profile_val, profile_str);
1665 gst_value_list_append_and_take_value (profiles, &profile_val);
1666 } while (1);
1667
1668 if (!profiles) {
1669 GST_WARNING_OBJECT (transform, "Couldn't query supported profile");
1670 return;
1671 }
1672 }
1673
1674 src_caps = gst_caps_from_string (codec_caps_str);
1675 if (profiles) {
1676 gst_caps_set_value (src_caps, "profile", profiles);
1677 g_value_unset (profiles);
1678 g_free (profiles);
1679 }
1680
1681 sink_caps = gst_caps_new_empty_simple ("video/x-raw");
1682 /* FIXME: don't hardcode max resolution, but MF doesn't provide
1683 * API for querying supported max resolution... */
1684 gst_caps_set_simple (sink_caps,
1685 "width", GST_TYPE_INT_RANGE, 64, 8192,
1686 "height", GST_TYPE_INT_RANGE, 64, 8192, NULL);
1687 gst_caps_set_simple (src_caps,
1688 "width", GST_TYPE_INT_RANGE, 64, 8192,
1689 "height", GST_TYPE_INT_RANGE, 64, 8192, NULL);
1690
1691 #if GST_MF_HAVE_D3D11
1692 /* Check whether this MFT can support D3D11 */
1693 if (d3d11_device && (have_NV12 || have_P010)) {
1694 g_object_get (transform, "d3d11-aware", &d3d11_aware, NULL);
1695 GST_DEBUG_OBJECT (transform, "d3d11 aware %d", d3d11_aware);
1696 }
1697
1698 if (d3d11_device && (have_NV12 || have_P010) && d3d11_aware) {
1699 gint64 adapter_luid = 0;
1700 GValue d3d11_formats = G_VALUE_INIT;
1701
1702 g_object_get (d3d11_device, "adapter-luid", &adapter_luid, NULL);
1703
1704 d3d11_caps = gst_caps_copy (sink_caps);
1705
1706 g_value_init (&d3d11_formats, GST_TYPE_LIST);
1707 if (have_NV12) {
1708 GValue val = G_VALUE_INIT;
1709 g_value_init (&val, G_TYPE_STRING);
1710 g_value_set_static_string (&val, "NV12");
1711 gst_value_list_append_and_take_value (&d3d11_formats, &val);
1712 }
1713
1714 if (have_P010) {
1715 GValue val = G_VALUE_INIT;
1716 g_value_init (&val, G_TYPE_STRING);
1717 g_value_set_static_string (&val, "P010_10LE");
1718 gst_value_list_append_and_take_value (&d3d11_formats, &val);
1719 }
1720
1721 gst_caps_set_value (d3d11_caps, "format", &d3d11_formats);
1722 g_value_unset (&d3d11_formats);
1723 gst_caps_set_features_simple (d3d11_caps,
1724 gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY));
1725 device_caps->d3d11_aware = TRUE;
1726 device_caps->adapter_luid = adapter_luid;
1727 }
1728 #endif
1729
1730 gst_caps_set_value (sink_caps, "format", supported_formats);
1731 g_value_unset (supported_formats);
1732 g_free (supported_formats);
1733
1734 if (d3d11_caps)
1735 gst_caps_append (sink_caps, d3d11_caps);
1736
1737 *sink_template = sink_caps;
1738 *src_template = src_caps;
1739
1740 #define CHECK_DEVICE_CAPS(codec_obj,api,val) \
1741 if (SUCCEEDED((codec_obj)->IsSupported(&(api)))) {\
1742 (device_caps)->val = TRUE; \
1743 }
1744
1745 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonRateControlMode, rc_mode);
1746 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonQuality, quality);
1747 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncAdaptiveMode, adaptive_mode);
1748 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonBufferSize, buffer_size);
1749 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonMaxBitRate, max_bitrate);
1750 CHECK_DEVICE_CAPS (codec_api,
1751 CODECAPI_AVEncCommonQualityVsSpeed, quality_vs_speed);
1752 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncH264CABACEnable, cabac);
1753 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncH264SPSID, sps_id);
1754 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncH264PPSID, pps_id);
1755 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncMPVDefaultBPictureCount, bframes);
1756 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncMPVGOPSize, gop_size);
1757 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncNumWorkerThreads, threads);
1758 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoContentType, content_type);
1759 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoEncodeQP, qp);
1760 CHECK_DEVICE_CAPS (codec_api,
1761 CODECAPI_AVEncVideoForceKeyFrame, force_keyframe);
1762 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVLowLatencyMode, low_latency);
1763 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoMinQP, min_qp);
1764 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoMaxQP, max_qp);
1765 CHECK_DEVICE_CAPS (codec_api,
1766 CODECAPI_AVEncVideoEncodeFrameTypeQP, frame_type_qp);
1767 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoMaxNumRefFrame, max_num_ref);
1768 if (device_caps->max_num_ref) {
1769 VARIANT min;
1770 VARIANT max;
1771 VARIANT step;
1772
1773 hr = codec_api->GetParameterRange (&CODECAPI_AVEncVideoMaxNumRefFrame,
1774 &min, &max, &step);
1775 if (SUCCEEDED (hr)) {
1776 device_caps->max_num_ref_high = max.uiVal;
1777 device_caps->max_num_ref_low = min.uiVal;
1778 VariantClear (&min);
1779 VariantClear (&max);
1780 VariantClear (&step);
1781 } else {
1782 device_caps->max_num_ref = FALSE;
1783 }
1784 }
1785 #undef CHECK_DEVICE_CAPS
1786
1787 return;
1788 }
1789
1790 static GstMFTransform *
gst_mf_video_enc_enum(guint enum_flags,GUID * subtype,guint device_index,GstMFVideoEncDeviceCaps * device_caps,GstObject * d3d11_device,GstCaps ** sink_template,GstCaps ** src_template)1791 gst_mf_video_enc_enum (guint enum_flags, GUID * subtype, guint device_index,
1792 GstMFVideoEncDeviceCaps * device_caps, GstObject * d3d11_device,
1793 GstCaps ** sink_template, GstCaps ** src_template)
1794 {
1795 GstMFTransformEnumParams enum_params = { 0, };
1796 MFT_REGISTER_TYPE_INFO output_type;
1797 GstMFTransform *transform;
1798 gint64 adapter_luid = 0;
1799
1800 *sink_template = NULL;
1801 *src_template = NULL;
1802 memset (device_caps, 0, sizeof (GstMFVideoEncDeviceCaps));
1803
1804 if (!IsEqualGUID (MFVideoFormat_H264, *subtype) &&
1805 !IsEqualGUID (MFVideoFormat_HEVC, *subtype) &&
1806 !IsEqualGUID (MFVideoFormat_VP90, *subtype)) {
1807 GST_ERROR ("Unknown subtype GUID");
1808
1809 return NULL;
1810 }
1811
1812 if (d3d11_device) {
1813 g_object_get (d3d11_device, "adapter-luid", &adapter_luid, NULL);
1814 if (!adapter_luid) {
1815 GST_ERROR ("Couldn't get adapter LUID");
1816 return NULL;
1817 }
1818 }
1819
1820 output_type.guidMajorType = MFMediaType_Video;
1821 output_type.guidSubtype = *subtype;
1822
1823 enum_params.category = MFT_CATEGORY_VIDEO_ENCODER;
1824 enum_params.output_typeinfo = &output_type;
1825 enum_params.device_index = device_index;
1826 enum_params.enum_flags = enum_flags;
1827 enum_params.adapter_luid = adapter_luid;
1828
1829 transform = gst_mf_transform_new (&enum_params);
1830 if (!transform)
1831 return NULL;
1832
1833 gst_mf_video_enc_enum_internal (transform, output_type.guidSubtype,
1834 d3d11_device, device_caps, sink_template, src_template);
1835
1836 return transform;
1837 }
1838
1839 static void
gst_mf_video_enc_register_internal(GstPlugin * plugin,guint rank,GUID * subtype,GTypeInfo * type_info,const GstMFVideoEncDeviceCaps * device_caps,guint32 enum_flags,guint device_index,GstMFTransform * transform,GstCaps * sink_caps,GstCaps * src_caps)1840 gst_mf_video_enc_register_internal (GstPlugin * plugin, guint rank,
1841 GUID * subtype, GTypeInfo * type_info,
1842 const GstMFVideoEncDeviceCaps * device_caps,
1843 guint32 enum_flags, guint device_index, GstMFTransform * transform,
1844 GstCaps * sink_caps, GstCaps * src_caps)
1845 {
1846 GType type;
1847 GTypeInfo local_type_info;
1848 gchar *type_name;
1849 gchar *feature_name;
1850 gint i;
1851 GstMFVideoEncClassData *cdata;
1852 gboolean is_default = TRUE;
1853 gchar *device_name = NULL;
1854 const gchar *type_name_prefix = NULL;
1855 const gchar *feature_name_prefix = NULL;
1856
1857 if (IsEqualGUID (MFVideoFormat_H264, *subtype)) {
1858 type_name_prefix = "H264";
1859 feature_name_prefix = "h264";
1860 } else if (IsEqualGUID (MFVideoFormat_HEVC, *subtype)) {
1861 type_name_prefix = "H265";
1862 feature_name_prefix = "h265";
1863 } else if (IsEqualGUID (MFVideoFormat_VP90, *subtype)) {
1864 type_name_prefix = "VP9";
1865 feature_name_prefix = "vp9";
1866 } else {
1867 g_assert_not_reached ();
1868 return;
1869 }
1870
1871 /* Must be checked already */
1872 g_object_get (transform, "device-name", &device_name, NULL);
1873 g_assert (device_name != NULL);
1874
1875 cdata = g_new0 (GstMFVideoEncClassData, 1);
1876 cdata->sink_caps = gst_caps_copy (sink_caps);
1877 cdata->src_caps = gst_caps_copy (src_caps);
1878 cdata->device_name = device_name;
1879 cdata->device_caps = *device_caps;
1880 cdata->enum_flags = enum_flags;
1881 cdata->device_index = device_index;
1882
1883 local_type_info = *type_info;
1884 local_type_info.class_data = cdata;
1885
1886 GST_MINI_OBJECT_FLAG_SET (cdata->sink_caps,
1887 GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
1888 GST_MINI_OBJECT_FLAG_SET (cdata->src_caps,
1889 GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
1890
1891 type_name = g_strdup_printf ("GstMF%sEnc", type_name_prefix);
1892 feature_name = g_strdup_printf ("mf%senc", feature_name_prefix);
1893
1894 i = 1;
1895 while (g_type_from_name (type_name) != 0) {
1896 g_free (type_name);
1897 g_free (feature_name);
1898 type_name = g_strdup_printf ("GstMF%sDevice%dEnc", type_name_prefix, i);
1899 feature_name = g_strdup_printf ("mf%sdevice%denc", feature_name_prefix, i);
1900 is_default = FALSE;
1901 i++;
1902 }
1903
1904 cdata->is_default = is_default;
1905
1906 type =
1907 g_type_register_static (GST_TYPE_MF_VIDEO_ENC, type_name,
1908 &local_type_info, (GTypeFlags) 0);
1909
1910 /* make lower rank than default device */
1911 if (rank > 0 && !is_default)
1912 rank--;
1913
1914 if (!is_default)
1915 gst_element_type_set_skip_documentation (type);
1916
1917 if (!gst_element_register (plugin, feature_name, rank, type))
1918 GST_WARNING ("Failed to register plugin '%s'", type_name);
1919
1920 g_free (type_name);
1921 g_free (feature_name);
1922 }
1923
1924 void
gst_mf_video_enc_register(GstPlugin * plugin,guint rank,GUID * subtype,GTypeInfo * type_info,GList * d3d11_device)1925 gst_mf_video_enc_register (GstPlugin * plugin, guint rank, GUID * subtype,
1926 GTypeInfo * type_info, GList * d3d11_device)
1927 {
1928 GstMFTransform *transform = NULL;
1929 GstCaps *sink_template = NULL;
1930 GstCaps *src_template = NULL;
1931 guint enum_flags;
1932 GstMFVideoEncDeviceCaps device_caps;
1933 guint i;
1934
1935 /* register hardware encoders first */
1936 enum_flags = (MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_ASYNCMFT |
1937 MFT_ENUM_FLAG_SORTANDFILTER | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY);
1938
1939 if (d3d11_device) {
1940 GList *iter;
1941 for (iter = d3d11_device; iter; iter = g_list_next (iter)) {
1942 GstObject *device = (GstObject *) iter->data;
1943
1944 transform = gst_mf_video_enc_enum (enum_flags, subtype, 0, &device_caps,
1945 device, &sink_template, &src_template);
1946
1947 /* No more MFT to enumerate */
1948 if (!transform)
1949 break;
1950
1951 /* Failed to open MFT */
1952 if (!sink_template) {
1953 gst_clear_object (&transform);
1954 continue;
1955 }
1956
1957 gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info,
1958 &device_caps, enum_flags, 0, transform, sink_template, src_template);
1959 gst_clear_object (&transform);
1960 gst_clear_caps (&sink_template);
1961 gst_clear_caps (&src_template);
1962 }
1963 } else {
1964 /* AMD seems to be able to support up to 12 GPUs */
1965 for (i = 0; i < 12; i++) {
1966 transform = gst_mf_video_enc_enum (enum_flags, subtype, i, &device_caps,
1967 NULL, &sink_template, &src_template);
1968
1969 /* No more MFT to enumerate */
1970 if (!transform)
1971 break;
1972
1973 /* Failed to open MFT */
1974 if (!sink_template) {
1975 gst_clear_object (&transform);
1976 continue;
1977 }
1978
1979 gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info,
1980 &device_caps, enum_flags, i, transform, sink_template, src_template);
1981 gst_clear_object (&transform);
1982 gst_clear_caps (&sink_template);
1983 gst_clear_caps (&src_template);
1984 }
1985 }
1986
1987 /* register software encoders */
1988 enum_flags = (MFT_ENUM_FLAG_SYNCMFT |
1989 MFT_ENUM_FLAG_SORTANDFILTER | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY);
1990
1991 transform = gst_mf_video_enc_enum (enum_flags, subtype, 0, &device_caps,
1992 NULL, &sink_template, &src_template);
1993
1994 if (!transform)
1995 goto done;
1996
1997 if (!sink_template)
1998 goto done;
1999
2000 gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info,
2001 &device_caps, enum_flags, 0, transform, sink_template, src_template);
2002
2003 done:
2004 gst_clear_object (&transform);
2005 gst_clear_caps (&sink_template);
2006 gst_clear_caps (&src_template);
2007 }
2008