1 /* GStreamer
2 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 #ifdef HAVE_CONFIG_H
21 #include "config.h"
22 #endif
23
24 #include <gst/gst.h>
25 #include "gstmfaudioenc.h"
26 #include <wrl.h>
27 #include <string.h>
28
29 /* *INDENT-OFF* */
30 using namespace Microsoft::WRL;
31 /* *INDENT-ON* */
32
33 GST_DEBUG_CATEGORY (gst_mf_audio_enc_debug);
34 #define GST_CAT_DEFAULT gst_mf_audio_enc_debug
35
36 #define gst_mf_audio_enc_parent_class parent_class
37 G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstMFAudioEnc, gst_mf_audio_enc,
38 GST_TYPE_AUDIO_ENCODER,
39 GST_DEBUG_CATEGORY_INIT (gst_mf_audio_enc_debug, "mfaudioenc", 0,
40 "mfaudioenc"));
41
42 static gboolean gst_mf_audio_enc_open (GstAudioEncoder * enc);
43 static gboolean gst_mf_audio_enc_close (GstAudioEncoder * enc);
44 static gboolean gst_mf_audio_enc_set_format (GstAudioEncoder * enc,
45 GstAudioInfo * info);
46 static GstFlowReturn gst_mf_audio_enc_handle_frame (GstAudioEncoder * enc,
47 GstBuffer * buffer);
48 static GstFlowReturn gst_mf_audio_enc_drain (GstAudioEncoder * enc);
49 static void gst_mf_audio_enc_flush (GstAudioEncoder * enc);
50
51 static void
gst_mf_audio_enc_class_init(GstMFAudioEncClass * klass)52 gst_mf_audio_enc_class_init (GstMFAudioEncClass * klass)
53 {
54 GstAudioEncoderClass *audioenc_class = GST_AUDIO_ENCODER_CLASS (klass);
55
56 audioenc_class->open = GST_DEBUG_FUNCPTR (gst_mf_audio_enc_open);
57 audioenc_class->close = GST_DEBUG_FUNCPTR (gst_mf_audio_enc_close);
58 audioenc_class->set_format = GST_DEBUG_FUNCPTR (gst_mf_audio_enc_set_format);
59 audioenc_class->handle_frame =
60 GST_DEBUG_FUNCPTR (gst_mf_audio_enc_handle_frame);
61 audioenc_class->flush = GST_DEBUG_FUNCPTR (gst_mf_audio_enc_flush);
62
63 gst_type_mark_as_plugin_api (GST_TYPE_MF_AUDIO_ENC, (GstPluginAPIFlags) 0);
64 }
65
66 static void
gst_mf_audio_enc_init(GstMFAudioEnc * self)67 gst_mf_audio_enc_init (GstMFAudioEnc * self)
68 {
69 gst_audio_encoder_set_drainable (GST_AUDIO_ENCODER (self), TRUE);
70 }
71
72 static gboolean
gst_mf_audio_enc_open(GstAudioEncoder * enc)73 gst_mf_audio_enc_open (GstAudioEncoder * enc)
74 {
75 GstMFAudioEnc *self = GST_MF_AUDIO_ENC (enc);
76 GstMFAudioEncClass *klass = GST_MF_AUDIO_ENC_GET_CLASS (enc);
77 GstMFTransformEnumParams enum_params = { 0, };
78 MFT_REGISTER_TYPE_INFO output_type;
79 gboolean ret;
80
81 output_type.guidMajorType = MFMediaType_Audio;
82 output_type.guidSubtype = klass->codec_id;
83
84 enum_params.category = MFT_CATEGORY_AUDIO_ENCODER;
85 enum_params.enum_flags = klass->enum_flags;
86 enum_params.output_typeinfo = &output_type;
87 enum_params.device_index = klass->device_index;
88
89 GST_DEBUG_OBJECT (self, "Create MFT with enum flags 0x%x, device index %d",
90 klass->enum_flags, klass->device_index);
91
92 self->transform = gst_mf_transform_new (&enum_params);
93 ret = !!self->transform;
94
95 if (!ret)
96 GST_ERROR_OBJECT (self, "Cannot create MFT object");
97
98 return ret;
99 }
100
101 static gboolean
gst_mf_audio_enc_close(GstAudioEncoder * enc)102 gst_mf_audio_enc_close (GstAudioEncoder * enc)
103 {
104 GstMFAudioEnc *self = GST_MF_AUDIO_ENC (enc);
105
106 gst_clear_object (&self->transform);
107
108 return TRUE;
109 }
110
111 static gboolean
gst_mf_audio_enc_set_format(GstAudioEncoder * enc,GstAudioInfo * info)112 gst_mf_audio_enc_set_format (GstAudioEncoder * enc, GstAudioInfo * info)
113 {
114 GstMFAudioEnc *self = GST_MF_AUDIO_ENC (enc);
115 GstMFAudioEncClass *klass = GST_MF_AUDIO_ENC_GET_CLASS (enc);
116 ComPtr < IMFMediaType > in_type;
117 ComPtr < IMFMediaType > out_type;
118
119 GST_DEBUG_OBJECT (self, "Set format");
120
121 gst_mf_audio_enc_drain (enc);
122
123 if (!gst_mf_transform_open (self->transform)) {
124 GST_ERROR_OBJECT (self, "Failed to open MFT");
125 return FALSE;
126 }
127
128 g_assert (klass->get_output_type != NULL);
129 if (!klass->get_output_type (self, info, &out_type)) {
130 GST_ERROR_OBJECT (self, "subclass failed to set output type");
131 return FALSE;
132 }
133
134 gst_mf_dump_attributes (out_type.Get (), "Set output type", GST_LEVEL_DEBUG);
135
136 if (!gst_mf_transform_set_output_type (self->transform, out_type.Get ())) {
137 GST_ERROR_OBJECT (self, "Couldn't set output type");
138 return FALSE;
139 }
140
141 g_assert (klass->get_input_type != NULL);
142 if (!klass->get_input_type (self, info, &in_type)) {
143 GST_ERROR_OBJECT (self, "subclass didn't provide input type");
144 return FALSE;
145 }
146
147 gst_mf_dump_attributes (in_type.Get (), "Set input type", GST_LEVEL_DEBUG);
148
149 if (!gst_mf_transform_set_input_type (self->transform, in_type.Get ())) {
150 GST_ERROR_OBJECT (self, "Couldn't set input media type");
151 return FALSE;
152 }
153
154 g_assert (klass->set_src_caps != NULL);
155 if (!klass->set_src_caps (self, info))
156 return FALSE;
157
158 g_assert (klass->frame_samples > 0);
159 gst_audio_encoder_set_frame_samples_min (enc, klass->frame_samples);
160 gst_audio_encoder_set_frame_samples_max (enc, klass->frame_samples);
161 gst_audio_encoder_set_frame_max (enc, 1);
162
163 /* mediafoundation encoder needs timestamp and duration */
164 self->sample_count = 0;
165 self->sample_duration_in_mf = gst_util_uint64_scale (klass->frame_samples,
166 10000000, GST_AUDIO_INFO_RATE (info));
167
168 GST_DEBUG_OBJECT (self,
169 "Calculated sample duration %" GST_TIME_FORMAT,
170 GST_TIME_ARGS (self->sample_duration_in_mf * 100));
171
172 return TRUE;
173 }
174
175 static gboolean
gst_mf_audio_enc_process_input(GstMFAudioEnc * self,GstBuffer * buffer)176 gst_mf_audio_enc_process_input (GstMFAudioEnc * self, GstBuffer * buffer)
177 {
178 HRESULT hr;
179 ComPtr < IMFSample > sample;
180 ComPtr < IMFMediaBuffer > media_buffer;
181 BYTE *data;
182 gboolean res = FALSE;
183 GstMapInfo info;
184 guint64 timestamp;
185
186 if (!gst_buffer_map (buffer, &info, GST_MAP_READ)) {
187 GST_ELEMENT_ERROR (self,
188 RESOURCE, READ, ("Couldn't map input buffer"), (NULL));
189 return FALSE;
190 }
191
192 GST_TRACE_OBJECT (self, "Process buffer %" GST_PTR_FORMAT, buffer);
193
194 timestamp = self->sample_count * self->sample_duration_in_mf;
195
196 hr = MFCreateSample (sample.GetAddressOf ());
197 if (!gst_mf_result (hr))
198 goto done;
199
200 hr = MFCreateMemoryBuffer (info.size, media_buffer.GetAddressOf ());
201 if (!gst_mf_result (hr))
202 goto done;
203
204 hr = media_buffer->Lock (&data, NULL, NULL);
205 if (!gst_mf_result (hr))
206 goto done;
207
208 memcpy (data, info.data, info.size);
209 media_buffer->Unlock ();
210
211 hr = media_buffer->SetCurrentLength (info.size);
212 if (!gst_mf_result (hr))
213 goto done;
214
215 hr = sample->AddBuffer (media_buffer.Get ());
216 if (!gst_mf_result (hr))
217 goto done;
218
219 hr = sample->SetSampleTime (timestamp);
220 if (!gst_mf_result (hr))
221 goto done;
222
223 hr = sample->SetSampleDuration (self->sample_duration_in_mf);
224 if (!gst_mf_result (hr))
225 goto done;
226
227 if (!gst_mf_transform_process_input (self->transform, sample.Get ())) {
228 GST_ERROR_OBJECT (self, "Failed to process input");
229 goto done;
230 }
231
232 self->sample_count++;
233
234 res = TRUE;
235
236 done:
237 gst_buffer_unmap (buffer, &info);
238
239 return res;
240 }
241
242 static GstFlowReturn
gst_mf_audio_enc_process_output(GstMFAudioEnc * self)243 gst_mf_audio_enc_process_output (GstMFAudioEnc * self)
244 {
245 GstMFAudioEncClass *klass = GST_MF_AUDIO_ENC_GET_CLASS (self);
246 HRESULT hr;
247 BYTE *data = nullptr;
248 ComPtr < IMFMediaBuffer > media_buffer;
249 ComPtr < IMFSample > sample;
250 GstBuffer *buffer;
251 GstFlowReturn res = GST_FLOW_ERROR;
252 DWORD buffer_len = 0;
253
254 res = gst_mf_transform_get_output (self->transform, sample.GetAddressOf ());
255
256 if (res != GST_FLOW_OK)
257 return res;
258
259 hr = sample->GetBufferByIndex (0, media_buffer.GetAddressOf ());
260 if (!gst_mf_result (hr))
261 return GST_FLOW_ERROR;
262
263 hr = media_buffer->Lock (&data, NULL, &buffer_len);
264 if (!gst_mf_result (hr))
265 return GST_FLOW_ERROR;
266
267 /* Can happen while draining */
268 if (buffer_len == 0 || !data) {
269 GST_DEBUG_OBJECT (self, "Empty media buffer");
270 media_buffer->Unlock ();
271 return GST_FLOW_OK;
272 }
273
274 buffer = gst_audio_encoder_allocate_output_buffer (GST_AUDIO_ENCODER (self),
275 buffer_len);
276 gst_buffer_fill (buffer, 0, data, buffer_len);
277 media_buffer->Unlock ();
278
279 return gst_audio_encoder_finish_frame (GST_AUDIO_ENCODER (self), buffer,
280 klass->frame_samples);
281 }
282
283 static GstFlowReturn
gst_mf_audio_enc_handle_frame(GstAudioEncoder * enc,GstBuffer * buffer)284 gst_mf_audio_enc_handle_frame (GstAudioEncoder * enc, GstBuffer * buffer)
285 {
286 GstMFAudioEnc *self = GST_MF_AUDIO_ENC (enc);
287 GstFlowReturn ret;
288
289 if (!buffer)
290 return gst_mf_audio_enc_drain (enc);
291
292 if (!gst_mf_audio_enc_process_input (self, buffer)) {
293 GST_ERROR_OBJECT (self, "Failed to process input");
294 return GST_FLOW_ERROR;
295 }
296
297 do {
298 ret = gst_mf_audio_enc_process_output (self);
299 } while (ret == GST_FLOW_OK);
300
301 if (ret == GST_MF_TRANSFORM_FLOW_NEED_DATA)
302 ret = GST_FLOW_OK;
303
304 return ret;
305 }
306
307 static GstFlowReturn
gst_mf_audio_enc_drain(GstAudioEncoder * enc)308 gst_mf_audio_enc_drain (GstAudioEncoder * enc)
309 {
310 GstMFAudioEnc *self = GST_MF_AUDIO_ENC (enc);
311 GstFlowReturn ret = GST_FLOW_OK;
312
313 if (!self->transform)
314 return GST_FLOW_OK;
315
316 gst_mf_transform_drain (self->transform);
317
318 do {
319 ret = gst_mf_audio_enc_process_output (self);
320 } while (ret == GST_FLOW_OK);
321
322 if (ret == GST_MF_TRANSFORM_FLOW_NEED_DATA)
323 ret = GST_FLOW_OK;
324
325 return ret;
326 }
327
328 static void
gst_mf_audio_enc_flush(GstAudioEncoder * enc)329 gst_mf_audio_enc_flush (GstAudioEncoder * enc)
330 {
331 GstMFAudioEnc *self = GST_MF_AUDIO_ENC (enc);
332
333 if (!self->transform)
334 return;
335
336 gst_mf_transform_flush (self->transform);
337 }
338