1 /* GStreamer
2 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 #ifdef HAVE_CONFIG_H
21 #include "config.h"
22 #endif
23
24 #include <gst/gst.h>
25 #include "gstmfutils.h"
26 #include "gstmfsourceobject.h"
27 #include "mediacapturewrapper.h"
28
29 #include "AsyncOperations.h"
30 #include <windows.ui.core.h>
31 #include <locale>
32 #include <codecvt>
33 #include <string.h>
34 #include <algorithm>
35 #include <iterator>
36
37 /* *INDENT-OFF* */
38 using namespace ABI::Windows::ApplicationModel::Core;
39 using namespace ABI::Windows::Foundation::Collections;
40 using namespace ABI::Windows::Media::Devices;
41 using namespace ABI::Windows::Media::MediaProperties;
42
43 G_BEGIN_DECLS
44
45 GST_DEBUG_CATEGORY_EXTERN (gst_mf_source_object_debug);
46 #define GST_CAT_DEFAULT gst_mf_source_object_debug
47
48 G_END_DECLS
49
50 static std::string
convert_hstring_to_string(HString * hstr)51 convert_hstring_to_string (HString * hstr)
52 {
53 const wchar_t *raw_hstr;
54
55 if (!hstr)
56 return std::string();
57
58 raw_hstr = hstr->GetRawBuffer (nullptr);
59 if (!raw_hstr)
60 return std::string();
61
62 std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
63
64 return converter.to_bytes (raw_hstr);
65 }
66
67 static std::string
gst_media_capture_subtype_to_video_format(const std::string & subtype)68 gst_media_capture_subtype_to_video_format (const std::string &subtype)
69 {
70 /* https://docs.microsoft.com/en-us/uwp/api/windows.media.mediaproperties.videoencodingproperties.subtype#Windows_Media_MediaProperties_VideoEncodingProperties_Subtype */
71 if (g_ascii_strcasecmp (subtype.c_str(), "RGB32") == 0)
72 return "BGRx";
73 else if (g_ascii_strcasecmp (subtype.c_str(), "ARGB32") == 0)
74 return "BGRA";
75 else if (g_ascii_strcasecmp (subtype.c_str(), "RGB24") == 0)
76 return "BGR";
77 else if (g_ascii_strcasecmp (subtype.c_str(), "NV12") == 0)
78 return "NV12";
79 else if (g_ascii_strcasecmp (subtype.c_str(), "YV12") == 0)
80 return "YV12";
81 else if (g_ascii_strcasecmp (subtype.c_str(), "IYUV") == 0 ||
82 g_ascii_strcasecmp (subtype.c_str(), "I420") == 0)
83 return "I420";
84 else if (g_ascii_strcasecmp (subtype.c_str(), "YUY2") == 0)
85 return "YUY2";
86
87 /* FIXME: add more */
88
89 return std::string();
90 }
91
GstWinRTMediaDescription()92 GstWinRTMediaDescription::GstWinRTMediaDescription()
93 : caps_(nullptr)
94 {
95 }
96
GstWinRTMediaDescription(const GstWinRTMediaDescription & other)97 GstWinRTMediaDescription::GstWinRTMediaDescription
98 (const GstWinRTMediaDescription& other)
99 : caps_(nullptr)
100 {
101 if (other.source_id_.IsValid())
102 other.source_id_.CopyTo(source_id_.GetAddressOf());
103 if (other.subtype_.IsValid())
104 other.subtype_.CopyTo(subtype_.GetAddressOf());
105 gst_caps_replace (&caps_, other.caps_);
106 }
107
~GstWinRTMediaDescription()108 GstWinRTMediaDescription::~GstWinRTMediaDescription()
109 {
110 Release();
111 }
112
113 void
Release()114 GstWinRTMediaDescription::Release()
115 {
116 source_id_.Release();
117 subtype_.Release();
118 gst_clear_caps(&caps_);
119 }
120
121 bool
IsValid() const122 GstWinRTMediaDescription::IsValid() const
123 {
124 if (!source_id_.IsValid())
125 return false;
126 if (!subtype_.IsValid())
127 return false;
128 if (!caps_)
129 return false;
130
131 return true;
132 }
133
134 HRESULT
Fill(HString & source_id,const ComPtr<IMediaCaptureVideoProfileMediaDescription> & desc,unsigned int info_index,unsigned int desc_index)135 GstWinRTMediaDescription::Fill(HString &source_id,
136 const ComPtr<IMediaCaptureVideoProfileMediaDescription>& desc,
137 unsigned int info_index, unsigned int desc_index)
138 {
139 Release();
140
141 if (!source_id.IsValid()) {
142 GST_WARNING("Invalid source id");
143 return E_FAIL;
144 }
145
146 ComPtr<IMediaCaptureVideoProfileMediaDescription2> desc2;
147 UINT32 width = 0;
148 UINT32 height = 0;
149 DOUBLE framerate = 0;
150 gint fps_n = 0, fps_d = 1;
151 HString hstr_subtype;
152 std::string subtype;
153 std::string format;
154 GstCaps *caps;
155 HRESULT hr;
156
157 hr = desc.As (&desc2);
158 if (!gst_mf_result (hr))
159 return hr;
160
161 hr = desc->get_Width (&width);
162 if (!gst_mf_result (hr))
163 return hr;
164
165 hr = desc->get_Height (&height);
166 if (!gst_mf_result (hr))
167 return hr;
168
169 hr = desc->get_FrameRate (&framerate);
170 if (gst_mf_result (hr) && framerate > 0)
171 gst_util_double_to_fraction (framerate, &fps_n, &fps_d);
172
173 hr = desc2->get_Subtype (hstr_subtype.GetAddressOf ());
174 if (!gst_mf_result (hr))
175 return hr;
176
177 subtype = convert_hstring_to_string (&hstr_subtype);
178 if (subtype.empty())
179 return E_FAIL;
180
181 format = gst_media_capture_subtype_to_video_format (subtype);
182 if (format.empty()) {
183 GST_LOG ("source-info %d, desc %d, unhandled subtype %s",
184 info_index, desc_index, subtype.c_str());
185 return E_FAIL;
186 }
187
188 caps = gst_caps_new_simple ("video/x-raw",
189 "format", G_TYPE_STRING, format.c_str(), "width", G_TYPE_INT, width,
190 "height", G_TYPE_INT, height, NULL);
191
192 if (fps_n > 0 && fps_d > 0)
193 gst_caps_set_simple (caps,
194 "framerate", GST_TYPE_FRACTION, fps_n, fps_d, NULL);
195
196 source_id.CopyTo (source_id_.GetAddressOf());
197 hstr_subtype.CopyTo (subtype_.GetAddressOf());
198 caps_ = caps;
199
200 GST_LOG ("source-info %d, desc %d, %" GST_PTR_FORMAT,
201 info_index, desc_index, caps_);
202
203 return S_OK;
204 }
205
GstWinRTMediaFrameSourceGroup()206 GstWinRTMediaFrameSourceGroup::GstWinRTMediaFrameSourceGroup()
207 {
208 }
209
GstWinRTMediaFrameSourceGroup(const GstWinRTMediaFrameSourceGroup & other)210 GstWinRTMediaFrameSourceGroup::GstWinRTMediaFrameSourceGroup
211 (const GstWinRTMediaFrameSourceGroup& other)
212 {
213 id_ = other.id_;
214 display_name_ = other.display_name_;
215 source_group_ = other.source_group_;
216 source_list_ = other.source_list_;
217 }
218
~GstWinRTMediaFrameSourceGroup()219 GstWinRTMediaFrameSourceGroup::~GstWinRTMediaFrameSourceGroup()
220 {
221 Release ();
222 }
223
224 void
Release()225 GstWinRTMediaFrameSourceGroup::Release()
226 {
227 id_.clear ();
228 display_name_.clear ();
229 source_group_.Reset ();
230 source_list_.clear ();
231 }
232
233 bool
Contain(const GstWinRTMediaDescription & desc)234 GstWinRTMediaFrameSourceGroup::Contain(const GstWinRTMediaDescription& desc)
235 {
236 if (!desc.IsValid())
237 return false;
238
239 if (source_list_.empty())
240 return false;
241
242 for (const auto& iter: source_list_) {
243 unsigned int dummy;
244 if (wcscmp (iter.source_id_.GetRawBuffer (&dummy),
245 desc.source_id_.GetRawBuffer (&dummy)))
246 continue;
247
248 if (wcscmp (iter.subtype_.GetRawBuffer (&dummy),
249 desc.subtype_.GetRawBuffer (&dummy)))
250 continue;
251
252 if (gst_caps_is_equal (iter.caps_, desc.caps_))
253 return true;
254 }
255
256 return false;
257 }
258
259 HRESULT
Fill(const ComPtr<IMediaFrameSourceGroup> & source_group,unsigned int index)260 GstWinRTMediaFrameSourceGroup::Fill
261 (const ComPtr<IMediaFrameSourceGroup> &source_group, unsigned int index)
262 {
263 HRESULT hr = S_OK;
264 HString hstr_id;
265 HString hstr_display_name;
266 ComPtr<IVectorView<MediaFrameSourceInfo*>> info_list;
267 UINT32 count = 0;
268 std::vector<GstWinRTMediaDescription> preview_list;
269 std::vector<GstWinRTMediaDescription> record_list;
270
271 Release();
272
273 hr = source_group->get_Id(hstr_id.GetAddressOf());
274 if (!gst_mf_result(hr))
275 goto error;
276
277 id_ = convert_hstring_to_string (&hstr_id);
278 if (id_.empty()) {
279 GST_WARNING ("source-group %d, Emptry source group id", index);
280 hr = E_FAIL;
281 goto error;
282 }
283
284 hr = source_group->get_DisplayName (hstr_display_name.GetAddressOf());
285 if (!gst_mf_result (hr))
286 goto error;
287
288 display_name_ = convert_hstring_to_string (&hstr_display_name);
289 if (display_name_.empty()) {
290 GST_WARNING ("source-group %d, Empty display name", index);
291 hr = E_FAIL;
292 goto error;
293 }
294
295 hr = source_group->get_SourceInfos (&info_list);
296 if (!gst_mf_result (hr))
297 goto error;
298
299 hr = info_list->get_Size (&count);
300 if (!gst_mf_result (hr))
301 goto error;
302
303 if (count == 0) {
304 GST_WARNING ("No available source info");
305 hr = E_FAIL;
306 goto error;
307 }
308
309 source_group_ = source_group;
310
311 GST_DEBUG ("source-group %d has %d entries", index, count);
312
313 for (UINT32 i = 0; i < count; i++) {
314 ComPtr<IMediaFrameSourceInfo> info;
315 ComPtr<IMediaFrameSourceInfo2> info2;
316 ComPtr<IVectorView<MediaCaptureVideoProfileMediaDescription*>> desc_list;
317 MediaFrameSourceKind source_kind;
318 MediaStreamType source_type;
319 UINT32 desc_count = 0;
320 HString source_id;
321 std::string source_id_str;
322 std::vector<GstWinRTMediaDescription> *target_list = nullptr;
323
324 hr = info_list->GetAt(i, &info);
325 if (!gst_mf_result (hr))
326 continue;
327
328 hr = info.As (&info2);
329 if (!gst_mf_result (hr))
330 continue;
331
332 hr = info->get_SourceKind (&source_kind);
333 if (!gst_mf_result (hr))
334 continue;
335
336 /* This can be depth, infrared or others */
337 /* FIXME: add audio support */
338 if (source_kind != MediaFrameSourceKind::MediaFrameSourceKind_Color) {
339 GST_FIXME ("source-group %d, source-info %d, non-color source kind %d",
340 index, i, (gint) source_kind);
341 continue;
342 }
343
344 hr = info->get_MediaStreamType (&source_type);
345 if (!gst_mf_result (hr))
346 continue;
347
348 /* FIXME: support audio */
349 if (source_type == MediaStreamType::MediaStreamType_VideoPreview) {
350 if (!preview_list.empty ()) {
351 GST_FIXME ("VideoPreview type was checked already");
352 continue;
353 }
354
355 target_list = &preview_list;
356 } else if (source_type == MediaStreamType::MediaStreamType_VideoRecord) {
357 if (!record_list.empty ()) {
358 GST_FIXME ("VideoRecord type was checked already");
359 continue;
360 }
361
362 target_list = &record_list;
363 } else {
364 GST_FIXME ("source-group %d, source-info %d, "
365 "type %d is not VideoPreview or VideoRecord",
366 index, i, (gint) source_type);
367 continue;
368 }
369
370 hr = info->get_Id (source_id.GetAddressOf ());
371 if (!gst_mf_result (hr))
372 continue;
373
374 hr = info2->get_VideoProfileMediaDescription (&desc_list);
375 if (!gst_mf_result (hr))
376 continue;
377
378 hr = desc_list->get_Size (&desc_count);
379 if (!gst_mf_result (hr))
380 continue;
381
382 if (desc_count == 0) {
383 GST_WARNING ("source-group %d, source-info %d, empty media description",
384 index, i);
385 continue;
386 }
387
388 source_id_str = convert_hstring_to_string (&source_id);
389 GST_DEBUG ("source-group %d, source-info %d, source-id %s source-type %d, "
390 "has %d desc", index, i, source_id_str.c_str (), (gint) source_type,
391 desc_count);
392
393 for (UINT32 j = 0; j < desc_count; j++) {
394 ComPtr<IMediaCaptureVideoProfileMediaDescription> desc;
395
396 hr = desc_list->GetAt (j, &desc);
397 if (!gst_mf_result (hr))
398 continue;
399
400 GstWinRTMediaDescription media_desc;
401 hr = media_desc.Fill(source_id, desc, i, j);
402 if (FAILED (hr))
403 continue;
404
405 target_list->push_back(media_desc);
406 }
407 }
408
409 if (!preview_list.empty () && !record_list.empty ()) {
410 /* FIXME: Some devices (e.g., Surface Book 2, Surface Pro X) will expose
411 * both MediaStreamType_VideoPreview and MediaStreamType_VideoRecord types
412 * for a logical device. And for some reason, MediaStreamType_VideoPreview
413 * seems to be selected between them while initiailzing device.
414 * But I cannot find any documentation for the decision rule.
415 * To be safe, we will select common formats between them.
416 */
417 std::vector<GstWinRTMediaDescription> common;
418
419 /* Sort first */
420 std::sort (preview_list.begin (),
421 preview_list.end (), WinRTCapsCompareFunc);
422 std::sort (record_list.begin (),
423 record_list.end (), WinRTCapsCompareFunc);
424
425 /* Find common formats */
426 std::set_intersection(preview_list.begin (), preview_list.end(),
427 record_list.begin (), record_list.end (), std::back_inserter (common),
428 WinRTCapsCompareFunc);
429 source_list_.insert (source_list_.end (), common.begin (),
430 common.end ());
431
432 #ifndef GST_DISABLE_GST_DEBUG
433 std::vector<GstWinRTMediaDescription> diff;
434 std::set_difference(preview_list.begin (), preview_list.end(),
435 record_list.begin (), record_list.end (),
436 std::inserter (diff, diff.begin ()), WinRTCapsCompareFunc);
437
438 for (auto iter: diff)
439 GST_FIXME ("Drop uncommon format %" GST_PTR_FORMAT, iter.caps_);
440 #endif
441 } else if (!preview_list.empty ()) {
442 source_list_.insert (source_list_.end (), preview_list.begin (),
443 preview_list.end ());
444 } else if (!record_list.empty ()) {
445 source_list_.insert (source_list_.end (), record_list.begin (),
446 record_list.end ());
447 }
448
449 if (source_list_.empty()) {
450 GST_WARNING ("No usable source infos");
451 hr = E_FAIL;
452 goto error;
453 }
454
455 return S_OK;
456
457 error:
458 Release ();
459 return hr;
460 }
461
MediaCaptureWrapper(gpointer dispatcher)462 MediaCaptureWrapper::MediaCaptureWrapper(gpointer dispatcher)
463 : user_data_(nullptr)
464 {
465 user_cb_.frame_arrived = nullptr;
466 user_cb_.failed = nullptr;
467
468 if (dispatcher) {
469 ComPtr<IInspectable> inspectable =
470 reinterpret_cast<IInspectable*> (dispatcher);
471 HRESULT hr;
472
473 hr = inspectable.As (&dispatcher_);
474 if (gst_mf_result (hr))
475 GST_INFO("Main UI dispatcher is available");
476 }
477 }
478
~MediaCaptureWrapper()479 MediaCaptureWrapper::~MediaCaptureWrapper()
480 {
481 stopCapture();
482
483 if (frame_reader_)
484 frame_reader_->remove_FrameArrived (token_frame_arrived_);
485
486 if (media_capture_)
487 media_capture_->remove_Failed (token_capture_failed_);
488 }
489
490 void
RegisterCb(const MediaCaptureWrapperCallbacks & cb,void * user_data)491 MediaCaptureWrapper::RegisterCb (const MediaCaptureWrapperCallbacks &cb,
492 void * user_data)
493 {
494 user_cb_.frame_arrived = cb.frame_arrived;
495 user_cb_.failed = cb.failed;
496 user_data_ = user_data;
497 }
498
499 HRESULT
EnumrateFrameSourceGroup(std::vector<GstWinRTMediaFrameSourceGroup> & group_list)500 MediaCaptureWrapper::EnumrateFrameSourceGroup
501 (std::vector<GstWinRTMediaFrameSourceGroup> &group_list)
502 {
503 return enumrateFrameSourceGroup (group_list);
504 }
505
506 HRESULT
SetSourceGroup(const GstWinRTMediaFrameSourceGroup & group)507 MediaCaptureWrapper::SetSourceGroup(const GstWinRTMediaFrameSourceGroup &group)
508 {
509 if (group.source_group_ == nullptr) {
510 GST_WARNING ("Invalid MediaFrameSourceGroup");
511 return E_FAIL;
512 }
513
514 if (group.source_list_.empty()) {
515 GST_WARNING ("group doesn't include source lifo");
516 return E_FAIL;
517 }
518
519 source_group_ =
520 std::unique_ptr<GstWinRTMediaFrameSourceGroup>
521 (new GstWinRTMediaFrameSourceGroup(group));
522
523 return S_OK;
524 }
525
526 HRESULT
SetMediaDescription(const GstWinRTMediaDescription & desc)527 MediaCaptureWrapper::SetMediaDescription(const GstWinRTMediaDescription &desc)
528 {
529 /* Must be source group was specified before this */
530 if (source_group_ == nullptr) {
531 GST_WARNING ("No frame source group was specified");
532 return E_FAIL;
533 }
534
535 if (!desc.IsValid()) {
536 GST_WARNING("Invalid MediaDescription");
537 return E_FAIL;
538 }
539
540 if (!source_group_->Contain(desc)) {
541 GST_WARNING ("MediaDescription is not part of current source group");
542 return E_FAIL;
543 }
544
545 media_desc_ =
546 std::unique_ptr<GstWinRTMediaDescription>
547 (new GstWinRTMediaDescription(desc));
548
549 return S_OK;
550 }
551
552 HRESULT
StartCapture()553 MediaCaptureWrapper::StartCapture()
554 {
555 HRESULT hr = S_OK;
556
557 hr = openMediaCapture ();
558 if (!gst_mf_result (hr))
559 return hr;
560
561 return startCapture ();
562 }
563
564 HRESULT
StopCapture()565 MediaCaptureWrapper::StopCapture()
566 {
567 return stopCapture ();
568 }
569
570 HRESULT
GetAvailableDescriptions(std::vector<GstWinRTMediaDescription> & desc_list)571 MediaCaptureWrapper::GetAvailableDescriptions
572 (std::vector<GstWinRTMediaDescription> &desc_list)
573 {
574 desc_list.clear();
575
576 if (!source_group_) {
577 GST_WARNING ("No frame source group available");
578 return E_FAIL;
579 }
580
581 desc_list = source_group_->source_list_;
582
583 return S_OK;
584 }
585
586 HRESULT
openMediaCapture()587 MediaCaptureWrapper::openMediaCapture()
588 {
589 HRESULT hr;
590
591 if (frame_reader_) {
592 GST_INFO ("Frame reader was configured");
593 return S_OK;
594 }
595
596 if (source_group_ == nullptr) {
597 GST_WARNING ("No frame source group was specified");
598 return E_FAIL;
599 }
600
601 if (media_desc_ == nullptr) {
602 GST_WARNING ("No media description was specified");
603 return E_FAIL;
604 }
605
606 hr = mediaCaptureInitPre ();
607 if (!gst_mf_result (hr))
608 return hr;
609
610 /* Wait user action and resulting mediaCaptureInitPost */
611 std::unique_lock<std::mutex> Lock(lock_);
612 if (!init_done_)
613 cond_.wait (Lock);
614
615 return frame_reader_ ? S_OK : E_FAIL;
616 }
617
618 HRESULT
mediaCaptureInitPre()619 MediaCaptureWrapper::mediaCaptureInitPre()
620 {
621 ComPtr<IAsyncAction> async_action;
622 HRESULT hr;
623
624 auto work_item = Callback<Implements<RuntimeClassFlags<ClassicCom>,
625 IDispatchedHandler, FtmBase>>([this]{
626 ComPtr<IMediaCaptureInitializationSettings> settings;
627 ComPtr<IMediaCaptureInitializationSettings5> settings5;
628 ComPtr<IMediaCapture> media_capture;
629 ComPtr<IMediaCapture5> media_capture5;
630 ComPtr<IAsyncAction> init_async;
631 HRESULT hr;
632 HStringReference hstr_setting =
633 HStringReference (
634 RuntimeClass_Windows_Media_Capture_MediaCaptureInitializationSettings);
635 HStringReference hstr_capture =
636 HStringReference (RuntimeClass_Windows_Media_Capture_MediaCapture);
637 IMediaFrameSourceGroup * source_group =
638 source_group_->source_group_.Get();
639
640 hr = ActivateInstance (hstr_setting.Get(), &settings);
641 if (!gst_mf_result (hr))
642 return hr;
643
644 hr = settings->put_StreamingCaptureMode (
645 StreamingCaptureMode::StreamingCaptureMode_Video);
646 if (!gst_mf_result (hr))
647 return hr;
648
649 hr = settings.As (&settings5);
650 if (!gst_mf_result (hr))
651 return hr;
652
653 hr = settings5->put_SourceGroup (source_group);
654 if (!gst_mf_result (hr))
655 return hr;
656
657 /* TODO: support D3D11 memory */
658 hr = settings5->put_MemoryPreference (
659 MediaCaptureMemoryPreference::MediaCaptureMemoryPreference_Cpu);
660 if (!gst_mf_result (hr))
661 return hr;
662
663 hr = settings5.As (&settings);
664 if (!gst_mf_result (hr))
665 return hr;
666
667 hr = ActivateInstance (hstr_capture.Get(), &media_capture5);
668 if (!gst_mf_result (hr))
669 return hr;
670
671 hr = media_capture5.As (&media_capture);
672 if (!gst_mf_result (hr))
673 return hr;
674
675 hr = media_capture->InitializeWithSettingsAsync (settings.Get(), &init_async);
676 if (!gst_mf_result (hr))
677 return hr;
678
679 return StartAsyncThen(
680 init_async.Get(),
681 [this, init_async, media_capture](_In_ HRESULT hr,
682 _In_ IAsyncAction *asyncResult, _In_ AsyncStatus asyncStatus) -> HRESULT
683 {
684 return mediaCaptureInitPost (init_async, media_capture);
685 });
686 });
687
688 init_done_ = false;
689
690 if (dispatcher_) {
691 hr = dispatcher_->RunAsync(CoreDispatcherPriority_Normal, work_item.Get(),
692 &async_action);
693 } else {
694 hr = work_item->Invoke ();
695 }
696
697 return hr;
698 }
699
700 HRESULT
mediaCaptureInitPost(ComPtr<IAsyncAction> init_async,ComPtr<IMediaCapture> media_capture)701 MediaCaptureWrapper::mediaCaptureInitPost (ComPtr<IAsyncAction> init_async,
702 ComPtr<IMediaCapture> media_capture)
703 {
704 std::unique_lock<std::mutex> Lock(lock_);
705 ComPtr<IMediaFrameSource> frame_source;
706 ComPtr<IMapView<HSTRING, MediaFrameSource*>> frameSources;
707 ComPtr<IMediaFrameSource> source;
708 ComPtr<IMediaFrameFormat> format;
709 ComPtr<IVectorView<MediaFrameFormat*>> formatList;
710 ComPtr<IMediaCapture5> media_capture5;
711 ComPtr<IAsyncAction> set_format_async;
712 ComPtr<IAsyncOperation<MediaFrameReader*>> create_reader_async;
713 ComPtr<IMediaFrameReader> frame_reader;
714 boolean has_key;
715 UINT32 count = 0;
716 GstVideoInfo videoInfo;
717 HRESULT hr;
718 bool is_I420_subtype = false;
719 std::string target_subtype;
720 ComPtr<ITypedEventHandler<MediaFrameReader*, MediaFrameArrivedEventArgs*>>
721 frame_arrived_handler = Callback<ITypedEventHandler<MediaFrameReader*,
722 MediaFrameArrivedEventArgs*>> ([&]
723 (IMediaFrameReader * reader, IMediaFrameArrivedEventArgs* args)
724 {
725 return onFrameArrived(reader, args);
726 }
727 );
728
729 GST_DEBUG ("InitializeWithSettingsAsync done");
730
731 hr = init_async->GetResults ();
732 if (!gst_mf_result (hr))
733 goto done;
734
735 if (!gst_video_info_from_caps (&videoInfo, media_desc_->caps_)) {
736 GST_WARNING ("Couldn't convert caps %" GST_PTR_FORMAT " to videoinfo",
737 media_desc_->caps_);
738 hr = E_FAIL;
739 goto done;
740 }
741
742 /* Windows defines two I420 format, one is I420 and the other is IYUV.
743 * So, if requested video format was I420, we should accept both I420 and
744 * IYUV formats */
745 is_I420_subtype = GST_VIDEO_INFO_FORMAT (&videoInfo) == GST_VIDEO_FORMAT_I420;
746 target_subtype = convert_hstring_to_string (&media_desc_->subtype_);
747
748 hr = media_capture.As (&media_capture5);
749 if (!gst_mf_result (hr))
750 goto done;
751
752 hr = media_capture5->get_FrameSources (&frameSources);
753 if (!gst_mf_result (hr))
754 goto done;
755
756 hr = frameSources->HasKey (media_desc_->source_id_.Get(), &has_key);
757 if (!gst_mf_result (hr))
758 goto done;
759
760 if (!has_key) {
761 GST_ERROR ("MediaFrameSource unavailable");
762 hr = E_FAIL;
763 goto done;
764 }
765
766 hr = frameSources->Lookup (media_desc_->source_id_.Get(), &source);
767 if (!gst_mf_result (hr))
768 goto done;
769
770 hr = source->get_SupportedFormats (&formatList);
771 if (!gst_mf_result (hr))
772 goto done;
773
774 hr = formatList->get_Size (&count);
775 if (!gst_mf_result (hr))
776 goto done;
777
778 if (count == 0) {
779 GST_ERROR ("No supported format object");
780 hr = E_FAIL;
781 goto done;
782 }
783
784 GST_DEBUG ("Has %d available formats", count);
785 GST_INFO ("Finding matching IMediaFrameFormat with %" GST_PTR_FORMAT,
786 media_desc_->caps_);
787
788 #ifndef GST_DISABLE_GST_DEBUG
789 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_LOG) {
790 GST_LOG ("Dump IMediaFrameFormat list");
791 for (UINT32 i = 0; i < count; i++) {
792 ComPtr<IMediaFrameFormat> fmt;
793 ComPtr<IVideoMediaFrameFormat> videoFmt;
794 ComPtr<IMediaRatio> ratio;
795 HString subtype;
796 UINT32 width = 0;
797 UINT32 height = 0;
798 UINT32 fps_n = 0;
799 UINT32 fps_d = 1;
800
801 hr = formatList->GetAt (i, &fmt);
802 if (!gst_mf_result (hr))
803 continue;
804
805 hr = fmt->get_VideoFormat (&videoFmt);
806 if (!gst_mf_result (hr))
807 continue;
808
809 hr = videoFmt->get_Width (&width);
810 if (!gst_mf_result (hr))
811 continue;
812
813 hr = videoFmt->get_Height (&height);
814 if (!gst_mf_result (hr))
815 continue;
816
817 hr = fmt->get_FrameRate (&ratio);
818 if (!gst_mf_result (hr))
819 continue;
820
821 hr = ratio->get_Numerator (&fps_n);
822 if (!gst_mf_result (hr))
823 continue;
824
825 hr = ratio->get_Denominator (&fps_d);
826 if (!gst_mf_result (hr))
827 continue;
828
829 hr = fmt->get_Subtype (subtype.GetAddressOf ());
830 if (!gst_mf_result (hr))
831 continue;
832
833 std::string cur_subtype = convert_hstring_to_string (&subtype);
834
835 GST_LOG ("\tIMediaFrameFormat[%d] subtpye: %s, resolution: %dx%d, "
836 "framerate: %d/%d", i, cur_subtype.c_str (), width, height, fps_n,
837 fps_d);
838 }
839 }
840 #endif
841
842 /* FIXME: support audio */
843 for (UINT32 i = 0; i < count; i++) {
844 ComPtr<IMediaFrameFormat> fmt;
845 ComPtr<IVideoMediaFrameFormat> videoFmt;
846 ComPtr<IMediaRatio> ratio;
847 HString subtype;
848 UINT32 width = 0;
849 UINT32 height = 0;
850 UINT32 fps_n = 0;
851 UINT32 fps_d = 1;
852
853 hr = formatList->GetAt (i, &fmt);
854 if (!gst_mf_result (hr))
855 continue;
856
857 hr = fmt->get_VideoFormat (&videoFmt);
858 if (!gst_mf_result (hr))
859 continue;
860
861 hr = videoFmt->get_Width (&width);
862 if (!gst_mf_result (hr))
863 continue;
864
865 hr = videoFmt->get_Height (&height);
866 if (!gst_mf_result (hr))
867 continue;
868
869 if (width != GST_VIDEO_INFO_WIDTH (&videoInfo) ||
870 height != GST_VIDEO_INFO_HEIGHT (&videoInfo)) {
871 GST_DEBUG ("IMediaFrameFormat[%d], resolution %dx%d is not equal to "
872 "target resolution %dx%d", i, width, height,
873 GST_VIDEO_INFO_WIDTH (&videoInfo),
874 GST_VIDEO_INFO_HEIGHT (&videoInfo));
875 continue;
876 }
877
878 hr = fmt->get_FrameRate (&ratio);
879 if (!gst_mf_result (hr))
880 continue;
881
882 hr = ratio->get_Numerator (&fps_n);
883 if (!gst_mf_result (hr))
884 continue;
885
886 hr = ratio->get_Denominator (&fps_d);
887 if (!gst_mf_result (hr))
888 continue;
889
890 if ((gint) fps_n != GST_VIDEO_INFO_FPS_N (&videoInfo) ||
891 (gint) fps_d != GST_VIDEO_INFO_FPS_D (&videoInfo)) {
892 GST_DEBUG ("IMediaFrameFormat[%d], framerate %d/%d is not equal to "
893 "target framerate %d/%d", i, width, height,
894 GST_VIDEO_INFO_FPS_N (&videoInfo),
895 GST_VIDEO_INFO_FPS_D (&videoInfo));
896 continue;
897 }
898
899 /* TODO: check major type for audio */
900 hr = fmt->get_Subtype (subtype.GetAddressOf ());
901 if (!gst_mf_result (hr))
902 continue;
903
904 std::string cur_subtype = convert_hstring_to_string (&subtype);
905
906 if (is_I420_subtype) {
907 /* Special handling for I420 */
908 if (cur_subtype != "I420" && cur_subtype != "IYUV") {
909 GST_LOG ("IMediaFrameFormat[%d], subtype %s is not equal to target %s",
910 i, cur_subtype.c_str (), target_subtype.c_str ());
911 continue;
912 }
913 } else if (cur_subtype != target_subtype) {
914 GST_LOG ("IMediaFrameFormat[%d], subtype %s is not equal to target %s",
915 i, cur_subtype.c_str (), target_subtype.c_str ());
916 continue;
917 }
918
919 format = fmt;
920 break;
921 }
922
923 if (!format) {
924 GST_ERROR (
925 "Couldn't find matching IMediaFrameFormat interface for %"
926 GST_PTR_FORMAT, media_desc_->caps_);
927 hr = E_FAIL;
928 goto done;
929 } else {
930 GST_INFO ("Found matching IMediaFrameFormat");
931 }
932
933 hr = source->SetFormatAsync (format.Get (), &set_format_async);
934 if (!gst_mf_result (hr))
935 goto done;
936
937 hr = SyncWait<void>(set_format_async.Get ());
938 if (!gst_mf_result (hr))
939 goto done;
940
941 hr = set_format_async->GetResults ();
942 if (!gst_mf_result (hr))
943 goto done;
944
945 hr = media_capture5->CreateFrameReaderAsync (source.Get(),
946 &create_reader_async);
947 if (!gst_mf_result (hr))
948 goto done;
949
950 hr = SyncWait<MediaFrameReader*>(create_reader_async.Get());
951 if (!gst_mf_result (hr))
952 goto done;
953
954 hr = create_reader_async->GetResults(&frame_reader);
955 if (!gst_mf_result (hr))
956 goto done;
957
958 hr = frame_reader->add_FrameArrived (frame_arrived_handler.Get(),
959 &token_frame_arrived_);
960 if (!gst_mf_result (hr))
961 goto done;
962
963 hr = media_capture->add_Failed
964 (Callback<IMediaCaptureFailedEventHandler> ([this]
965 (IMediaCapture * capture, IMediaCaptureFailedEventArgs* args)
966 {
967 return onCaptureFailed(capture, args);
968 }
969 ).Get(),
970 &token_capture_failed_);
971
972 if (!gst_mf_result (hr))
973 goto done;
974
975 frame_reader_ = frame_reader;
976 media_capture_ = media_capture;
977
978 done:
979 init_done_ = true;
980 cond_.notify_all();
981
982 return S_OK;
983 }
984
985 HRESULT
startCapture()986 MediaCaptureWrapper::startCapture()
987 {
988 HRESULT hr;
989
990 if (!frame_reader_) {
991 GST_ERROR ("Frame reader wasn't configured");
992 return E_FAIL;
993 }
994
995 ComPtr<IAsyncOperation<MediaFrameReaderStartStatus>> start_async;
996 hr = frame_reader_->StartAsync (&start_async);
997 if (!gst_mf_result (hr))
998 return hr;
999
1000 hr = SyncWait<MediaFrameReaderStartStatus>(start_async.Get());
1001 if (!gst_mf_result (hr))
1002 return hr;
1003
1004 MediaFrameReaderStartStatus reader_status;
1005 hr = start_async->GetResults(&reader_status);
1006 if (!gst_mf_result (hr))
1007 return hr;
1008
1009 if (reader_status !=
1010 MediaFrameReaderStartStatus::MediaFrameReaderStartStatus_Success) {
1011 GST_ERROR ("Cannot start frame reader, status %d",
1012 (gint) reader_status);
1013 return E_FAIL;
1014 }
1015
1016 return S_OK;
1017 }
1018
1019 HRESULT
stopCapture()1020 MediaCaptureWrapper::stopCapture()
1021 {
1022 HRESULT hr = S_OK;
1023
1024 if (frame_reader_) {
1025 ComPtr<IAsyncAction> async_action;
1026
1027 hr = frame_reader_->StopAsync (&async_action);
1028 if (gst_mf_result (hr))
1029 hr = SyncWait<void>(async_action.Get ());
1030 }
1031
1032 return hr;
1033 }
1034
1035 HRESULT
onFrameArrived(IMediaFrameReader * reader,IMediaFrameArrivedEventArgs * args)1036 MediaCaptureWrapper::onFrameArrived(IMediaFrameReader *reader,
1037 IMediaFrameArrivedEventArgs *args)
1038 {
1039 HRESULT hr;
1040 ComPtr<IMediaFrameReference> frame_ref;
1041 ComPtr<IVideoMediaFrame> video_frame;
1042 ComPtr<ISoftwareBitmap> bitmap;
1043
1044 hr = reader->TryAcquireLatestFrame (&frame_ref);
1045 if (!gst_mf_result (hr))
1046 return hr;
1047
1048 if (!frame_ref)
1049 return S_OK;
1050
1051 /* nothing to do if no callback was installed */
1052 if (!user_cb_.frame_arrived)
1053 return S_OK;
1054
1055 return user_cb_.frame_arrived (frame_ref.Get(), user_data_);
1056 }
1057
1058 HRESULT
onCaptureFailed(IMediaCapture * capture,IMediaCaptureFailedEventArgs * args)1059 MediaCaptureWrapper::onCaptureFailed(IMediaCapture *capture,
1060 IMediaCaptureFailedEventArgs *args)
1061 {
1062 HRESULT hr;
1063 UINT32 error_code = 0;
1064 HString hstr_error_msg;
1065 std::string error_msg;
1066
1067 hr = args->get_Code (&error_code);
1068 gst_mf_result (hr);
1069
1070 hr = args->get_Message (hstr_error_msg.GetAddressOf());
1071 gst_mf_result (hr);
1072
1073 error_msg = convert_hstring_to_string (&hstr_error_msg);
1074
1075 GST_WARNING ("Have error %s (%d)", error_msg.c_str(), error_code);
1076
1077 if (user_cb_.failed)
1078 user_cb_.failed (error_msg, error_code, user_data_);
1079
1080 return S_OK;
1081 }
1082
1083 HRESULT
enumrateFrameSourceGroup(std::vector<GstWinRTMediaFrameSourceGroup> & groupList)1084 MediaCaptureWrapper::enumrateFrameSourceGroup
1085 (std::vector<GstWinRTMediaFrameSourceGroup> &groupList)
1086 {
1087 ComPtr<IMediaFrameSourceGroupStatics> frame_source_group_statics;
1088 ComPtr<IAsyncOperation<IVectorView<MediaFrameSourceGroup*>*>> async_op;
1089 ComPtr<IVectorView<MediaFrameSourceGroup*>> source_group_list;
1090 HRESULT hr;
1091 unsigned int cnt = 0;
1092 HStringReference hstr_frame_source_group =
1093 HStringReference (
1094 RuntimeClass_Windows_Media_Capture_Frames_MediaFrameSourceGroup);
1095
1096 groupList.clear();
1097
1098 hr = GetActivationFactory (hstr_frame_source_group.Get(),
1099 &frame_source_group_statics);
1100 if (!gst_mf_result(hr))
1101 return hr;
1102
1103 hr = frame_source_group_statics->FindAllAsync (&async_op);
1104 if (!gst_mf_result(hr))
1105 return hr;
1106
1107 hr = SyncWait<IVectorView<MediaFrameSourceGroup*>*>(async_op.Get(), 5000);
1108 if (!gst_mf_result(hr))
1109 return hr;
1110
1111 hr = async_op->GetResults (&source_group_list);
1112 if (!gst_mf_result(hr))
1113 return hr;
1114
1115 hr = source_group_list->get_Size (&cnt);
1116 if (!gst_mf_result(hr))
1117 return hr;
1118
1119 if (cnt == 0) {
1120 GST_WARNING ("No available source group");
1121 return E_FAIL;
1122 }
1123
1124 GST_DEBUG("Have %u source group", cnt);
1125
1126 for (unsigned int i = 0; i < cnt; i++) {
1127 ComPtr<IMediaFrameSourceGroup> group;
1128
1129 hr = source_group_list->GetAt (i, &group);
1130 if (!gst_mf_result(hr))
1131 continue;
1132
1133 GstWinRTMediaFrameSourceGroup source_group;
1134 hr = source_group.Fill(group, i);
1135 if (!gst_mf_result (hr))
1136 continue;
1137
1138 groupList.push_back (source_group);
1139 }
1140
1141 if (groupList.empty ()) {
1142 GST_WARNING("No available source group");
1143 return E_FAIL;
1144 }
1145
1146 return S_OK;
1147 }
1148
1149 HRESULT
FindCoreDispatcherForCurrentThread(ICoreDispatcher ** dispatcher)1150 FindCoreDispatcherForCurrentThread (ICoreDispatcher ** dispatcher)
1151 {
1152 HStringReference hstr_core_app =
1153 HStringReference(RuntimeClass_Windows_ApplicationModel_Core_CoreApplication);
1154 HRESULT hr;
1155
1156 ComPtr<ICoreApplication> core_app;
1157 hr = GetActivationFactory (hstr_core_app.Get(), &core_app);
1158 if (FAILED (hr))
1159 return hr;
1160
1161 ComPtr<ICoreApplicationView> core_app_view;
1162 hr = core_app->GetCurrentView (&core_app_view);
1163 if (FAILED (hr))
1164 return hr;
1165
1166 ComPtr<ICoreWindow> core_window;
1167 hr = core_app_view->get_CoreWindow (&core_window);
1168 if (FAILED (hr))
1169 return hr;
1170
1171 return core_window->get_Dispatcher (dispatcher);
1172 }
1173
1174 bool
WinRTCapsCompareFunc(const GstWinRTMediaDescription & a,const GstWinRTMediaDescription & b)1175 WinRTCapsCompareFunc (const GstWinRTMediaDescription & a,
1176 const GstWinRTMediaDescription & b)
1177 {
1178 return gst_mf_source_object_caps_compare (a.caps_, b.caps_) < 0;
1179 }
1180
1181 /* *INDENT-ON* */
1182