• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* GStreamer
2  * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
3  *
4  * This library is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Library General Public
6  * License as published by the Free Software Foundation; either
7  * version 2 of the License, or (at your option) any later version.
8  *
9  * This library is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Library General Public License for more details.
13  *
14  * You should have received a copy of the GNU Library General Public
15  * License along with this library; if not, write to the
16  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17  * Boston, MA 02110-1301, USA.
18  */
19 
20 #ifdef HAVE_CONFIG_H
21 #include "config.h"
22 #endif
23 
24 #include <gst/gst.h>
25 #include "gstmfutils.h"
26 #include "gstmfsourceobject.h"
27 #include "mediacapturewrapper.h"
28 
29 #include "AsyncOperations.h"
30 #include <windows.ui.core.h>
31 #include <locale>
32 #include <codecvt>
33 #include <string.h>
34 #include <algorithm>
35 #include <iterator>
36 
37 /* *INDENT-OFF* */
38 using namespace ABI::Windows::ApplicationModel::Core;
39 using namespace ABI::Windows::Foundation::Collections;
40 using namespace ABI::Windows::Media::Devices;
41 using namespace ABI::Windows::Media::MediaProperties;
42 
43 G_BEGIN_DECLS
44 
45 GST_DEBUG_CATEGORY_EXTERN (gst_mf_source_object_debug);
46 #define GST_CAT_DEFAULT gst_mf_source_object_debug
47 
48 G_END_DECLS
49 
50 static std::string
convert_hstring_to_string(HString * hstr)51 convert_hstring_to_string (HString * hstr)
52 {
53   const wchar_t *raw_hstr;
54 
55   if (!hstr)
56     return std::string();
57 
58   raw_hstr = hstr->GetRawBuffer (nullptr);
59   if (!raw_hstr)
60     return std::string();
61 
62   std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
63 
64   return converter.to_bytes (raw_hstr);
65 }
66 
67 static std::string
gst_media_capture_subtype_to_video_format(const std::string & subtype)68 gst_media_capture_subtype_to_video_format (const std::string &subtype)
69 {
70   /* https://docs.microsoft.com/en-us/uwp/api/windows.media.mediaproperties.videoencodingproperties.subtype#Windows_Media_MediaProperties_VideoEncodingProperties_Subtype */
71   if (g_ascii_strcasecmp (subtype.c_str(), "RGB32") == 0)
72     return "BGRx";
73   else if (g_ascii_strcasecmp (subtype.c_str(), "ARGB32") == 0)
74     return "BGRA";
75   else if (g_ascii_strcasecmp (subtype.c_str(), "RGB24") == 0)
76     return "BGR";
77   else if (g_ascii_strcasecmp (subtype.c_str(), "NV12") == 0)
78     return "NV12";
79   else if (g_ascii_strcasecmp (subtype.c_str(), "YV12") == 0)
80     return "YV12";
81   else if (g_ascii_strcasecmp (subtype.c_str(), "IYUV") == 0 ||
82            g_ascii_strcasecmp (subtype.c_str(), "I420") == 0)
83     return "I420";
84   else if (g_ascii_strcasecmp (subtype.c_str(), "YUY2") == 0)
85     return "YUY2";
86 
87   /* FIXME: add more */
88 
89   return std::string();
90 }
91 
GstWinRTMediaDescription()92 GstWinRTMediaDescription::GstWinRTMediaDescription()
93   : caps_(nullptr)
94 {
95 }
96 
GstWinRTMediaDescription(const GstWinRTMediaDescription & other)97 GstWinRTMediaDescription::GstWinRTMediaDescription
98     (const GstWinRTMediaDescription& other)
99   : caps_(nullptr)
100 {
101   if (other.source_id_.IsValid())
102     other.source_id_.CopyTo(source_id_.GetAddressOf());
103   if (other.subtype_.IsValid())
104     other.subtype_.CopyTo(subtype_.GetAddressOf());
105   gst_caps_replace (&caps_, other.caps_);
106 }
107 
~GstWinRTMediaDescription()108 GstWinRTMediaDescription::~GstWinRTMediaDescription()
109 {
110   Release();
111 }
112 
113 void
Release()114 GstWinRTMediaDescription::Release()
115 {
116   source_id_.Release();
117   subtype_.Release();
118   gst_clear_caps(&caps_);
119 }
120 
121 bool
IsValid() const122 GstWinRTMediaDescription::IsValid() const
123 {
124   if (!source_id_.IsValid())
125     return false;
126   if (!subtype_.IsValid())
127     return false;
128   if (!caps_)
129     return false;
130 
131   return true;
132 }
133 
134 HRESULT
Fill(HString & source_id,const ComPtr<IMediaCaptureVideoProfileMediaDescription> & desc,unsigned int info_index,unsigned int desc_index)135 GstWinRTMediaDescription::Fill(HString &source_id,
136     const ComPtr<IMediaCaptureVideoProfileMediaDescription>& desc,
137     unsigned int info_index, unsigned int desc_index)
138 {
139   Release();
140 
141   if (!source_id.IsValid()) {
142     GST_WARNING("Invalid source id");
143     return E_FAIL;
144   }
145 
146   ComPtr<IMediaCaptureVideoProfileMediaDescription2> desc2;
147   UINT32 width = 0;
148   UINT32 height = 0;
149   DOUBLE framerate = 0;
150   gint fps_n = 0, fps_d = 1;
151   HString hstr_subtype;
152   std::string subtype;
153   std::string format;
154   GstCaps *caps;
155   HRESULT hr;
156 
157   hr = desc.As (&desc2);
158   if (!gst_mf_result (hr))
159     return hr;
160 
161   hr = desc->get_Width (&width);
162   if (!gst_mf_result (hr))
163     return hr;
164 
165   hr = desc->get_Height (&height);
166   if (!gst_mf_result (hr))
167     return hr;
168 
169   hr = desc->get_FrameRate (&framerate);
170   if (gst_mf_result (hr) && framerate > 0)
171     gst_util_double_to_fraction (framerate, &fps_n, &fps_d);
172 
173   hr = desc2->get_Subtype (hstr_subtype.GetAddressOf ());
174   if (!gst_mf_result (hr))
175     return hr;
176 
177   subtype = convert_hstring_to_string (&hstr_subtype);
178   if (subtype.empty())
179     return E_FAIL;
180 
181   format = gst_media_capture_subtype_to_video_format (subtype);
182   if (format.empty()) {
183     GST_LOG ("source-info %d, desc %d, unhandled subtype %s",
184         info_index, desc_index, subtype.c_str());
185     return E_FAIL;
186   }
187 
188   caps = gst_caps_new_simple ("video/x-raw",
189       "format", G_TYPE_STRING, format.c_str(), "width", G_TYPE_INT, width,
190       "height", G_TYPE_INT, height, NULL);
191 
192   if (fps_n > 0 && fps_d > 0)
193     gst_caps_set_simple (caps,
194         "framerate", GST_TYPE_FRACTION, fps_n, fps_d, NULL);
195 
196   source_id.CopyTo (source_id_.GetAddressOf());
197   hstr_subtype.CopyTo (subtype_.GetAddressOf());
198   caps_ = caps;
199 
200   GST_LOG ("source-info %d, desc %d, %" GST_PTR_FORMAT,
201       info_index, desc_index, caps_);
202 
203   return S_OK;
204 }
205 
GstWinRTMediaFrameSourceGroup()206 GstWinRTMediaFrameSourceGroup::GstWinRTMediaFrameSourceGroup()
207 {
208 }
209 
GstWinRTMediaFrameSourceGroup(const GstWinRTMediaFrameSourceGroup & other)210 GstWinRTMediaFrameSourceGroup::GstWinRTMediaFrameSourceGroup
211     (const GstWinRTMediaFrameSourceGroup& other)
212 {
213   id_ = other.id_;
214   display_name_ = other.display_name_;
215   source_group_ = other.source_group_;
216   source_list_ = other.source_list_;
217 }
218 
~GstWinRTMediaFrameSourceGroup()219 GstWinRTMediaFrameSourceGroup::~GstWinRTMediaFrameSourceGroup()
220 {
221   Release ();
222 }
223 
224 void
Release()225 GstWinRTMediaFrameSourceGroup::Release()
226 {
227   id_.clear ();
228   display_name_.clear ();
229   source_group_.Reset ();
230   source_list_.clear ();
231 }
232 
233 bool
Contain(const GstWinRTMediaDescription & desc)234 GstWinRTMediaFrameSourceGroup::Contain(const GstWinRTMediaDescription& desc)
235 {
236   if (!desc.IsValid())
237     return false;
238 
239   if (source_list_.empty())
240     return false;
241 
242   for (const auto& iter: source_list_) {
243     unsigned int dummy;
244     if (wcscmp (iter.source_id_.GetRawBuffer (&dummy),
245         desc.source_id_.GetRawBuffer (&dummy)))
246       continue;
247 
248     if (wcscmp (iter.subtype_.GetRawBuffer (&dummy),
249         desc.subtype_.GetRawBuffer (&dummy)))
250       continue;
251 
252     if (gst_caps_is_equal (iter.caps_, desc.caps_))
253       return true;
254   }
255 
256   return false;
257 }
258 
259 HRESULT
Fill(const ComPtr<IMediaFrameSourceGroup> & source_group,unsigned int index)260 GstWinRTMediaFrameSourceGroup::Fill
261     (const ComPtr<IMediaFrameSourceGroup> &source_group, unsigned int index)
262 {
263   HRESULT hr = S_OK;
264   HString hstr_id;
265   HString hstr_display_name;
266   ComPtr<IVectorView<MediaFrameSourceInfo*>> info_list;
267   UINT32 count = 0;
268   std::vector<GstWinRTMediaDescription> preview_list;
269   std::vector<GstWinRTMediaDescription> record_list;
270 
271   Release();
272 
273   hr = source_group->get_Id(hstr_id.GetAddressOf());
274   if (!gst_mf_result(hr))
275     goto error;
276 
277   id_ = convert_hstring_to_string (&hstr_id);
278   if (id_.empty()) {
279     GST_WARNING ("source-group %d, Emptry source group id", index);
280     hr = E_FAIL;
281     goto error;
282   }
283 
284   hr = source_group->get_DisplayName (hstr_display_name.GetAddressOf());
285   if (!gst_mf_result (hr))
286     goto error;
287 
288   display_name_ = convert_hstring_to_string (&hstr_display_name);
289   if (display_name_.empty()) {
290     GST_WARNING ("source-group %d, Empty display name", index);
291     hr = E_FAIL;
292     goto error;
293   }
294 
295   hr = source_group->get_SourceInfos (&info_list);
296   if (!gst_mf_result (hr))
297     goto error;
298 
299   hr = info_list->get_Size (&count);
300   if (!gst_mf_result (hr))
301     goto error;
302 
303   if (count == 0) {
304     GST_WARNING ("No available source info");
305     hr = E_FAIL;
306     goto error;
307   }
308 
309   source_group_ = source_group;
310 
311   GST_DEBUG ("source-group %d has %d entries", index, count);
312 
313   for (UINT32 i = 0; i < count; i++) {
314     ComPtr<IMediaFrameSourceInfo> info;
315     ComPtr<IMediaFrameSourceInfo2> info2;
316     ComPtr<IVectorView<MediaCaptureVideoProfileMediaDescription*>> desc_list;
317     MediaFrameSourceKind source_kind;
318     MediaStreamType source_type;
319     UINT32 desc_count = 0;
320     HString source_id;
321     std::string source_id_str;
322     std::vector<GstWinRTMediaDescription> *target_list = nullptr;
323 
324     hr = info_list->GetAt(i, &info);
325     if (!gst_mf_result (hr))
326       continue;
327 
328     hr = info.As (&info2);
329     if (!gst_mf_result (hr))
330       continue;
331 
332     hr = info->get_SourceKind (&source_kind);
333     if (!gst_mf_result (hr))
334       continue;
335 
336     /* This can be depth, infrared or others */
337     /* FIXME: add audio support */
338     if (source_kind != MediaFrameSourceKind::MediaFrameSourceKind_Color) {
339       GST_FIXME ("source-group %d, source-info %d, non-color source kind %d",
340           index, i, (gint) source_kind);
341       continue;
342     }
343 
344     hr = info->get_MediaStreamType (&source_type);
345     if (!gst_mf_result (hr))
346       continue;
347 
348     /* FIXME: support audio */
349     if (source_type == MediaStreamType::MediaStreamType_VideoPreview) {
350       if (!preview_list.empty ()) {
351         GST_FIXME ("VideoPreview type was checked already");
352         continue;
353       }
354 
355       target_list = &preview_list;
356     } else if (source_type == MediaStreamType::MediaStreamType_VideoRecord) {
357       if (!record_list.empty ()) {
358         GST_FIXME ("VideoRecord type was checked already");
359         continue;
360       }
361 
362       target_list = &record_list;
363     } else {
364       GST_FIXME ("source-group %d, source-info %d, "
365           "type %d is not VideoPreview or VideoRecord",
366           index, i, (gint) source_type);
367       continue;
368     }
369 
370     hr = info->get_Id (source_id.GetAddressOf ());
371     if (!gst_mf_result (hr))
372       continue;
373 
374     hr = info2->get_VideoProfileMediaDescription (&desc_list);
375     if (!gst_mf_result (hr))
376       continue;
377 
378     hr = desc_list->get_Size (&desc_count);
379     if (!gst_mf_result (hr))
380       continue;
381 
382     if (desc_count == 0) {
383       GST_WARNING ("source-group %d, source-info %d, empty media description",
384           index, i);
385       continue;
386     }
387 
388     source_id_str = convert_hstring_to_string (&source_id);
389     GST_DEBUG ("source-group %d, source-info %d, source-id %s source-type %d, "
390         "has %d desc", index, i, source_id_str.c_str (), (gint) source_type,
391         desc_count);
392 
393     for (UINT32 j = 0; j < desc_count; j++) {
394       ComPtr<IMediaCaptureVideoProfileMediaDescription> desc;
395 
396       hr = desc_list->GetAt (j, &desc);
397       if (!gst_mf_result (hr))
398         continue;
399 
400       GstWinRTMediaDescription media_desc;
401       hr = media_desc.Fill(source_id, desc, i, j);
402       if (FAILED (hr))
403         continue;
404 
405       target_list->push_back(media_desc);
406     }
407   }
408 
409   if (!preview_list.empty () && !record_list.empty ()) {
410     /* FIXME: Some devices (e.g., Surface Book 2, Surface Pro X) will expose
411      * both MediaStreamType_VideoPreview and MediaStreamType_VideoRecord types
412      * for a logical device. And for some reason, MediaStreamType_VideoPreview
413      * seems to be selected between them while initiailzing device.
414      * But I cannot find any documentation for the decision rule.
415      * To be safe, we will select common formats between them.
416      */
417     std::vector<GstWinRTMediaDescription> common;
418 
419     /* Sort first */
420     std::sort (preview_list.begin (),
421         preview_list.end (), WinRTCapsCompareFunc);
422     std::sort (record_list.begin (),
423         record_list.end (), WinRTCapsCompareFunc);
424 
425     /* Find common formats */
426     std::set_intersection(preview_list.begin (), preview_list.end(),
427         record_list.begin (), record_list.end (), std::back_inserter (common),
428         WinRTCapsCompareFunc);
429     source_list_.insert (source_list_.end (), common.begin (),
430         common.end ());
431 
432 #ifndef GST_DISABLE_GST_DEBUG
433     std::vector<GstWinRTMediaDescription> diff;
434     std::set_difference(preview_list.begin (), preview_list.end(),
435         record_list.begin (), record_list.end (),
436         std::inserter (diff, diff.begin ()), WinRTCapsCompareFunc);
437 
438     for (auto iter: diff)
439       GST_FIXME ("Drop uncommon format %" GST_PTR_FORMAT, iter.caps_);
440 #endif
441   } else if (!preview_list.empty ()) {
442     source_list_.insert (source_list_.end (), preview_list.begin (),
443         preview_list.end ());
444   } else if (!record_list.empty ()) {
445     source_list_.insert (source_list_.end (), record_list.begin (),
446         record_list.end ());
447   }
448 
449   if (source_list_.empty()) {
450     GST_WARNING ("No usable source infos");
451     hr = E_FAIL;
452     goto error;
453   }
454 
455   return S_OK;
456 
457 error:
458   Release ();
459   return hr;
460 }
461 
MediaCaptureWrapper(gpointer dispatcher)462 MediaCaptureWrapper::MediaCaptureWrapper(gpointer dispatcher)
463   : user_data_(nullptr)
464 {
465   user_cb_.frame_arrived = nullptr;
466   user_cb_.failed = nullptr;
467 
468   if (dispatcher) {
469     ComPtr<IInspectable> inspectable =
470         reinterpret_cast<IInspectable*> (dispatcher);
471     HRESULT hr;
472 
473     hr = inspectable.As (&dispatcher_);
474     if (gst_mf_result (hr))
475       GST_INFO("Main UI dispatcher is available");
476   }
477 }
478 
~MediaCaptureWrapper()479 MediaCaptureWrapper::~MediaCaptureWrapper()
480 {
481   stopCapture();
482 
483   if (frame_reader_)
484     frame_reader_->remove_FrameArrived (token_frame_arrived_);
485 
486   if (media_capture_)
487     media_capture_->remove_Failed (token_capture_failed_);
488 }
489 
490 void
RegisterCb(const MediaCaptureWrapperCallbacks & cb,void * user_data)491 MediaCaptureWrapper::RegisterCb (const MediaCaptureWrapperCallbacks &cb,
492     void * user_data)
493 {
494   user_cb_.frame_arrived = cb.frame_arrived;
495   user_cb_.failed = cb.failed;
496   user_data_ = user_data;
497 }
498 
499 HRESULT
EnumrateFrameSourceGroup(std::vector<GstWinRTMediaFrameSourceGroup> & group_list)500 MediaCaptureWrapper::EnumrateFrameSourceGroup
501     (std::vector<GstWinRTMediaFrameSourceGroup> &group_list)
502 {
503   return enumrateFrameSourceGroup (group_list);
504 }
505 
506 HRESULT
SetSourceGroup(const GstWinRTMediaFrameSourceGroup & group)507 MediaCaptureWrapper::SetSourceGroup(const GstWinRTMediaFrameSourceGroup &group)
508 {
509   if (group.source_group_ == nullptr) {
510     GST_WARNING ("Invalid MediaFrameSourceGroup");
511     return E_FAIL;
512   }
513 
514   if (group.source_list_.empty()) {
515     GST_WARNING ("group doesn't include source lifo");
516     return E_FAIL;
517   }
518 
519   source_group_ =
520       std::unique_ptr<GstWinRTMediaFrameSourceGroup>
521       (new GstWinRTMediaFrameSourceGroup(group));
522 
523   return S_OK;
524 }
525 
526 HRESULT
SetMediaDescription(const GstWinRTMediaDescription & desc)527 MediaCaptureWrapper::SetMediaDescription(const GstWinRTMediaDescription &desc)
528 {
529   /* Must be source group was specified before this */
530   if (source_group_ == nullptr) {
531     GST_WARNING ("No frame source group was specified");
532     return E_FAIL;
533   }
534 
535   if (!desc.IsValid()) {
536     GST_WARNING("Invalid MediaDescription");
537     return E_FAIL;
538   }
539 
540   if (!source_group_->Contain(desc)) {
541     GST_WARNING ("MediaDescription is not part of current source group");
542     return E_FAIL;
543   }
544 
545   media_desc_ =
546       std::unique_ptr<GstWinRTMediaDescription>
547       (new GstWinRTMediaDescription(desc));
548 
549   return S_OK;
550 }
551 
552 HRESULT
StartCapture()553 MediaCaptureWrapper::StartCapture()
554 {
555   HRESULT hr = S_OK;
556 
557   hr = openMediaCapture ();
558   if (!gst_mf_result (hr))
559     return hr;
560 
561   return startCapture ();
562 }
563 
564 HRESULT
StopCapture()565 MediaCaptureWrapper::StopCapture()
566 {
567   return stopCapture ();
568 }
569 
570 HRESULT
GetAvailableDescriptions(std::vector<GstWinRTMediaDescription> & desc_list)571 MediaCaptureWrapper::GetAvailableDescriptions
572     (std::vector<GstWinRTMediaDescription> &desc_list)
573 {
574   desc_list.clear();
575 
576   if (!source_group_) {
577     GST_WARNING ("No frame source group available");
578     return E_FAIL;
579   }
580 
581   desc_list = source_group_->source_list_;
582 
583   return S_OK;
584 }
585 
586 HRESULT
openMediaCapture()587 MediaCaptureWrapper::openMediaCapture()
588 {
589   HRESULT hr;
590 
591   if (frame_reader_) {
592     GST_INFO ("Frame reader was configured");
593     return S_OK;
594   }
595 
596   if (source_group_ == nullptr) {
597     GST_WARNING ("No frame source group was specified");
598     return E_FAIL;
599   }
600 
601   if (media_desc_ == nullptr) {
602     GST_WARNING ("No media description was specified");
603     return E_FAIL;
604   }
605 
606   hr = mediaCaptureInitPre ();
607   if (!gst_mf_result (hr))
608     return hr;
609 
610   /* Wait user action and resulting mediaCaptureInitPost */
611   std::unique_lock<std::mutex> Lock(lock_);
612   if (!init_done_)
613     cond_.wait (Lock);
614 
615   return frame_reader_ ? S_OK : E_FAIL;
616 }
617 
618 HRESULT
mediaCaptureInitPre()619 MediaCaptureWrapper::mediaCaptureInitPre()
620 {
621   ComPtr<IAsyncAction> async_action;
622   HRESULT hr;
623 
624   auto work_item = Callback<Implements<RuntimeClassFlags<ClassicCom>,
625         IDispatchedHandler, FtmBase>>([this]{
626     ComPtr<IMediaCaptureInitializationSettings> settings;
627     ComPtr<IMediaCaptureInitializationSettings5> settings5;
628     ComPtr<IMediaCapture> media_capture;
629     ComPtr<IMediaCapture5> media_capture5;
630     ComPtr<IAsyncAction> init_async;
631     HRESULT hr;
632     HStringReference hstr_setting =
633         HStringReference (
634             RuntimeClass_Windows_Media_Capture_MediaCaptureInitializationSettings);
635     HStringReference hstr_capture =
636         HStringReference (RuntimeClass_Windows_Media_Capture_MediaCapture);
637     IMediaFrameSourceGroup * source_group =
638         source_group_->source_group_.Get();
639 
640     hr = ActivateInstance (hstr_setting.Get(), &settings);
641     if (!gst_mf_result (hr))
642       return hr;
643 
644     hr = settings->put_StreamingCaptureMode (
645         StreamingCaptureMode::StreamingCaptureMode_Video);
646     if (!gst_mf_result (hr))
647       return hr;
648 
649     hr = settings.As (&settings5);
650     if (!gst_mf_result (hr))
651       return hr;
652 
653     hr = settings5->put_SourceGroup (source_group);
654     if (!gst_mf_result (hr))
655       return hr;
656 
657     /* TODO: support D3D11 memory */
658     hr = settings5->put_MemoryPreference (
659         MediaCaptureMemoryPreference::MediaCaptureMemoryPreference_Cpu);
660     if (!gst_mf_result (hr))
661       return hr;
662 
663     hr = settings5.As (&settings);
664     if (!gst_mf_result (hr))
665       return hr;
666 
667     hr = ActivateInstance (hstr_capture.Get(), &media_capture5);
668     if (!gst_mf_result (hr))
669       return hr;
670 
671     hr = media_capture5.As (&media_capture);
672     if (!gst_mf_result (hr))
673       return hr;
674 
675     hr = media_capture->InitializeWithSettingsAsync (settings.Get(), &init_async);
676     if (!gst_mf_result (hr))
677       return hr;
678 
679     return StartAsyncThen(
680         init_async.Get(),
681         [this, init_async, media_capture](_In_ HRESULT hr,
682         _In_ IAsyncAction *asyncResult, _In_ AsyncStatus asyncStatus) -> HRESULT
683       {
684         return mediaCaptureInitPost (init_async, media_capture);
685       });
686   });
687 
688   init_done_ = false;
689 
690   if (dispatcher_) {
691     hr = dispatcher_->RunAsync(CoreDispatcherPriority_Normal, work_item.Get(),
692           &async_action);
693   } else {
694     hr = work_item->Invoke ();
695   }
696 
697   return hr;
698 }
699 
700 HRESULT
mediaCaptureInitPost(ComPtr<IAsyncAction> init_async,ComPtr<IMediaCapture> media_capture)701 MediaCaptureWrapper::mediaCaptureInitPost (ComPtr<IAsyncAction> init_async,
702     ComPtr<IMediaCapture> media_capture)
703 {
704   std::unique_lock<std::mutex> Lock(lock_);
705   ComPtr<IMediaFrameSource> frame_source;
706   ComPtr<IMapView<HSTRING, MediaFrameSource*>> frameSources;
707   ComPtr<IMediaFrameSource> source;
708   ComPtr<IMediaFrameFormat> format;
709   ComPtr<IVectorView<MediaFrameFormat*>> formatList;
710   ComPtr<IMediaCapture5> media_capture5;
711   ComPtr<IAsyncAction> set_format_async;
712   ComPtr<IAsyncOperation<MediaFrameReader*>> create_reader_async;
713   ComPtr<IMediaFrameReader> frame_reader;
714   boolean has_key;
715   UINT32 count = 0;
716   GstVideoInfo videoInfo;
717   HRESULT hr;
718   bool is_I420_subtype = false;
719   std::string target_subtype;
720   ComPtr<ITypedEventHandler<MediaFrameReader*, MediaFrameArrivedEventArgs*>>
721       frame_arrived_handler = Callback<ITypedEventHandler<MediaFrameReader*,
722           MediaFrameArrivedEventArgs*>> ([&]
723           (IMediaFrameReader * reader, IMediaFrameArrivedEventArgs* args)
724             {
725                 return onFrameArrived(reader, args);
726             }
727         );
728 
729   GST_DEBUG ("InitializeWithSettingsAsync done");
730 
731   hr = init_async->GetResults ();
732   if (!gst_mf_result (hr))
733     goto done;
734 
735   if (!gst_video_info_from_caps (&videoInfo, media_desc_->caps_)) {
736     GST_WARNING ("Couldn't convert caps %" GST_PTR_FORMAT " to videoinfo",
737         media_desc_->caps_);
738     hr = E_FAIL;
739     goto done;
740   }
741 
742   /* Windows defines two I420 format, one is I420 and the other is IYUV.
743    * So, if requested video format was I420, we should accept both I420 and
744    * IYUV formats */
745   is_I420_subtype = GST_VIDEO_INFO_FORMAT (&videoInfo) == GST_VIDEO_FORMAT_I420;
746   target_subtype = convert_hstring_to_string (&media_desc_->subtype_);
747 
748   hr = media_capture.As (&media_capture5);
749   if (!gst_mf_result (hr))
750     goto done;
751 
752   hr = media_capture5->get_FrameSources (&frameSources);
753   if (!gst_mf_result (hr))
754     goto done;
755 
756   hr = frameSources->HasKey (media_desc_->source_id_.Get(), &has_key);
757   if (!gst_mf_result (hr))
758     goto done;
759 
760   if (!has_key) {
761     GST_ERROR ("MediaFrameSource unavailable");
762     hr = E_FAIL;
763     goto done;
764   }
765 
766   hr = frameSources->Lookup (media_desc_->source_id_.Get(), &source);
767   if (!gst_mf_result (hr))
768     goto done;
769 
770   hr = source->get_SupportedFormats (&formatList);
771   if (!gst_mf_result (hr))
772     goto done;
773 
774   hr = formatList->get_Size (&count);
775   if (!gst_mf_result (hr))
776     goto done;
777 
778   if (count == 0) {
779     GST_ERROR ("No supported format object");
780     hr = E_FAIL;
781     goto done;
782   }
783 
784   GST_DEBUG ("Has %d available formats", count);
785   GST_INFO ("Finding matching IMediaFrameFormat with %" GST_PTR_FORMAT,
786       media_desc_->caps_);
787 
788 #ifndef GST_DISABLE_GST_DEBUG
789   if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_LOG) {
790     GST_LOG ("Dump IMediaFrameFormat list");
791     for (UINT32 i = 0; i < count; i++) {
792       ComPtr<IMediaFrameFormat> fmt;
793       ComPtr<IVideoMediaFrameFormat> videoFmt;
794       ComPtr<IMediaRatio> ratio;
795       HString subtype;
796       UINT32 width = 0;
797       UINT32 height = 0;
798       UINT32 fps_n = 0;
799       UINT32 fps_d = 1;
800 
801       hr = formatList->GetAt (i, &fmt);
802       if (!gst_mf_result (hr))
803         continue;
804 
805       hr = fmt->get_VideoFormat (&videoFmt);
806       if (!gst_mf_result (hr))
807         continue;
808 
809       hr = videoFmt->get_Width (&width);
810       if (!gst_mf_result (hr))
811         continue;
812 
813       hr = videoFmt->get_Height (&height);
814       if (!gst_mf_result (hr))
815         continue;
816 
817       hr = fmt->get_FrameRate (&ratio);
818       if (!gst_mf_result (hr))
819         continue;
820 
821       hr = ratio->get_Numerator (&fps_n);
822       if (!gst_mf_result (hr))
823         continue;
824 
825       hr = ratio->get_Denominator (&fps_d);
826       if (!gst_mf_result (hr))
827         continue;
828 
829       hr = fmt->get_Subtype (subtype.GetAddressOf ());
830       if (!gst_mf_result (hr))
831         continue;
832 
833       std::string cur_subtype = convert_hstring_to_string (&subtype);
834 
835       GST_LOG ("\tIMediaFrameFormat[%d] subtpye: %s, resolution: %dx%d, "
836           "framerate: %d/%d", i, cur_subtype.c_str (), width, height, fps_n,
837           fps_d);
838     }
839   }
840 #endif
841 
842   /* FIXME: support audio */
843   for (UINT32 i = 0; i < count; i++) {
844     ComPtr<IMediaFrameFormat> fmt;
845     ComPtr<IVideoMediaFrameFormat> videoFmt;
846     ComPtr<IMediaRatio> ratio;
847     HString subtype;
848     UINT32 width = 0;
849     UINT32 height = 0;
850     UINT32 fps_n = 0;
851     UINT32 fps_d = 1;
852 
853     hr = formatList->GetAt (i, &fmt);
854     if (!gst_mf_result (hr))
855       continue;
856 
857     hr = fmt->get_VideoFormat (&videoFmt);
858     if (!gst_mf_result (hr))
859       continue;
860 
861     hr = videoFmt->get_Width (&width);
862     if (!gst_mf_result (hr))
863       continue;
864 
865     hr = videoFmt->get_Height (&height);
866     if (!gst_mf_result (hr))
867       continue;
868 
869     if (width != GST_VIDEO_INFO_WIDTH (&videoInfo) ||
870         height != GST_VIDEO_INFO_HEIGHT (&videoInfo)) {
871       GST_DEBUG ("IMediaFrameFormat[%d], resolution %dx%d is not equal to "
872           "target resolution %dx%d", i, width, height,
873           GST_VIDEO_INFO_WIDTH (&videoInfo),
874           GST_VIDEO_INFO_HEIGHT (&videoInfo));
875       continue;
876     }
877 
878     hr = fmt->get_FrameRate (&ratio);
879     if (!gst_mf_result (hr))
880       continue;
881 
882     hr = ratio->get_Numerator (&fps_n);
883     if (!gst_mf_result (hr))
884       continue;
885 
886     hr = ratio->get_Denominator (&fps_d);
887     if (!gst_mf_result (hr))
888       continue;
889 
890     if ((gint) fps_n != GST_VIDEO_INFO_FPS_N (&videoInfo) ||
891         (gint) fps_d != GST_VIDEO_INFO_FPS_D (&videoInfo)) {
892       GST_DEBUG ("IMediaFrameFormat[%d], framerate %d/%d is not equal to "
893           "target framerate %d/%d", i, width, height,
894           GST_VIDEO_INFO_FPS_N (&videoInfo),
895           GST_VIDEO_INFO_FPS_D (&videoInfo));
896       continue;
897     }
898 
899     /* TODO: check major type for audio */
900     hr = fmt->get_Subtype (subtype.GetAddressOf ());
901     if (!gst_mf_result (hr))
902       continue;
903 
904     std::string cur_subtype = convert_hstring_to_string (&subtype);
905 
906     if (is_I420_subtype) {
907       /* Special handling for I420 */
908       if (cur_subtype != "I420" && cur_subtype != "IYUV") {
909         GST_LOG ("IMediaFrameFormat[%d], subtype %s is not equal to target %s",
910           i, cur_subtype.c_str (), target_subtype.c_str ());
911         continue;
912       }
913     } else if (cur_subtype != target_subtype) {
914       GST_LOG ("IMediaFrameFormat[%d], subtype %s is not equal to target %s",
915           i, cur_subtype.c_str (), target_subtype.c_str ());
916       continue;
917     }
918 
919     format = fmt;
920     break;
921   }
922 
923   if (!format) {
924     GST_ERROR (
925         "Couldn't find matching IMediaFrameFormat interface for %"
926         GST_PTR_FORMAT, media_desc_->caps_);
927     hr = E_FAIL;
928     goto done;
929   } else {
930     GST_INFO ("Found matching IMediaFrameFormat");
931   }
932 
933   hr = source->SetFormatAsync (format.Get (), &set_format_async);
934   if (!gst_mf_result (hr))
935     goto done;
936 
937   hr = SyncWait<void>(set_format_async.Get ());
938   if (!gst_mf_result (hr))
939     goto done;
940 
941   hr = set_format_async->GetResults ();
942   if (!gst_mf_result (hr))
943     goto done;
944 
945   hr = media_capture5->CreateFrameReaderAsync (source.Get(),
946       &create_reader_async);
947   if (!gst_mf_result (hr))
948     goto done;
949 
950   hr = SyncWait<MediaFrameReader*>(create_reader_async.Get());
951   if (!gst_mf_result (hr))
952     goto done;
953 
954   hr = create_reader_async->GetResults(&frame_reader);
955   if (!gst_mf_result (hr))
956     goto done;
957 
958   hr = frame_reader->add_FrameArrived (frame_arrived_handler.Get(),
959       &token_frame_arrived_);
960   if (!gst_mf_result (hr))
961     goto done;
962 
963   hr = media_capture->add_Failed
964       (Callback<IMediaCaptureFailedEventHandler> ([this]
965           (IMediaCapture * capture, IMediaCaptureFailedEventArgs* args)
966             {
967                 return onCaptureFailed(capture, args);
968             }
969         ).Get(),
970       &token_capture_failed_);
971 
972   if (!gst_mf_result (hr))
973     goto done;
974 
975   frame_reader_ = frame_reader;
976   media_capture_ = media_capture;
977 
978 done:
979   init_done_ = true;
980   cond_.notify_all();
981 
982   return S_OK;
983 }
984 
985 HRESULT
startCapture()986 MediaCaptureWrapper::startCapture()
987 {
988   HRESULT hr;
989 
990   if (!frame_reader_) {
991     GST_ERROR ("Frame reader wasn't configured");
992     return E_FAIL;
993   }
994 
995   ComPtr<IAsyncOperation<MediaFrameReaderStartStatus>> start_async;
996   hr = frame_reader_->StartAsync (&start_async);
997   if (!gst_mf_result (hr))
998     return hr;
999 
1000   hr = SyncWait<MediaFrameReaderStartStatus>(start_async.Get());
1001   if (!gst_mf_result (hr))
1002     return hr;
1003 
1004   MediaFrameReaderStartStatus reader_status;
1005   hr = start_async->GetResults(&reader_status);
1006   if (!gst_mf_result (hr))
1007     return hr;
1008 
1009   if (reader_status !=
1010       MediaFrameReaderStartStatus::MediaFrameReaderStartStatus_Success) {
1011     GST_ERROR ("Cannot start frame reader, status %d",
1012         (gint) reader_status);
1013     return E_FAIL;
1014   }
1015 
1016   return S_OK;
1017 }
1018 
1019 HRESULT
stopCapture()1020 MediaCaptureWrapper::stopCapture()
1021 {
1022   HRESULT hr = S_OK;
1023 
1024   if (frame_reader_) {
1025     ComPtr<IAsyncAction> async_action;
1026 
1027     hr = frame_reader_->StopAsync (&async_action);
1028     if (gst_mf_result (hr))
1029       hr = SyncWait<void>(async_action.Get ());
1030   }
1031 
1032   return hr;
1033 }
1034 
1035 HRESULT
onFrameArrived(IMediaFrameReader * reader,IMediaFrameArrivedEventArgs * args)1036 MediaCaptureWrapper::onFrameArrived(IMediaFrameReader *reader,
1037     IMediaFrameArrivedEventArgs *args)
1038 {
1039   HRESULT hr;
1040   ComPtr<IMediaFrameReference> frame_ref;
1041   ComPtr<IVideoMediaFrame> video_frame;
1042   ComPtr<ISoftwareBitmap> bitmap;
1043 
1044   hr = reader->TryAcquireLatestFrame (&frame_ref);
1045   if (!gst_mf_result (hr))
1046     return hr;
1047 
1048   if (!frame_ref)
1049     return S_OK;
1050 
1051   /* nothing to do if no callback was installed */
1052   if (!user_cb_.frame_arrived)
1053     return S_OK;
1054 
1055   return user_cb_.frame_arrived (frame_ref.Get(), user_data_);
1056 }
1057 
1058 HRESULT
onCaptureFailed(IMediaCapture * capture,IMediaCaptureFailedEventArgs * args)1059 MediaCaptureWrapper::onCaptureFailed(IMediaCapture *capture,
1060     IMediaCaptureFailedEventArgs *args)
1061 {
1062   HRESULT hr;
1063   UINT32 error_code = 0;
1064   HString hstr_error_msg;
1065   std::string error_msg;
1066 
1067   hr = args->get_Code (&error_code);
1068   gst_mf_result (hr);
1069 
1070   hr = args->get_Message (hstr_error_msg.GetAddressOf());
1071   gst_mf_result (hr);
1072 
1073   error_msg = convert_hstring_to_string (&hstr_error_msg);
1074 
1075   GST_WARNING ("Have error %s (%d)", error_msg.c_str(), error_code);
1076 
1077   if (user_cb_.failed)
1078     user_cb_.failed (error_msg, error_code, user_data_);
1079 
1080   return S_OK;
1081 }
1082 
1083 HRESULT
enumrateFrameSourceGroup(std::vector<GstWinRTMediaFrameSourceGroup> & groupList)1084 MediaCaptureWrapper::enumrateFrameSourceGroup
1085     (std::vector<GstWinRTMediaFrameSourceGroup> &groupList)
1086 {
1087   ComPtr<IMediaFrameSourceGroupStatics> frame_source_group_statics;
1088   ComPtr<IAsyncOperation<IVectorView<MediaFrameSourceGroup*>*>> async_op;
1089   ComPtr<IVectorView<MediaFrameSourceGroup*>> source_group_list;
1090   HRESULT hr;
1091   unsigned int cnt = 0;
1092   HStringReference hstr_frame_source_group =
1093       HStringReference (
1094           RuntimeClass_Windows_Media_Capture_Frames_MediaFrameSourceGroup);
1095 
1096   groupList.clear();
1097 
1098   hr = GetActivationFactory (hstr_frame_source_group.Get(),
1099                             &frame_source_group_statics);
1100   if (!gst_mf_result(hr))
1101     return hr;
1102 
1103   hr = frame_source_group_statics->FindAllAsync (&async_op);
1104   if (!gst_mf_result(hr))
1105     return hr;
1106 
1107   hr = SyncWait<IVectorView<MediaFrameSourceGroup*>*>(async_op.Get(), 5000);
1108   if (!gst_mf_result(hr))
1109     return hr;
1110 
1111   hr = async_op->GetResults (&source_group_list);
1112   if (!gst_mf_result(hr))
1113     return hr;
1114 
1115   hr = source_group_list->get_Size (&cnt);
1116   if (!gst_mf_result(hr))
1117     return hr;
1118 
1119   if (cnt == 0) {
1120     GST_WARNING ("No available source group");
1121     return E_FAIL;
1122   }
1123 
1124   GST_DEBUG("Have %u source group", cnt);
1125 
1126   for (unsigned int i = 0; i < cnt; i++) {
1127     ComPtr<IMediaFrameSourceGroup> group;
1128 
1129     hr = source_group_list->GetAt (i, &group);
1130     if (!gst_mf_result(hr))
1131       continue;
1132 
1133     GstWinRTMediaFrameSourceGroup source_group;
1134     hr = source_group.Fill(group, i);
1135     if (!gst_mf_result (hr))
1136       continue;
1137 
1138     groupList.push_back (source_group);
1139   }
1140 
1141   if (groupList.empty ()) {
1142     GST_WARNING("No available source group");
1143     return E_FAIL;
1144   }
1145 
1146   return S_OK;
1147 }
1148 
1149 HRESULT
FindCoreDispatcherForCurrentThread(ICoreDispatcher ** dispatcher)1150 FindCoreDispatcherForCurrentThread (ICoreDispatcher ** dispatcher)
1151 {
1152   HStringReference hstr_core_app =
1153       HStringReference(RuntimeClass_Windows_ApplicationModel_Core_CoreApplication);
1154   HRESULT hr;
1155 
1156   ComPtr<ICoreApplication> core_app;
1157   hr = GetActivationFactory (hstr_core_app.Get(), &core_app);
1158   if (FAILED (hr))
1159     return hr;
1160 
1161   ComPtr<ICoreApplicationView> core_app_view;
1162   hr = core_app->GetCurrentView (&core_app_view);
1163   if (FAILED (hr))
1164     return hr;
1165 
1166   ComPtr<ICoreWindow> core_window;
1167   hr = core_app_view->get_CoreWindow (&core_window);
1168   if (FAILED (hr))
1169     return hr;
1170 
1171   return core_window->get_Dispatcher (dispatcher);
1172 }
1173 
1174 bool
WinRTCapsCompareFunc(const GstWinRTMediaDescription & a,const GstWinRTMediaDescription & b)1175 WinRTCapsCompareFunc (const GstWinRTMediaDescription & a,
1176     const GstWinRTMediaDescription & b)
1177 {
1178   return gst_mf_source_object_caps_compare (a.caps_, b.caps_) < 0;
1179 }
1180 
1181 /* *INDENT-ON* */
1182