1 /*
2 * Copyright 2012 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "examples/peerconnection/client/conductor.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15
16 #include <memory>
17 #include <utility>
18 #include <vector>
19
20 #include "absl/memory/memory.h"
21 #include "absl/types/optional.h"
22 #include "api/audio/audio_mixer.h"
23 #include "api/audio_codecs/audio_decoder_factory.h"
24 #include "api/audio_codecs/audio_encoder_factory.h"
25 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
26 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
27 #include "api/audio_options.h"
28 #include "api/create_peerconnection_factory.h"
29 #include "api/rtp_sender_interface.h"
30 #include "api/video_codecs/builtin_video_decoder_factory.h"
31 #include "api/video_codecs/builtin_video_encoder_factory.h"
32 #include "api/video_codecs/video_decoder_factory.h"
33 #include "api/video_codecs/video_encoder_factory.h"
34 #include "examples/peerconnection/client/defaults.h"
35 #include "modules/audio_device/include/audio_device.h"
36 #include "modules/audio_processing/include/audio_processing.h"
37 #include "modules/video_capture/video_capture.h"
38 #include "modules/video_capture/video_capture_factory.h"
39 #include "p2p/base/port_allocator.h"
40 #include "pc/video_track_source.h"
41 #include "rtc_base/checks.h"
42 #include "rtc_base/logging.h"
43 #include "rtc_base/rtc_certificate_generator.h"
44 #include "rtc_base/strings/json.h"
45 #include "test/vcm_capturer.h"
46
47 namespace {
48 // Names used for a IceCandidate JSON object.
49 const char kCandidateSdpMidName[] = "sdpMid";
50 const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex";
51 const char kCandidateSdpName[] = "candidate";
52
53 // Names used for a SessionDescription JSON object.
54 const char kSessionDescriptionTypeName[] = "type";
55 const char kSessionDescriptionSdpName[] = "sdp";
56
57 class DummySetSessionDescriptionObserver
58 : public webrtc::SetSessionDescriptionObserver {
59 public:
Create()60 static rtc::scoped_refptr<DummySetSessionDescriptionObserver> Create() {
61 return rtc::make_ref_counted<DummySetSessionDescriptionObserver>();
62 }
OnSuccess()63 virtual void OnSuccess() { RTC_LOG(LS_INFO) << __FUNCTION__; }
OnFailure(webrtc::RTCError error)64 virtual void OnFailure(webrtc::RTCError error) {
65 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << ToString(error.type()) << ": "
66 << error.message();
67 }
68 };
69
70 class CapturerTrackSource : public webrtc::VideoTrackSource {
71 public:
Create()72 static rtc::scoped_refptr<CapturerTrackSource> Create() {
73 const size_t kWidth = 640;
74 const size_t kHeight = 480;
75 const size_t kFps = 30;
76 std::unique_ptr<webrtc::test::VcmCapturer> capturer;
77 std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
78 webrtc::VideoCaptureFactory::CreateDeviceInfo());
79 if (!info) {
80 return nullptr;
81 }
82 int num_devices = info->NumberOfDevices();
83 for (int i = 0; i < num_devices; ++i) {
84 capturer = absl::WrapUnique(
85 webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i));
86 if (capturer) {
87 return rtc::make_ref_counted<CapturerTrackSource>(std::move(capturer));
88 }
89 }
90
91 return nullptr;
92 }
93
94 protected:
CapturerTrackSource(std::unique_ptr<webrtc::test::VcmCapturer> capturer)95 explicit CapturerTrackSource(
96 std::unique_ptr<webrtc::test::VcmCapturer> capturer)
97 : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
98
99 private:
source()100 rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
101 return capturer_.get();
102 }
103 std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
104 };
105
106 } // namespace
107
Conductor(PeerConnectionClient * client,MainWindow * main_wnd)108 Conductor::Conductor(PeerConnectionClient* client, MainWindow* main_wnd)
109 : peer_id_(-1), loopback_(false), client_(client), main_wnd_(main_wnd) {
110 client_->RegisterObserver(this);
111 main_wnd->RegisterObserver(this);
112 }
113
~Conductor()114 Conductor::~Conductor() {
115 RTC_DCHECK(!peer_connection_);
116 }
117
connection_active() const118 bool Conductor::connection_active() const {
119 return peer_connection_ != nullptr;
120 }
121
Close()122 void Conductor::Close() {
123 client_->SignOut();
124 DeletePeerConnection();
125 }
126
InitializePeerConnection()127 bool Conductor::InitializePeerConnection() {
128 RTC_DCHECK(!peer_connection_factory_);
129 RTC_DCHECK(!peer_connection_);
130
131 if (!signaling_thread_.get()) {
132 signaling_thread_ = rtc::Thread::CreateWithSocketServer();
133 signaling_thread_->Start();
134 }
135 peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
136 nullptr /* network_thread */, nullptr /* worker_thread */,
137 signaling_thread_.get(), nullptr /* default_adm */,
138 webrtc::CreateBuiltinAudioEncoderFactory(),
139 webrtc::CreateBuiltinAudioDecoderFactory(),
140 webrtc::CreateBuiltinVideoEncoderFactory(),
141 webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */,
142 nullptr /* audio_processing */);
143
144 if (!peer_connection_factory_) {
145 main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
146 true);
147 DeletePeerConnection();
148 return false;
149 }
150
151 if (!CreatePeerConnection()) {
152 main_wnd_->MessageBox("Error", "CreatePeerConnection failed", true);
153 DeletePeerConnection();
154 }
155
156 AddTracks();
157
158 return peer_connection_ != nullptr;
159 }
160
ReinitializePeerConnectionForLoopback()161 bool Conductor::ReinitializePeerConnectionForLoopback() {
162 loopback_ = true;
163 std::vector<rtc::scoped_refptr<webrtc::RtpSenderInterface>> senders =
164 peer_connection_->GetSenders();
165 peer_connection_ = nullptr;
166 // Loopback is only possible if encryption is disabled.
167 webrtc::PeerConnectionFactoryInterface::Options options;
168 options.disable_encryption = true;
169 peer_connection_factory_->SetOptions(options);
170 if (CreatePeerConnection()) {
171 for (const auto& sender : senders) {
172 peer_connection_->AddTrack(sender->track(), sender->stream_ids());
173 }
174 peer_connection_->CreateOffer(
175 this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
176 }
177 options.disable_encryption = false;
178 peer_connection_factory_->SetOptions(options);
179 return peer_connection_ != nullptr;
180 }
181
CreatePeerConnection()182 bool Conductor::CreatePeerConnection() {
183 RTC_DCHECK(peer_connection_factory_);
184 RTC_DCHECK(!peer_connection_);
185
186 webrtc::PeerConnectionInterface::RTCConfiguration config;
187 config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
188 webrtc::PeerConnectionInterface::IceServer server;
189 server.uri = GetPeerConnectionString();
190 config.servers.push_back(server);
191
192 webrtc::PeerConnectionDependencies pc_dependencies(this);
193 auto error_or_peer_connection =
194 peer_connection_factory_->CreatePeerConnectionOrError(
195 config, std::move(pc_dependencies));
196 if (error_or_peer_connection.ok()) {
197 peer_connection_ = std::move(error_or_peer_connection.value());
198 }
199 return peer_connection_ != nullptr;
200 }
201
DeletePeerConnection()202 void Conductor::DeletePeerConnection() {
203 main_wnd_->StopLocalRenderer();
204 main_wnd_->StopRemoteRenderer();
205 peer_connection_ = nullptr;
206 peer_connection_factory_ = nullptr;
207 peer_id_ = -1;
208 loopback_ = false;
209 }
210
EnsureStreamingUI()211 void Conductor::EnsureStreamingUI() {
212 RTC_DCHECK(peer_connection_);
213 if (main_wnd_->IsWindow()) {
214 if (main_wnd_->current_ui() != MainWindow::STREAMING)
215 main_wnd_->SwitchToStreamingUI();
216 }
217 }
218
219 //
220 // PeerConnectionObserver implementation.
221 //
222
OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> & streams)223 void Conductor::OnAddTrack(
224 rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
225 const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>>&
226 streams) {
227 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id();
228 main_wnd_->QueueUIThreadCallback(NEW_TRACK_ADDED,
229 receiver->track().release());
230 }
231
OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver)232 void Conductor::OnRemoveTrack(
233 rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) {
234 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id();
235 main_wnd_->QueueUIThreadCallback(TRACK_REMOVED, receiver->track().release());
236 }
237
OnIceCandidate(const webrtc::IceCandidateInterface * candidate)238 void Conductor::OnIceCandidate(const webrtc::IceCandidateInterface* candidate) {
239 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
240 // For loopback test. To save some connecting delay.
241 if (loopback_) {
242 if (!peer_connection_->AddIceCandidate(candidate)) {
243 RTC_LOG(LS_WARNING) << "Failed to apply the received candidate";
244 }
245 return;
246 }
247
248 Json::Value jmessage;
249 jmessage[kCandidateSdpMidName] = candidate->sdp_mid();
250 jmessage[kCandidateSdpMlineIndexName] = candidate->sdp_mline_index();
251 std::string sdp;
252 if (!candidate->ToString(&sdp)) {
253 RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
254 return;
255 }
256 jmessage[kCandidateSdpName] = sdp;
257
258 Json::StreamWriterBuilder factory;
259 SendMessage(Json::writeString(factory, jmessage));
260 }
261
262 //
263 // PeerConnectionClientObserver implementation.
264 //
265
OnSignedIn()266 void Conductor::OnSignedIn() {
267 RTC_LOG(LS_INFO) << __FUNCTION__;
268 main_wnd_->SwitchToPeerList(client_->peers());
269 }
270
OnDisconnected()271 void Conductor::OnDisconnected() {
272 RTC_LOG(LS_INFO) << __FUNCTION__;
273
274 DeletePeerConnection();
275
276 if (main_wnd_->IsWindow())
277 main_wnd_->SwitchToConnectUI();
278 }
279
OnPeerConnected(int id,const std::string & name)280 void Conductor::OnPeerConnected(int id, const std::string& name) {
281 RTC_LOG(LS_INFO) << __FUNCTION__;
282 // Refresh the list if we're showing it.
283 if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
284 main_wnd_->SwitchToPeerList(client_->peers());
285 }
286
OnPeerDisconnected(int id)287 void Conductor::OnPeerDisconnected(int id) {
288 RTC_LOG(LS_INFO) << __FUNCTION__;
289 if (id == peer_id_) {
290 RTC_LOG(LS_INFO) << "Our peer disconnected";
291 main_wnd_->QueueUIThreadCallback(PEER_CONNECTION_CLOSED, NULL);
292 } else {
293 // Refresh the list if we're showing it.
294 if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
295 main_wnd_->SwitchToPeerList(client_->peers());
296 }
297 }
298
OnMessageFromPeer(int peer_id,const std::string & message)299 void Conductor::OnMessageFromPeer(int peer_id, const std::string& message) {
300 RTC_DCHECK(peer_id_ == peer_id || peer_id_ == -1);
301 RTC_DCHECK(!message.empty());
302
303 if (!peer_connection_.get()) {
304 RTC_DCHECK(peer_id_ == -1);
305 peer_id_ = peer_id;
306
307 if (!InitializePeerConnection()) {
308 RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
309 client_->SignOut();
310 return;
311 }
312 } else if (peer_id != peer_id_) {
313 RTC_DCHECK(peer_id_ != -1);
314 RTC_LOG(LS_WARNING)
315 << "Received a message from unknown peer while already in a "
316 "conversation with a different peer.";
317 return;
318 }
319
320 Json::CharReaderBuilder factory;
321 std::unique_ptr<Json::CharReader> reader =
322 absl::WrapUnique(factory.newCharReader());
323 Json::Value jmessage;
324 if (!reader->parse(message.data(), message.data() + message.length(),
325 &jmessage, nullptr)) {
326 RTC_LOG(LS_WARNING) << "Received unknown message. " << message;
327 return;
328 }
329 std::string type_str;
330 std::string json_object;
331
332 rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName,
333 &type_str);
334 if (!type_str.empty()) {
335 if (type_str == "offer-loopback") {
336 // This is a loopback call.
337 // Recreate the peerconnection with DTLS disabled.
338 if (!ReinitializePeerConnectionForLoopback()) {
339 RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
340 DeletePeerConnection();
341 client_->SignOut();
342 }
343 return;
344 }
345 absl::optional<webrtc::SdpType> type_maybe =
346 webrtc::SdpTypeFromString(type_str);
347 if (!type_maybe) {
348 RTC_LOG(LS_ERROR) << "Unknown SDP type: " << type_str;
349 return;
350 }
351 webrtc::SdpType type = *type_maybe;
352 std::string sdp;
353 if (!rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionSdpName,
354 &sdp)) {
355 RTC_LOG(LS_WARNING)
356 << "Can't parse received session description message.";
357 return;
358 }
359 webrtc::SdpParseError error;
360 std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
361 webrtc::CreateSessionDescription(type, sdp, &error);
362 if (!session_description) {
363 RTC_LOG(LS_WARNING)
364 << "Can't parse received session description message. "
365 "SdpParseError was: "
366 << error.description;
367 return;
368 }
369 RTC_LOG(LS_INFO) << " Received session description :" << message;
370 peer_connection_->SetRemoteDescription(
371 DummySetSessionDescriptionObserver::Create().get(),
372 session_description.release());
373 if (type == webrtc::SdpType::kOffer) {
374 peer_connection_->CreateAnswer(
375 this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
376 }
377 } else {
378 std::string sdp_mid;
379 int sdp_mlineindex = 0;
380 std::string sdp;
381 if (!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpMidName,
382 &sdp_mid) ||
383 !rtc::GetIntFromJsonObject(jmessage, kCandidateSdpMlineIndexName,
384 &sdp_mlineindex) ||
385 !rtc::GetStringFromJsonObject(jmessage, kCandidateSdpName, &sdp)) {
386 RTC_LOG(LS_WARNING) << "Can't parse received message.";
387 return;
388 }
389 webrtc::SdpParseError error;
390 std::unique_ptr<webrtc::IceCandidateInterface> candidate(
391 webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error));
392 if (!candidate.get()) {
393 RTC_LOG(LS_WARNING) << "Can't parse received candidate message. "
394 "SdpParseError was: "
395 << error.description;
396 return;
397 }
398 if (!peer_connection_->AddIceCandidate(candidate.get())) {
399 RTC_LOG(LS_WARNING) << "Failed to apply the received candidate";
400 return;
401 }
402 RTC_LOG(LS_INFO) << " Received candidate :" << message;
403 }
404 }
405
OnMessageSent(int err)406 void Conductor::OnMessageSent(int err) {
407 // Process the next pending message if any.
408 main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, NULL);
409 }
410
OnServerConnectionFailure()411 void Conductor::OnServerConnectionFailure() {
412 main_wnd_->MessageBox("Error", ("Failed to connect to " + server_).c_str(),
413 true);
414 }
415
416 //
417 // MainWndCallback implementation.
418 //
419
StartLogin(const std::string & server,int port)420 void Conductor::StartLogin(const std::string& server, int port) {
421 if (client_->is_connected())
422 return;
423 server_ = server;
424 client_->Connect(server, port, GetPeerName());
425 }
426
DisconnectFromServer()427 void Conductor::DisconnectFromServer() {
428 if (client_->is_connected())
429 client_->SignOut();
430 }
431
ConnectToPeer(int peer_id)432 void Conductor::ConnectToPeer(int peer_id) {
433 RTC_DCHECK(peer_id_ == -1);
434 RTC_DCHECK(peer_id != -1);
435
436 if (peer_connection_.get()) {
437 main_wnd_->MessageBox(
438 "Error", "We only support connecting to one peer at a time", true);
439 return;
440 }
441
442 if (InitializePeerConnection()) {
443 peer_id_ = peer_id;
444 peer_connection_->CreateOffer(
445 this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
446 } else {
447 main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
448 }
449 }
450
AddTracks()451 void Conductor::AddTracks() {
452 if (!peer_connection_->GetSenders().empty()) {
453 return; // Already added tracks.
454 }
455
456 rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(
457 peer_connection_factory_->CreateAudioTrack(
458 kAudioLabel,
459 peer_connection_factory_->CreateAudioSource(cricket::AudioOptions())
460 .get()));
461 auto result_or_error = peer_connection_->AddTrack(audio_track, {kStreamId});
462 if (!result_or_error.ok()) {
463 RTC_LOG(LS_ERROR) << "Failed to add audio track to PeerConnection: "
464 << result_or_error.error().message();
465 }
466
467 rtc::scoped_refptr<CapturerTrackSource> video_device =
468 CapturerTrackSource::Create();
469 if (video_device) {
470 rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(
471 peer_connection_factory_->CreateVideoTrack(kVideoLabel,
472 video_device.get()));
473 main_wnd_->StartLocalRenderer(video_track_.get());
474
475 result_or_error = peer_connection_->AddTrack(video_track_, {kStreamId});
476 if (!result_or_error.ok()) {
477 RTC_LOG(LS_ERROR) << "Failed to add video track to PeerConnection: "
478 << result_or_error.error().message();
479 }
480 } else {
481 RTC_LOG(LS_ERROR) << "OpenVideoCaptureDevice failed";
482 }
483
484 main_wnd_->SwitchToStreamingUI();
485 }
486
DisconnectFromCurrentPeer()487 void Conductor::DisconnectFromCurrentPeer() {
488 RTC_LOG(LS_INFO) << __FUNCTION__;
489 if (peer_connection_.get()) {
490 client_->SendHangUp(peer_id_);
491 DeletePeerConnection();
492 }
493
494 if (main_wnd_->IsWindow())
495 main_wnd_->SwitchToPeerList(client_->peers());
496 }
497
UIThreadCallback(int msg_id,void * data)498 void Conductor::UIThreadCallback(int msg_id, void* data) {
499 switch (msg_id) {
500 case PEER_CONNECTION_CLOSED:
501 RTC_LOG(LS_INFO) << "PEER_CONNECTION_CLOSED";
502 DeletePeerConnection();
503
504 if (main_wnd_->IsWindow()) {
505 if (client_->is_connected()) {
506 main_wnd_->SwitchToPeerList(client_->peers());
507 } else {
508 main_wnd_->SwitchToConnectUI();
509 }
510 } else {
511 DisconnectFromServer();
512 }
513 break;
514
515 case SEND_MESSAGE_TO_PEER: {
516 RTC_LOG(LS_INFO) << "SEND_MESSAGE_TO_PEER";
517 std::string* msg = reinterpret_cast<std::string*>(data);
518 if (msg) {
519 // For convenience, we always run the message through the queue.
520 // This way we can be sure that messages are sent to the server
521 // in the same order they were signaled without much hassle.
522 pending_messages_.push_back(msg);
523 }
524
525 if (!pending_messages_.empty() && !client_->IsSendingMessage()) {
526 msg = pending_messages_.front();
527 pending_messages_.pop_front();
528
529 if (!client_->SendToPeer(peer_id_, *msg) && peer_id_ != -1) {
530 RTC_LOG(LS_ERROR) << "SendToPeer failed";
531 DisconnectFromServer();
532 }
533 delete msg;
534 }
535
536 if (!peer_connection_.get())
537 peer_id_ = -1;
538
539 break;
540 }
541
542 case NEW_TRACK_ADDED: {
543 auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
544 if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
545 auto* video_track = static_cast<webrtc::VideoTrackInterface*>(track);
546 main_wnd_->StartRemoteRenderer(video_track);
547 }
548 track->Release();
549 break;
550 }
551
552 case TRACK_REMOVED: {
553 // Remote peer stopped sending a track.
554 auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
555 track->Release();
556 break;
557 }
558
559 default:
560 RTC_DCHECK_NOTREACHED();
561 break;
562 }
563 }
564
OnSuccess(webrtc::SessionDescriptionInterface * desc)565 void Conductor::OnSuccess(webrtc::SessionDescriptionInterface* desc) {
566 peer_connection_->SetLocalDescription(
567 DummySetSessionDescriptionObserver::Create().get(), desc);
568
569 std::string sdp;
570 desc->ToString(&sdp);
571
572 // For loopback test. To save some connecting delay.
573 if (loopback_) {
574 // Replace message type from "offer" to "answer"
575 std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
576 webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp);
577 peer_connection_->SetRemoteDescription(
578 DummySetSessionDescriptionObserver::Create().get(),
579 session_description.release());
580 return;
581 }
582
583 Json::Value jmessage;
584 jmessage[kSessionDescriptionTypeName] =
585 webrtc::SdpTypeToString(desc->GetType());
586 jmessage[kSessionDescriptionSdpName] = sdp;
587
588 Json::StreamWriterBuilder factory;
589 SendMessage(Json::writeString(factory, jmessage));
590 }
591
OnFailure(webrtc::RTCError error)592 void Conductor::OnFailure(webrtc::RTCError error) {
593 RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message();
594 }
595
SendMessage(const std::string & json_object)596 void Conductor::SendMessage(const std::string& json_object) {
597 std::string* msg = new std::string(json_object);
598 main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, msg);
599 }
600