1 /*
2 * Copyright 2012 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "examples/peerconnection/client/conductor.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15
16 #include <memory>
17 #include <utility>
18 #include <vector>
19
20 #include "absl/memory/memory.h"
21 #include "absl/types/optional.h"
22 #include "api/audio/audio_mixer.h"
23 #include "api/audio_codecs/audio_decoder_factory.h"
24 #include "api/audio_codecs/audio_encoder_factory.h"
25 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
26 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
27 #include "api/audio_options.h"
28 #include "api/create_peerconnection_factory.h"
29 #include "api/rtp_sender_interface.h"
30 #include "api/video_codecs/builtin_video_decoder_factory.h"
31 #include "api/video_codecs/builtin_video_encoder_factory.h"
32 #include "api/video_codecs/video_decoder_factory.h"
33 #include "api/video_codecs/video_encoder_factory.h"
34 #include "examples/peerconnection/client/defaults.h"
35 #include "modules/audio_device/include/audio_device.h"
36 #include "modules/audio_processing/include/audio_processing.h"
37 #include "modules/video_capture/video_capture.h"
38 #include "modules/video_capture/video_capture_factory.h"
39 #include "p2p/base/port_allocator.h"
40 #include "pc/video_track_source.h"
41 #include "rtc_base/checks.h"
42 #include "rtc_base/logging.h"
43 #include "rtc_base/ref_counted_object.h"
44 #include "rtc_base/rtc_certificate_generator.h"
45 #include "rtc_base/strings/json.h"
46 #include "test/vcm_capturer.h"
47
48 namespace {
49 // Names used for a IceCandidate JSON object.
50 const char kCandidateSdpMidName[] = "sdpMid";
51 const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex";
52 const char kCandidateSdpName[] = "candidate";
53
54 // Names used for a SessionDescription JSON object.
55 const char kSessionDescriptionTypeName[] = "type";
56 const char kSessionDescriptionSdpName[] = "sdp";
57
58 class DummySetSessionDescriptionObserver
59 : public webrtc::SetSessionDescriptionObserver {
60 public:
Create()61 static DummySetSessionDescriptionObserver* Create() {
62 return new rtc::RefCountedObject<DummySetSessionDescriptionObserver>();
63 }
OnSuccess()64 virtual void OnSuccess() { RTC_LOG(INFO) << __FUNCTION__; }
OnFailure(webrtc::RTCError error)65 virtual void OnFailure(webrtc::RTCError error) {
66 RTC_LOG(INFO) << __FUNCTION__ << " " << ToString(error.type()) << ": "
67 << error.message();
68 }
69 };
70
71 class CapturerTrackSource : public webrtc::VideoTrackSource {
72 public:
Create()73 static rtc::scoped_refptr<CapturerTrackSource> Create() {
74 const size_t kWidth = 640;
75 const size_t kHeight = 480;
76 const size_t kFps = 30;
77 std::unique_ptr<webrtc::test::VcmCapturer> capturer;
78 std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
79 webrtc::VideoCaptureFactory::CreateDeviceInfo());
80 if (!info) {
81 return nullptr;
82 }
83 int num_devices = info->NumberOfDevices();
84 for (int i = 0; i < num_devices; ++i) {
85 capturer = absl::WrapUnique(
86 webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i));
87 if (capturer) {
88 return new rtc::RefCountedObject<CapturerTrackSource>(
89 std::move(capturer));
90 }
91 }
92
93 return nullptr;
94 }
95
96 protected:
CapturerTrackSource(std::unique_ptr<webrtc::test::VcmCapturer> capturer)97 explicit CapturerTrackSource(
98 std::unique_ptr<webrtc::test::VcmCapturer> capturer)
99 : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
100
101 private:
source()102 rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
103 return capturer_.get();
104 }
105 std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
106 };
107
108 } // namespace
109
Conductor(PeerConnectionClient * client,MainWindow * main_wnd)110 Conductor::Conductor(PeerConnectionClient* client, MainWindow* main_wnd)
111 : peer_id_(-1), loopback_(false), client_(client), main_wnd_(main_wnd) {
112 client_->RegisterObserver(this);
113 main_wnd->RegisterObserver(this);
114 }
115
~Conductor()116 Conductor::~Conductor() {
117 RTC_DCHECK(!peer_connection_);
118 }
119
connection_active() const120 bool Conductor::connection_active() const {
121 return peer_connection_ != nullptr;
122 }
123
Close()124 void Conductor::Close() {
125 client_->SignOut();
126 DeletePeerConnection();
127 }
128
InitializePeerConnection()129 bool Conductor::InitializePeerConnection() {
130 RTC_DCHECK(!peer_connection_factory_);
131 RTC_DCHECK(!peer_connection_);
132
133 peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
134 nullptr /* network_thread */, nullptr /* worker_thread */,
135 nullptr /* signaling_thread */, nullptr /* default_adm */,
136 webrtc::CreateBuiltinAudioEncoderFactory(),
137 webrtc::CreateBuiltinAudioDecoderFactory(),
138 webrtc::CreateBuiltinVideoEncoderFactory(),
139 webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */,
140 nullptr /* audio_processing */);
141
142 if (!peer_connection_factory_) {
143 main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
144 true);
145 DeletePeerConnection();
146 return false;
147 }
148
149 if (!CreatePeerConnection(/*dtls=*/true)) {
150 main_wnd_->MessageBox("Error", "CreatePeerConnection failed", true);
151 DeletePeerConnection();
152 }
153
154 AddTracks();
155
156 return peer_connection_ != nullptr;
157 }
158
ReinitializePeerConnectionForLoopback()159 bool Conductor::ReinitializePeerConnectionForLoopback() {
160 loopback_ = true;
161 std::vector<rtc::scoped_refptr<webrtc::RtpSenderInterface>> senders =
162 peer_connection_->GetSenders();
163 peer_connection_ = nullptr;
164 if (CreatePeerConnection(/*dtls=*/false)) {
165 for (const auto& sender : senders) {
166 peer_connection_->AddTrack(sender->track(), sender->stream_ids());
167 }
168 peer_connection_->CreateOffer(
169 this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
170 }
171 return peer_connection_ != nullptr;
172 }
173
CreatePeerConnection(bool dtls)174 bool Conductor::CreatePeerConnection(bool dtls) {
175 RTC_DCHECK(peer_connection_factory_);
176 RTC_DCHECK(!peer_connection_);
177
178 webrtc::PeerConnectionInterface::RTCConfiguration config;
179 config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
180 config.enable_dtls_srtp = dtls;
181 webrtc::PeerConnectionInterface::IceServer server;
182 server.uri = GetPeerConnectionString();
183 config.servers.push_back(server);
184
185 peer_connection_ = peer_connection_factory_->CreatePeerConnection(
186 config, nullptr, nullptr, this);
187 return peer_connection_ != nullptr;
188 }
189
DeletePeerConnection()190 void Conductor::DeletePeerConnection() {
191 main_wnd_->StopLocalRenderer();
192 main_wnd_->StopRemoteRenderer();
193 peer_connection_ = nullptr;
194 peer_connection_factory_ = nullptr;
195 peer_id_ = -1;
196 loopback_ = false;
197 }
198
EnsureStreamingUI()199 void Conductor::EnsureStreamingUI() {
200 RTC_DCHECK(peer_connection_);
201 if (main_wnd_->IsWindow()) {
202 if (main_wnd_->current_ui() != MainWindow::STREAMING)
203 main_wnd_->SwitchToStreamingUI();
204 }
205 }
206
207 //
208 // PeerConnectionObserver implementation.
209 //
210
OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> & streams)211 void Conductor::OnAddTrack(
212 rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
213 const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>>&
214 streams) {
215 RTC_LOG(INFO) << __FUNCTION__ << " " << receiver->id();
216 main_wnd_->QueueUIThreadCallback(NEW_TRACK_ADDED,
217 receiver->track().release());
218 }
219
OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver)220 void Conductor::OnRemoveTrack(
221 rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) {
222 RTC_LOG(INFO) << __FUNCTION__ << " " << receiver->id();
223 main_wnd_->QueueUIThreadCallback(TRACK_REMOVED, receiver->track().release());
224 }
225
OnIceCandidate(const webrtc::IceCandidateInterface * candidate)226 void Conductor::OnIceCandidate(const webrtc::IceCandidateInterface* candidate) {
227 RTC_LOG(INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
228 // For loopback test. To save some connecting delay.
229 if (loopback_) {
230 if (!peer_connection_->AddIceCandidate(candidate)) {
231 RTC_LOG(WARNING) << "Failed to apply the received candidate";
232 }
233 return;
234 }
235
236 Json::StyledWriter writer;
237 Json::Value jmessage;
238
239 jmessage[kCandidateSdpMidName] = candidate->sdp_mid();
240 jmessage[kCandidateSdpMlineIndexName] = candidate->sdp_mline_index();
241 std::string sdp;
242 if (!candidate->ToString(&sdp)) {
243 RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
244 return;
245 }
246 jmessage[kCandidateSdpName] = sdp;
247 SendMessage(writer.write(jmessage));
248 }
249
250 //
251 // PeerConnectionClientObserver implementation.
252 //
253
OnSignedIn()254 void Conductor::OnSignedIn() {
255 RTC_LOG(INFO) << __FUNCTION__;
256 main_wnd_->SwitchToPeerList(client_->peers());
257 }
258
OnDisconnected()259 void Conductor::OnDisconnected() {
260 RTC_LOG(INFO) << __FUNCTION__;
261
262 DeletePeerConnection();
263
264 if (main_wnd_->IsWindow())
265 main_wnd_->SwitchToConnectUI();
266 }
267
OnPeerConnected(int id,const std::string & name)268 void Conductor::OnPeerConnected(int id, const std::string& name) {
269 RTC_LOG(INFO) << __FUNCTION__;
270 // Refresh the list if we're showing it.
271 if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
272 main_wnd_->SwitchToPeerList(client_->peers());
273 }
274
OnPeerDisconnected(int id)275 void Conductor::OnPeerDisconnected(int id) {
276 RTC_LOG(INFO) << __FUNCTION__;
277 if (id == peer_id_) {
278 RTC_LOG(INFO) << "Our peer disconnected";
279 main_wnd_->QueueUIThreadCallback(PEER_CONNECTION_CLOSED, NULL);
280 } else {
281 // Refresh the list if we're showing it.
282 if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
283 main_wnd_->SwitchToPeerList(client_->peers());
284 }
285 }
286
OnMessageFromPeer(int peer_id,const std::string & message)287 void Conductor::OnMessageFromPeer(int peer_id, const std::string& message) {
288 RTC_DCHECK(peer_id_ == peer_id || peer_id_ == -1);
289 RTC_DCHECK(!message.empty());
290
291 if (!peer_connection_.get()) {
292 RTC_DCHECK(peer_id_ == -1);
293 peer_id_ = peer_id;
294
295 if (!InitializePeerConnection()) {
296 RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
297 client_->SignOut();
298 return;
299 }
300 } else if (peer_id != peer_id_) {
301 RTC_DCHECK(peer_id_ != -1);
302 RTC_LOG(WARNING)
303 << "Received a message from unknown peer while already in a "
304 "conversation with a different peer.";
305 return;
306 }
307
308 Json::Reader reader;
309 Json::Value jmessage;
310 if (!reader.parse(message, jmessage)) {
311 RTC_LOG(WARNING) << "Received unknown message. " << message;
312 return;
313 }
314 std::string type_str;
315 std::string json_object;
316
317 rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName,
318 &type_str);
319 if (!type_str.empty()) {
320 if (type_str == "offer-loopback") {
321 // This is a loopback call.
322 // Recreate the peerconnection with DTLS disabled.
323 if (!ReinitializePeerConnectionForLoopback()) {
324 RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
325 DeletePeerConnection();
326 client_->SignOut();
327 }
328 return;
329 }
330 absl::optional<webrtc::SdpType> type_maybe =
331 webrtc::SdpTypeFromString(type_str);
332 if (!type_maybe) {
333 RTC_LOG(LS_ERROR) << "Unknown SDP type: " << type_str;
334 return;
335 }
336 webrtc::SdpType type = *type_maybe;
337 std::string sdp;
338 if (!rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionSdpName,
339 &sdp)) {
340 RTC_LOG(WARNING) << "Can't parse received session description message.";
341 return;
342 }
343 webrtc::SdpParseError error;
344 std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
345 webrtc::CreateSessionDescription(type, sdp, &error);
346 if (!session_description) {
347 RTC_LOG(WARNING) << "Can't parse received session description message. "
348 "SdpParseError was: "
349 << error.description;
350 return;
351 }
352 RTC_LOG(INFO) << " Received session description :" << message;
353 peer_connection_->SetRemoteDescription(
354 DummySetSessionDescriptionObserver::Create(),
355 session_description.release());
356 if (type == webrtc::SdpType::kOffer) {
357 peer_connection_->CreateAnswer(
358 this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
359 }
360 } else {
361 std::string sdp_mid;
362 int sdp_mlineindex = 0;
363 std::string sdp;
364 if (!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpMidName,
365 &sdp_mid) ||
366 !rtc::GetIntFromJsonObject(jmessage, kCandidateSdpMlineIndexName,
367 &sdp_mlineindex) ||
368 !rtc::GetStringFromJsonObject(jmessage, kCandidateSdpName, &sdp)) {
369 RTC_LOG(WARNING) << "Can't parse received message.";
370 return;
371 }
372 webrtc::SdpParseError error;
373 std::unique_ptr<webrtc::IceCandidateInterface> candidate(
374 webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error));
375 if (!candidate.get()) {
376 RTC_LOG(WARNING) << "Can't parse received candidate message. "
377 "SdpParseError was: "
378 << error.description;
379 return;
380 }
381 if (!peer_connection_->AddIceCandidate(candidate.get())) {
382 RTC_LOG(WARNING) << "Failed to apply the received candidate";
383 return;
384 }
385 RTC_LOG(INFO) << " Received candidate :" << message;
386 }
387 }
388
OnMessageSent(int err)389 void Conductor::OnMessageSent(int err) {
390 // Process the next pending message if any.
391 main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, NULL);
392 }
393
OnServerConnectionFailure()394 void Conductor::OnServerConnectionFailure() {
395 main_wnd_->MessageBox("Error", ("Failed to connect to " + server_).c_str(),
396 true);
397 }
398
399 //
400 // MainWndCallback implementation.
401 //
402
StartLogin(const std::string & server,int port)403 void Conductor::StartLogin(const std::string& server, int port) {
404 if (client_->is_connected())
405 return;
406 server_ = server;
407 client_->Connect(server, port, GetPeerName());
408 }
409
DisconnectFromServer()410 void Conductor::DisconnectFromServer() {
411 if (client_->is_connected())
412 client_->SignOut();
413 }
414
ConnectToPeer(int peer_id)415 void Conductor::ConnectToPeer(int peer_id) {
416 RTC_DCHECK(peer_id_ == -1);
417 RTC_DCHECK(peer_id != -1);
418
419 if (peer_connection_.get()) {
420 main_wnd_->MessageBox(
421 "Error", "We only support connecting to one peer at a time", true);
422 return;
423 }
424
425 if (InitializePeerConnection()) {
426 peer_id_ = peer_id;
427 peer_connection_->CreateOffer(
428 this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
429 } else {
430 main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
431 }
432 }
433
AddTracks()434 void Conductor::AddTracks() {
435 if (!peer_connection_->GetSenders().empty()) {
436 return; // Already added tracks.
437 }
438
439 rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(
440 peer_connection_factory_->CreateAudioTrack(
441 kAudioLabel, peer_connection_factory_->CreateAudioSource(
442 cricket::AudioOptions())));
443 auto result_or_error = peer_connection_->AddTrack(audio_track, {kStreamId});
444 if (!result_or_error.ok()) {
445 RTC_LOG(LS_ERROR) << "Failed to add audio track to PeerConnection: "
446 << result_or_error.error().message();
447 }
448
449 rtc::scoped_refptr<CapturerTrackSource> video_device =
450 CapturerTrackSource::Create();
451 if (video_device) {
452 rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(
453 peer_connection_factory_->CreateVideoTrack(kVideoLabel, video_device));
454 main_wnd_->StartLocalRenderer(video_track_);
455
456 result_or_error = peer_connection_->AddTrack(video_track_, {kStreamId});
457 if (!result_or_error.ok()) {
458 RTC_LOG(LS_ERROR) << "Failed to add video track to PeerConnection: "
459 << result_or_error.error().message();
460 }
461 } else {
462 RTC_LOG(LS_ERROR) << "OpenVideoCaptureDevice failed";
463 }
464
465 main_wnd_->SwitchToStreamingUI();
466 }
467
DisconnectFromCurrentPeer()468 void Conductor::DisconnectFromCurrentPeer() {
469 RTC_LOG(INFO) << __FUNCTION__;
470 if (peer_connection_.get()) {
471 client_->SendHangUp(peer_id_);
472 DeletePeerConnection();
473 }
474
475 if (main_wnd_->IsWindow())
476 main_wnd_->SwitchToPeerList(client_->peers());
477 }
478
UIThreadCallback(int msg_id,void * data)479 void Conductor::UIThreadCallback(int msg_id, void* data) {
480 switch (msg_id) {
481 case PEER_CONNECTION_CLOSED:
482 RTC_LOG(INFO) << "PEER_CONNECTION_CLOSED";
483 DeletePeerConnection();
484
485 if (main_wnd_->IsWindow()) {
486 if (client_->is_connected()) {
487 main_wnd_->SwitchToPeerList(client_->peers());
488 } else {
489 main_wnd_->SwitchToConnectUI();
490 }
491 } else {
492 DisconnectFromServer();
493 }
494 break;
495
496 case SEND_MESSAGE_TO_PEER: {
497 RTC_LOG(INFO) << "SEND_MESSAGE_TO_PEER";
498 std::string* msg = reinterpret_cast<std::string*>(data);
499 if (msg) {
500 // For convenience, we always run the message through the queue.
501 // This way we can be sure that messages are sent to the server
502 // in the same order they were signaled without much hassle.
503 pending_messages_.push_back(msg);
504 }
505
506 if (!pending_messages_.empty() && !client_->IsSendingMessage()) {
507 msg = pending_messages_.front();
508 pending_messages_.pop_front();
509
510 if (!client_->SendToPeer(peer_id_, *msg) && peer_id_ != -1) {
511 RTC_LOG(LS_ERROR) << "SendToPeer failed";
512 DisconnectFromServer();
513 }
514 delete msg;
515 }
516
517 if (!peer_connection_.get())
518 peer_id_ = -1;
519
520 break;
521 }
522
523 case NEW_TRACK_ADDED: {
524 auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
525 if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
526 auto* video_track = static_cast<webrtc::VideoTrackInterface*>(track);
527 main_wnd_->StartRemoteRenderer(video_track);
528 }
529 track->Release();
530 break;
531 }
532
533 case TRACK_REMOVED: {
534 // Remote peer stopped sending a track.
535 auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
536 track->Release();
537 break;
538 }
539
540 default:
541 RTC_NOTREACHED();
542 break;
543 }
544 }
545
OnSuccess(webrtc::SessionDescriptionInterface * desc)546 void Conductor::OnSuccess(webrtc::SessionDescriptionInterface* desc) {
547 peer_connection_->SetLocalDescription(
548 DummySetSessionDescriptionObserver::Create(), desc);
549
550 std::string sdp;
551 desc->ToString(&sdp);
552
553 // For loopback test. To save some connecting delay.
554 if (loopback_) {
555 // Replace message type from "offer" to "answer"
556 std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
557 webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp);
558 peer_connection_->SetRemoteDescription(
559 DummySetSessionDescriptionObserver::Create(),
560 session_description.release());
561 return;
562 }
563
564 Json::StyledWriter writer;
565 Json::Value jmessage;
566 jmessage[kSessionDescriptionTypeName] =
567 webrtc::SdpTypeToString(desc->GetType());
568 jmessage[kSessionDescriptionSdpName] = sdp;
569 SendMessage(writer.write(jmessage));
570 }
571
OnFailure(webrtc::RTCError error)572 void Conductor::OnFailure(webrtc::RTCError error) {
573 RTC_LOG(LERROR) << ToString(error.type()) << ": " << error.message();
574 }
575
SendMessage(const std::string & json_object)576 void Conductor::SendMessage(const std::string& json_object) {
577 std::string* msg = new std::string(json_object);
578 main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, msg);
579 }
580