1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <cstdint>
12 #include <memory>
13
14 #include "api/media_stream_interface.h"
15 #include "api/test/create_network_emulation_manager.h"
16 #include "api/test/create_peer_connection_quality_test_frame_generator.h"
17 #include "api/test/create_peerconnection_quality_test_fixture.h"
18 #include "api/test/network_emulation_manager.h"
19 #include "api/test/peerconnection_quality_test_fixture.h"
20 #include "call/simulated_network.h"
21 #include "system_wrappers/include/field_trial.h"
22 #include "test/field_trial.h"
23 #include "test/gtest.h"
24 #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
25 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
26 #include "test/pc/e2e/network_quality_metrics_reporter.h"
27 #include "test/testsupport/file_utils.h"
28
29 namespace webrtc {
30 namespace webrtc_pc_e2e {
31 namespace {
32
33 class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
34 public:
35 using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer;
36 using RunParams = PeerConnectionE2EQualityTestFixture::RunParams;
37 using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
38 using VideoCodecConfig =
39 PeerConnectionE2EQualityTestFixture::VideoCodecConfig;
40 using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig;
41 using ScreenShareConfig =
42 PeerConnectionE2EQualityTestFixture::ScreenShareConfig;
43 using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams;
44 using VideoSimulcastConfig =
45 PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig;
46 using EchoEmulationConfig =
47 PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
48
RunTest(const std::string & test_case_name,const RunParams & run_params,rtc::FunctionView<void (PeerConfigurer *)> alice_configurer,rtc::FunctionView<void (PeerConfigurer *)> bob_configurer)49 void RunTest(const std::string& test_case_name,
50 const RunParams& run_params,
51 rtc::FunctionView<void(PeerConfigurer*)> alice_configurer,
52 rtc::FunctionView<void(PeerConfigurer*)> bob_configurer) {
53 // Setup emulated network
54 std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
55 CreateNetworkEmulationManager();
56
57 auto alice_network_behavior =
58 std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig());
59 SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get();
60 EmulatedNetworkNode* alice_node =
61 network_emulation_manager->CreateEmulatedNode(
62 std::move(alice_network_behavior));
63 EmulatedNetworkNode* bob_node =
64 network_emulation_manager->CreateEmulatedNode(
65 std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
66 auto* alice_endpoint =
67 network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
68 EmulatedEndpoint* bob_endpoint =
69 network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
70 network_emulation_manager->CreateRoute(alice_endpoint, {alice_node},
71 bob_endpoint);
72 network_emulation_manager->CreateRoute(bob_endpoint, {bob_node},
73 alice_endpoint);
74
75 // Create analyzers.
76 std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer =
77 std::make_unique<DefaultVideoQualityAnalyzer>(
78 network_emulation_manager->time_controller()->GetClock());
79 // This is only done for the sake of smoke testing. In general there should
80 // be no need to explicitly pull data from analyzers after the run.
81 auto* video_analyzer_ptr =
82 static_cast<DefaultVideoQualityAnalyzer*>(video_quality_analyzer.get());
83
84 auto fixture = CreatePeerConnectionE2EQualityTestFixture(
85 test_case_name, *network_emulation_manager->time_controller(),
86 /*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer));
87 fixture->ExecuteAt(TimeDelta::Seconds(1),
88 [alice_network_behavior_ptr](TimeDelta) {
89 BuiltInNetworkBehaviorConfig config;
90 config.loss_percent = 5;
91 alice_network_behavior_ptr->SetConfig(config);
92 });
93
94 // Setup components. We need to provide rtc::NetworkManager compatible with
95 // emulated network layer.
96 EmulatedNetworkManagerInterface* alice_network =
97 network_emulation_manager->CreateEmulatedNetworkManagerInterface(
98 {alice_endpoint});
99 EmulatedNetworkManagerInterface* bob_network =
100 network_emulation_manager->CreateEmulatedNetworkManagerInterface(
101 {bob_endpoint});
102
103 fixture->AddPeer(alice_network->network_thread(),
104 alice_network->network_manager(), alice_configurer);
105 fixture->AddPeer(bob_network->network_thread(),
106 bob_network->network_manager(), bob_configurer);
107 fixture->AddQualityMetricsReporter(
108 std::make_unique<NetworkQualityMetricsReporter>(alice_network,
109 bob_network));
110
111 fixture->Run(run_params);
112
113 EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration);
114 for (auto stream_key : video_analyzer_ptr->GetKnownVideoStreams()) {
115 FrameCounters stream_conters =
116 video_analyzer_ptr->GetPerStreamCounters().at(stream_key);
117 // On some devices the pipeline can be too slow, so we actually can't
118 // force real constraints here. Lets just check, that at least 1
119 // frame passed whole pipeline.
120 int64_t expected_min_fps = run_params.run_duration.seconds() * 15;
121 EXPECT_GE(stream_conters.captured, expected_min_fps)
122 << stream_key.ToString();
123 EXPECT_GE(stream_conters.pre_encoded, 1) << stream_key.ToString();
124 EXPECT_GE(stream_conters.encoded, 1) << stream_key.ToString();
125 EXPECT_GE(stream_conters.received, 1) << stream_key.ToString();
126 EXPECT_GE(stream_conters.decoded, 1) << stream_key.ToString();
127 EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString();
128 }
129 }
130 };
131
132 } // namespace
133
134 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
135 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
136 #define MAYBE_Smoke DISABLED_Smoke
137 #else
138 #define MAYBE_Smoke Smoke
139 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Smoke)140 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
141 RunParams run_params(TimeDelta::Seconds(2));
142 run_params.video_codecs = {
143 VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
144 run_params.use_flex_fec = true;
145 run_params.use_ulp_fec = true;
146 run_params.video_encoder_bitrate_multiplier = 1.1;
147 test::ScopedFieldTrials field_trials(
148 std::string(field_trial::GetFieldTrialString()) +
149 "WebRTC-UseStandardBytesStats/Enabled/");
150 RunTest(
151 "smoke", run_params,
152 [](PeerConfigurer* alice) {
153 VideoConfig video(160, 120, 15);
154 video.stream_label = "alice-video";
155 video.sync_group = "alice-media";
156 alice->AddVideoConfig(std::move(video));
157
158 AudioConfig audio;
159 audio.stream_label = "alice-audio";
160 audio.mode = AudioConfig::Mode::kFile;
161 audio.input_file_name =
162 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
163 audio.sampling_frequency_in_hz = 48000;
164 audio.sync_group = "alice-media";
165 alice->SetAudioConfig(std::move(audio));
166 },
167 [](PeerConfigurer* charlie) {
168 charlie->SetName("charlie");
169 VideoConfig video(160, 120, 15);
170 video.stream_label = "charlie-video";
171 video.temporal_layers_count = 2;
172 charlie->AddVideoConfig(std::move(video));
173
174 AudioConfig audio;
175 audio.stream_label = "charlie-audio";
176 audio.mode = AudioConfig::Mode::kFile;
177 audio.input_file_name =
178 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
179 charlie->SetAudioConfig(std::move(audio));
180 });
181 }
182
183 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
184 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
185 #define MAYBE_Screenshare DISABLED_Screenshare
186 #else
187 #define MAYBE_Screenshare Screenshare
188 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Screenshare)189 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) {
190 RunParams run_params(TimeDelta::Seconds(2));
191 test::ScopedFieldTrials field_trials(
192 std::string(field_trial::GetFieldTrialString()) +
193 "WebRTC-UseStandardBytesStats/Enabled/");
194 RunTest(
195 "screenshare", run_params,
196 [](PeerConfigurer* alice) {
197 VideoConfig screenshare(320, 180, 30);
198 screenshare.stream_label = "alice-screenshare";
199 screenshare.content_hint = VideoTrackInterface::ContentHint::kText;
200 ScreenShareConfig screen_share_config =
201 ScreenShareConfig(TimeDelta::Seconds(2));
202 screen_share_config.scrolling_params = ScrollingParams(
203 TimeDelta::Millis(1800), kDefaultSlidesWidth, kDefaultSlidesHeight);
204 auto screen_share_frame_generator =
205 CreateScreenShareFrameGenerator(screenshare, screen_share_config);
206 alice->AddVideoConfig(std::move(screenshare),
207 std::move(screen_share_frame_generator));
208 },
209 [](PeerConfigurer* charlie) {});
210 }
211
212 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
213 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
214 #define MAYBE_Echo DISABLED_Echo
215 #else
216 #define MAYBE_Echo Echo
217 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Echo)218 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) {
219 RunParams run_params(TimeDelta::Seconds(2));
220 run_params.echo_emulation_config = EchoEmulationConfig();
221 RunTest(
222 "smoke", run_params,
223 [](PeerConfigurer* alice) {
224 AudioConfig audio;
225 audio.stream_label = "alice-audio";
226 audio.mode = AudioConfig::Mode::kFile;
227 audio.input_file_name =
228 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
229 audio.sampling_frequency_in_hz = 48000;
230 alice->SetAudioConfig(std::move(audio));
231 },
232 [](PeerConfigurer* bob) {
233 AudioConfig audio;
234 audio.stream_label = "bob-audio";
235 audio.mode = AudioConfig::Mode::kFile;
236 audio.input_file_name =
237 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
238 bob->SetAudioConfig(std::move(audio));
239 });
240 }
241
242 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
243 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
244 #define MAYBE_Simulcast DISABLED_Simulcast
245 #else
246 #define MAYBE_Simulcast Simulcast
247 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Simulcast)248 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) {
249 RunParams run_params(TimeDelta::Seconds(2));
250 run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
251 RunTest(
252 "simulcast", run_params,
253 [](PeerConfigurer* alice) {
254 VideoConfig simulcast(1280, 720, 15);
255 simulcast.stream_label = "alice-simulcast";
256 simulcast.simulcast_config = VideoSimulcastConfig(2, 0);
257 alice->AddVideoConfig(std::move(simulcast));
258
259 AudioConfig audio;
260 audio.stream_label = "alice-audio";
261 audio.mode = AudioConfig::Mode::kFile;
262 audio.input_file_name =
263 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
264 alice->SetAudioConfig(std::move(audio));
265 },
266 [](PeerConfigurer* bob) {});
267 }
268
269 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
270 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
271 #define MAYBE_Svc DISABLED_Svc
272 #else
273 #define MAYBE_Svc Svc
274 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Svc)275 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) {
276 RunParams run_params(TimeDelta::Seconds(2));
277 run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)};
278 RunTest(
279 "simulcast", run_params,
280 [](PeerConfigurer* alice) {
281 VideoConfig simulcast(1280, 720, 15);
282 simulcast.stream_label = "alice-svc";
283 // Because we have network with packets loss we can analyze only the
284 // highest spatial layer in SVC mode.
285 simulcast.simulcast_config = VideoSimulcastConfig(2, 1);
286 alice->AddVideoConfig(std::move(simulcast));
287
288 AudioConfig audio;
289 audio.stream_label = "alice-audio";
290 audio.mode = AudioConfig::Mode::kFile;
291 audio.input_file_name =
292 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
293 alice->SetAudioConfig(std::move(audio));
294 },
295 [](PeerConfigurer* bob) {});
296 }
297
298 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
299 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
300 #define MAYBE_HighBitrate DISABLED_HighBitrate
301 #else
302 #define MAYBE_HighBitrate HighBitrate
303 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_HighBitrate)304 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) {
305 RunParams run_params(TimeDelta::Seconds(2));
306 run_params.video_codecs = {
307 VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
308
309 RunTest(
310 "smoke", run_params,
311 [](PeerConfigurer* alice) {
312 BitrateSettings bitrate_settings;
313 bitrate_settings.start_bitrate_bps = 3'000'000;
314 bitrate_settings.max_bitrate_bps = 3'000'000;
315 alice->SetBitrateSettings(bitrate_settings);
316 VideoConfig video(800, 600, 15);
317 video.stream_label = "alice-video";
318 video.min_encode_bitrate_bps = 500'000;
319 video.max_encode_bitrate_bps = 3'000'000;
320 alice->AddVideoConfig(std::move(video));
321
322 AudioConfig audio;
323 audio.stream_label = "alice-audio";
324 audio.mode = AudioConfig::Mode::kFile;
325 audio.input_file_name =
326 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
327 audio.sampling_frequency_in_hz = 48000;
328 alice->SetAudioConfig(std::move(audio));
329 },
330 [](PeerConfigurer* bob) {});
331 }
332
333 } // namespace webrtc_pc_e2e
334 } // namespace webrtc
335