1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <sstream>
12
13 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
14 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
15 #include "webrtc/test/testsupport/fileutils.h"
16 #include "webrtc/test/testsupport/perf_test.h"
17
18 using std::string;
19
20 namespace webrtc {
21 namespace testing {
22 namespace bwe {
23 enum Estimator { kAbsSendTime, kTransmissionOffset };
24
EstimatorConfigs(Estimator estimator,int flow_id)25 BweTestConfig::EstimatorConfig EstimatorConfigs(Estimator estimator,
26 int flow_id) {
27 static const RemoteBitrateEstimatorFactory factories[] = {
28 RemoteBitrateEstimatorFactory(),
29 AbsoluteSendTimeRemoteBitrateEstimatorFactory()
30 };
31 switch (estimator) {
32 case kTransmissionOffset:
33 return BweTestConfig::EstimatorConfig("TOF", flow_id, &factories[0],
34 kMimdControl, false, false);
35 case kAbsSendTime:
36 return BweTestConfig::EstimatorConfig("AST", flow_id, &factories[1],
37 kMimdControl, false, false);
38 }
39 assert(false);
40 return BweTestConfig::EstimatorConfig();
41 }
42
43 struct DefaultBweTestConfig {
44 BweTestConfig bwe_test_config;
45 size_t number_of_senders;
46 };
47
MakeBweTestConfig(uint32_t sender_count,Estimator estimator)48 DefaultBweTestConfig MakeBweTestConfig(uint32_t sender_count,
49 Estimator estimator) {
50 DefaultBweTestConfig result;
51 result.bwe_test_config.estimator_configs.push_back(
52 EstimatorConfigs(estimator, 0));
53 result.number_of_senders = sender_count;
54 return result;
55 }
56
57 class DefaultBweTest : public BweTest,
58 public ::testing::TestWithParam<DefaultBweTestConfig> {
59 public:
DefaultBweTest()60 DefaultBweTest() : packet_senders_() {}
~DefaultBweTest()61 virtual ~DefaultBweTest() {}
62
SetUp()63 virtual void SetUp() {
64 const DefaultBweTestConfig& config = GetParam();
65 SetupTestFromConfig(config.bwe_test_config);
66 for (size_t i = 0; i < config.number_of_senders; ++i) {
67 packet_senders_.push_back(new VideoSender(0, this, 30, 300, 0, 0));
68 }
69 }
70
TearDown()71 virtual void TearDown() {
72 while (!packet_senders_.empty()) {
73 delete packet_senders_.front();
74 packet_senders_.pop_front();
75 }
76 }
77
78 protected:
79 std::list<PacketSender*> packet_senders_;
80 };
81
82 INSTANTIATE_TEST_CASE_P(VideoSendersTest, DefaultBweTest,
83 ::testing::Values(MakeBweTestConfig(1, kAbsSendTime),
84 MakeBweTestConfig(3, kAbsSendTime),
85 MakeBweTestConfig(1, kTransmissionOffset),
86 MakeBweTestConfig(3, kTransmissionOffset)));
87
TEST_P(DefaultBweTest,UnlimitedSpeed)88 TEST_P(DefaultBweTest, UnlimitedSpeed) {
89 VerboseLogging(false);
90 RunFor(10 * 60 * 1000);
91 }
92
TEST_P(DefaultBweTest,DISABLED_SteadyLoss)93 TEST_P(DefaultBweTest, DISABLED_SteadyLoss) {
94 LossFilter loss(this);
95 loss.SetLoss(20.0);
96 RunFor(10 * 60 * 1000);
97 }
98
TEST_P(DefaultBweTest,IncreasingLoss1)99 TEST_P(DefaultBweTest, IncreasingLoss1) {
100 LossFilter loss(this);
101 for (int i = 0; i < 76; ++i) {
102 loss.SetLoss(i);
103 RunFor(5000);
104 }
105 }
106
TEST_P(DefaultBweTest,SteadyDelay)107 TEST_P(DefaultBweTest, SteadyDelay) {
108 DelayFilter delay(this);
109 delay.SetDelay(1000);
110 RunFor(10 * 60 * 1000);
111 }
112
TEST_P(DefaultBweTest,DISABLED_IncreasingDelay1)113 TEST_P(DefaultBweTest, DISABLED_IncreasingDelay1) {
114 DelayFilter delay(this);
115 RunFor(10 * 60 * 1000);
116 for (int i = 0; i < 30 * 2; ++i) {
117 delay.SetDelay(i);
118 RunFor(10 * 1000);
119 }
120 RunFor(10 * 60 * 1000);
121 }
122
TEST_P(DefaultBweTest,IncreasingDelay2)123 TEST_P(DefaultBweTest, IncreasingDelay2) {
124 DelayFilter delay(this);
125 RateCounterFilter counter(this);
126 RunFor(1 * 60 * 1000);
127 for (int i = 1; i < 51; ++i) {
128 delay.SetDelay(10.0f * i);
129 RunFor(10 * 1000);
130 }
131 delay.SetDelay(0.0f);
132 RunFor(10 * 60 * 1000);
133 }
134
TEST_P(DefaultBweTest,JumpyDelay1)135 TEST_P(DefaultBweTest, JumpyDelay1) {
136 DelayFilter delay(this);
137 RunFor(10 * 60 * 1000);
138 for (int i = 1; i < 200; ++i) {
139 delay.SetDelay((10 * i) % 500);
140 RunFor(1000);
141 delay.SetDelay(1.0f);
142 RunFor(1000);
143 }
144 delay.SetDelay(0.0f);
145 RunFor(10 * 60 * 1000);
146 }
147
TEST_P(DefaultBweTest,SteadyJitter)148 TEST_P(DefaultBweTest, SteadyJitter) {
149 JitterFilter jitter(this);
150 RateCounterFilter counter(this);
151 jitter.SetJitter(20);
152 RunFor(2 * 60 * 1000);
153 }
154
TEST_P(DefaultBweTest,IncreasingJitter1)155 TEST_P(DefaultBweTest, IncreasingJitter1) {
156 JitterFilter jitter(this);
157 for (int i = 0; i < 2 * 60 * 2; ++i) {
158 jitter.SetJitter(i);
159 RunFor(10 * 1000);
160 }
161 RunFor(10 * 60 * 1000);
162 }
163
TEST_P(DefaultBweTest,IncreasingJitter2)164 TEST_P(DefaultBweTest, IncreasingJitter2) {
165 JitterFilter jitter(this);
166 RunFor(30 * 1000);
167 for (int i = 1; i < 51; ++i) {
168 jitter.SetJitter(10.0f * i);
169 RunFor(10 * 1000);
170 }
171 jitter.SetJitter(0.0f);
172 RunFor(10 * 60 * 1000);
173 }
174
TEST_P(DefaultBweTest,SteadyReorder)175 TEST_P(DefaultBweTest, SteadyReorder) {
176 ReorderFilter reorder(this);
177 reorder.SetReorder(20.0);
178 RunFor(10 * 60 * 1000);
179 }
180
TEST_P(DefaultBweTest,IncreasingReorder1)181 TEST_P(DefaultBweTest, IncreasingReorder1) {
182 ReorderFilter reorder(this);
183 for (int i = 0; i < 76; ++i) {
184 reorder.SetReorder(i);
185 RunFor(5000);
186 }
187 }
188
TEST_P(DefaultBweTest,DISABLED_SteadyChoke)189 TEST_P(DefaultBweTest, DISABLED_SteadyChoke) {
190 ChokeFilter choke(this);
191 choke.SetCapacity(140);
192 RunFor(10 * 60 * 1000);
193 }
194
TEST_P(DefaultBweTest,DISABLED_IncreasingChoke1)195 TEST_P(DefaultBweTest, DISABLED_IncreasingChoke1) {
196 ChokeFilter choke(this);
197 for (int i = 1200; i >= 100; i -= 100) {
198 choke.SetCapacity(i);
199 RunFor(5000);
200 }
201 }
202
TEST_P(DefaultBweTest,DISABLED_IncreasingChoke2)203 TEST_P(DefaultBweTest, DISABLED_IncreasingChoke2) {
204 ChokeFilter choke(this);
205 RunFor(60 * 1000);
206 for (int i = 1200; i >= 100; i -= 20) {
207 choke.SetCapacity(i);
208 RunFor(1000);
209 }
210 }
211
TEST_P(DefaultBweTest,DISABLED_Multi1)212 TEST_P(DefaultBweTest, DISABLED_Multi1) {
213 DelayFilter delay(this);
214 ChokeFilter choke(this);
215 RateCounterFilter counter(this);
216 choke.SetCapacity(1000);
217 RunFor(1 * 60 * 1000);
218 for (int i = 1; i < 51; ++i) {
219 delay.SetDelay(100.0f * i);
220 RunFor(10 * 1000);
221 }
222 RunFor(500 * 1000);
223 delay.SetDelay(0.0f);
224 RunFor(5 * 60 * 1000);
225 }
226
TEST_P(DefaultBweTest,Multi2)227 TEST_P(DefaultBweTest, Multi2) {
228 ChokeFilter choke(this);
229 JitterFilter jitter(this);
230 RateCounterFilter counter(this);
231 choke.SetCapacity(2000);
232 jitter.SetJitter(120);
233 RunFor(5 * 60 * 1000);
234 }
235
236 // This test fixture is used to instantiate tests running with adaptive video
237 // senders.
238 class BweFeedbackTest : public BweTest,
239 public ::testing::TestWithParam<BweTestConfig> {
240 public:
BweFeedbackTest()241 BweFeedbackTest() : BweTest() {}
~BweFeedbackTest()242 virtual ~BweFeedbackTest() {}
243
SetUp()244 virtual void SetUp() {
245 BweTestConfig config;
246 config.estimator_configs.push_back(EstimatorConfigs(kAbsSendTime, 0));
247 SetupTestFromConfig(config);
248 }
249
PrintResults(double max_throughput_kbps,Stats<double> throughput_kbps,Stats<double> delay_ms)250 void PrintResults(double max_throughput_kbps, Stats<double> throughput_kbps,
251 Stats<double> delay_ms) {
252 double utilization = throughput_kbps.GetMean() / max_throughput_kbps;
253 webrtc::test::PrintResult("BwePerformance",
254 GetTestName(),
255 "Utilization",
256 utilization * 100.0,
257 "%",
258 false);
259 std::stringstream ss;
260 ss << throughput_kbps.GetStdDev() / throughput_kbps.GetMean();
261 webrtc::test::PrintResult("BwePerformance",
262 GetTestName(),
263 "Utilization var coeff",
264 ss.str(),
265 "",
266 false);
267 webrtc::test::PrintResult("BwePerformance",
268 GetTestName(),
269 "Average delay",
270 delay_ms.AsString(),
271 "ms",
272 false);
273 }
274
275 private:
276 DISALLOW_COPY_AND_ASSIGN(BweFeedbackTest);
277 };
278
TEST_F(BweFeedbackTest,Choke1000kbps500kbps1000kbps)279 TEST_F(BweFeedbackTest, Choke1000kbps500kbps1000kbps) {
280 AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
281 ChokeFilter filter(this);
282 RateCounterFilter counter(this, "receiver_input");
283 const int kHighCapacityKbps = 1000;
284 const int kLowCapacityKbps = 500;
285 filter.SetCapacity(kHighCapacityKbps);
286 filter.SetMaxDelay(500);
287 RunFor(60 * 1000);
288 filter.SetCapacity(kLowCapacityKbps);
289 RunFor(60 * 1000);
290 filter.SetCapacity(kHighCapacityKbps);
291 RunFor(60 * 1000);
292 PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
293 counter.GetBitrateStats(), filter.GetDelayStats());
294 }
295
TEST_F(BweFeedbackTest,Choke200kbps30kbps200kbps)296 TEST_F(BweFeedbackTest, Choke200kbps30kbps200kbps) {
297 AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
298 ChokeFilter filter(this);
299 RateCounterFilter counter(this, "receiver_input");
300 const int kHighCapacityKbps = 200;
301 const int kLowCapacityKbps = 30;
302 filter.SetCapacity(kHighCapacityKbps);
303 filter.SetMaxDelay(500);
304 RunFor(60 * 1000);
305 filter.SetCapacity(kLowCapacityKbps);
306 RunFor(60 * 1000);
307 filter.SetCapacity(kHighCapacityKbps);
308 RunFor(60 * 1000);
309
310 PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
311 counter.GetBitrateStats(), filter.GetDelayStats());
312 }
313
TEST_F(BweFeedbackTest,Verizon4gDownlinkTest)314 TEST_F(BweFeedbackTest, Verizon4gDownlinkTest) {
315 AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
316 RateCounterFilter counter1(this, "sender_output");
317 TraceBasedDeliveryFilter filter(this, "link_capacity");
318 RateCounterFilter counter2(this, "receiver_input");
319 ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
320 RunFor(22 * 60 * 1000);
321 PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
322 filter.GetDelayStats());
323 }
324
325 // webrtc:3277
TEST_F(BweFeedbackTest,DISABLED_GoogleWifiTrace3Mbps)326 TEST_F(BweFeedbackTest, DISABLED_GoogleWifiTrace3Mbps) {
327 AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
328 RateCounterFilter counter1(this, "sender_output");
329 TraceBasedDeliveryFilter filter(this, "link_capacity");
330 filter.SetMaxDelay(500);
331 RateCounterFilter counter2(this, "receiver_input");
332 ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
333 RunFor(300 * 1000);
334 PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
335 filter.GetDelayStats());
336 }
337 } // namespace bwe
338 } // namespace testing
339 } // namespace webrtc
340