1#!/usr/bin/env python3.4 2# 3# Copyright 2017 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import collections 18import json 19import logging 20import os 21import statistics 22import time 23from acts import asserts 24from acts import base_test 25from acts import utils 26from acts.controllers.utils_lib import ssh 27from acts.metrics.loggers.blackbox import BlackboxMetricLogger 28from acts.test_utils.wifi import wifi_performance_test_utils as wputils 29from acts.test_utils.wifi import wifi_retail_ap as retail_ap 30from acts.test_utils.wifi import wifi_test_utils as wutils 31 32 33class WifiPingTest(base_test.BaseTestClass): 34 """Class for ping-based Wifi performance tests. 35 36 This class implements WiFi ping performance tests such as range and RTT. 37 The class setups up the AP in the desired configurations, configures 38 and connects the phone to the AP, and runs For an example config file to 39 run this test class see example_connectivity_performance_ap_sta.json. 40 """ 41 42 TEST_TIMEOUT = 10 43 RSSI_POLL_INTERVAL = 0.2 44 SHORT_SLEEP = 1 45 MED_SLEEP = 5 46 MAX_CONSECUTIVE_ZEROS = 5 47 DISCONNECTED_PING_RESULT = { 48 "connected": 0, 49 "rtt": [], 50 "time_stamp": [], 51 "ping_interarrivals": [], 52 "packet_loss_percentage": 100 53 } 54 55 def __init__(self, controllers): 56 base_test.BaseTestClass.__init__(self, controllers) 57 self.ping_range_metric = BlackboxMetricLogger.for_test_case( 58 metric_name='ping_range') 59 self.ping_rtt_metric = BlackboxMetricLogger.for_test_case( 60 metric_name='ping_rtt') 61 self.tests = ( 62 "test_ping_range_ch1_VHT20", "test_fast_ping_rtt_ch1_VHT20", 63 "test_slow_ping_rtt_ch1_VHT20", "test_ping_range_ch6_VHT20", 64 "test_fast_ping_rtt_ch6_VHT20", "test_slow_ping_rtt_ch6_VHT20", 65 "test_ping_range_ch11_VHT20", "test_fast_ping_rtt_ch11_VHT20", 66 "test_slow_ping_rtt_ch11_VHT20", "test_ping_range_ch36_VHT20", 67 "test_fast_ping_rtt_ch36_VHT20", "test_slow_ping_rtt_ch36_VHT20", 68 "test_ping_range_ch36_VHT40", "test_fast_ping_rtt_ch36_VHT40", 69 "test_slow_ping_rtt_ch36_VHT40", "test_ping_range_ch36_VHT80", 70 "test_fast_ping_rtt_ch36_VHT80", "test_slow_ping_rtt_ch36_VHT80", 71 "test_ping_range_ch40_VHT20", "test_ping_range_ch44_VHT20", 72 "test_ping_range_ch44_VHT40", "test_ping_range_ch48_VHT20", 73 "test_ping_range_ch149_VHT20", "test_fast_ping_rtt_ch149_VHT20", 74 "test_slow_ping_rtt_ch149_VHT20", "test_ping_range_ch149_VHT40", 75 "test_fast_ping_rtt_ch149_VHT40", "test_slow_ping_rtt_ch149_VHT40", 76 "test_ping_range_ch149_VHT80", "test_fast_ping_rtt_ch149_VHT80", 77 "test_slow_ping_rtt_ch149_VHT80", "test_ping_range_ch153_VHT20", 78 "test_ping_range_ch157_VHT20", "test_ping_range_ch157_VHT40", 79 "test_ping_range_ch161_VHT20") 80 81 def setup_class(self): 82 self.client_dut = self.android_devices[-1] 83 req_params = [ 84 "ping_test_params", "testbed_params", "main_network", 85 "RetailAccessPoints", "RemoteServer" 86 ] 87 opt_params = ["golden_files_list"] 88 self.unpack_userparams(req_params, opt_params) 89 self.testclass_params = self.ping_test_params 90 self.num_atten = self.attenuators[0].instrument.num_atten 91 self.ping_server = ssh.connection.SshConnection( 92 ssh.settings.from_config(self.RemoteServer[0]["ssh_config"])) 93 self.access_points = retail_ap.create(self.RetailAccessPoints) 94 self.access_point = self.access_points[0] 95 self.log.info("Access Point Configuration: {}".format( 96 self.access_point.ap_settings)) 97 self.log_path = os.path.join(logging.log_path, "results") 98 utils.create_dir(self.log_path) 99 if not hasattr(self, "golden_files_list"): 100 self.golden_files_list = [ 101 os.path.join(self.testbed_params["golden_results_path"], file) 102 for file in os.listdir( 103 self.testbed_params["golden_results_path"]) 104 ] 105 self.testclass_results = [] 106 107 # Turn WiFi ON 108 for dev in self.android_devices: 109 wutils.wifi_toggle_state(dev, True) 110 111 def teardown_class(self): 112 # Turn WiFi OFF 113 for dev in self.android_devices: 114 wutils.wifi_toggle_state(dev, False) 115 self.process_testclass_results() 116 117 def process_testclass_results(self): 118 """Saves all test results to enable comparison.""" 119 testclass_summary = {} 120 for test in self.testclass_results: 121 if "range" in test["test_name"]: 122 testclass_summary[test["test_name"]] = test["range"] 123 # Save results 124 results_file_path = "{}/testclass_summary.json".format(self.log_path) 125 with open(results_file_path, 'w') as results_file: 126 json.dump(testclass_summary, results_file, indent=4) 127 128 def pass_fail_check_ping_rtt(self, ping_range_result): 129 """Check the test result and decide if it passed or failed. 130 131 The function computes RTT statistics and fails any tests in which the 132 tail of the ping latency results exceeds the threshold defined in the 133 configuration file. 134 135 Args: 136 ping_range_result: dict containing ping results and other meta data 137 """ 138 ignored_fraction = (self.testclass_params["rtt_ignored_interval"] / 139 self.testclass_params["rtt_ping_duration"]) 140 sorted_rtt = [ 141 sorted(x["rtt"][round(ignored_fraction * len(x["rtt"])):]) 142 for x in ping_range_result["ping_results"] 143 ] 144 mean_rtt = [statistics.mean(x) for x in sorted_rtt] 145 std_rtt = [statistics.stdev(x) for x in sorted_rtt] 146 rtt_at_test_percentile = [ 147 x[int((1 - self.testclass_params["rtt_test_percentile"] / 100) * 148 len(x))] for x in sorted_rtt 149 ] 150 # Set blackbox metric 151 self.ping_rtt_metric.metric_value = max(rtt_at_test_percentile) 152 # Evaluate test pass/fail 153 test_failed = False 154 for idx, rtt in enumerate(rtt_at_test_percentile): 155 if rtt > self.testclass_params["rtt_threshold"] * 1000: 156 test_failed = True 157 self.log.info( 158 "RTT Failed. Test %ile RTT = {}ms. Mean = {}ms. Stdev = {}" 159 .format(rtt, mean_rtt[idx], std_rtt[idx])) 160 if test_failed: 161 asserts.fail("RTT above threshold") 162 else: 163 asserts.explicit_pass( 164 "Test Passed. RTTs at test percentile = {}".format( 165 rtt_at_test_percentile)) 166 167 def pass_fail_check_ping_range(self, ping_range_result): 168 """Check the test result and decide if it passed or failed. 169 170 Checks whether the attenuation at which ping packet losses begin to 171 exceed the threshold matches the range derived from golden 172 rate-vs-range result files. The test fails is ping range is 173 range_gap_threshold worse than RvR range. 174 175 Args: 176 ping_range_result: dict containing ping results and meta data 177 """ 178 # Get target range 179 rvr_range = self.get_range_from_rvr() 180 # Set Blackbox metric 181 self.ping_range_metric.metric_value = ping_range_result["range"] 182 # Evaluate test pass/fail 183 if ping_range_result["range"] - rvr_range < -self.testclass_params[ 184 "range_gap_threshold"]: 185 asserts.fail( 186 "Attenuation at range is {}dB. Golden range is {}dB".format( 187 ping_range_result["range"], rvr_range)) 188 else: 189 asserts.explicit_pass( 190 "Attenuation at range is {}dB. Golden range is {}dB".format( 191 ping_range_result["range"], rvr_range)) 192 193 def process_ping_results(self, testcase_params, ping_range_result): 194 """Saves and plots ping results. 195 196 Args: 197 ping_range_result: dict containing ping results and metadata 198 """ 199 # Compute range 200 ping_loss_over_att = [ 201 x["packet_loss_percentage"] 202 for x in ping_range_result["ping_results"] 203 ] 204 ping_loss_above_threshold = [ 205 x > testcase_params["range_ping_loss_threshold"] 206 for x in ping_loss_over_att 207 ] 208 for idx in range(len(ping_loss_above_threshold)): 209 if all(ping_loss_above_threshold[idx:]): 210 range_index = max(idx, 1) - 1 211 break 212 else: 213 range_index = -1 214 ping_range_result["atten_at_range"] = testcase_params["atten_range"][ 215 range_index] 216 ping_range_result["peak_throughput"] = "{}%".format( 217 100 - min(ping_loss_over_att)) 218 ping_range_result["range"] = (ping_range_result["atten_at_range"] + 219 ping_range_result["fixed_attenuation"]) 220 221 # Save results 222 results_file_path = "{}/{}.json".format(self.log_path, 223 self.current_test_name) 224 with open(results_file_path, 'w') as results_file: 225 json.dump(ping_range_result, results_file, indent=4) 226 227 # Plot results 228 x_data = [ 229 list(range(len(x["rtt"]))) 230 for x in ping_range_result["ping_results"] if len(x["rtt"]) > 1 231 ] 232 rtt_data = [ 233 x["rtt"] for x in ping_range_result["ping_results"] 234 if len(x["rtt"]) > 1 235 ] 236 legend = [ 237 "RTT @ {}dB".format(att) 238 for att in ping_range_result["attenuation"] 239 ] 240 241 data_sets = [x_data, rtt_data] 242 fig_property = { 243 "title": self.current_test_name, 244 "x_label": 'Sample Index', 245 "y_label": 'Round Trip Time (ms)', 246 "linewidth": 3, 247 "markersize": 0 248 } 249 output_file_path = "{}/{}.html".format(self.log_path, 250 self.current_test_name) 251 wputils.bokeh_plot( 252 data_sets, 253 legend, 254 fig_property, 255 shaded_region=None, 256 output_file_path=output_file_path) 257 258 def get_range_from_rvr(self): 259 """Function gets range from RvR golden results 260 261 The function fetches the attenuation at which the RvR throughput goes 262 to zero. 263 264 Returns: 265 range: range derived from looking at rvr curves 266 """ 267 # Fetch the golden RvR results 268 test_name = self.current_test_name 269 rvr_golden_file_name = "test_rvr_TCP_DL_" + "_".join( 270 test_name.split("_")[3:]) 271 golden_path = [ 272 file_name for file_name in self.golden_files_list 273 if rvr_golden_file_name in file_name 274 ] 275 with open(golden_path[0], 'r') as golden_file: 276 golden_results = json.load(golden_file) 277 # Get 0 Mbps attenuation and backoff by low_rssi_backoff_from_range 278 try: 279 atten_idx = golden_results["throughput_receive"].index(0) 280 rvr_range = (golden_results["attenuation"][atten_idx - 1] + 281 golden_results["fixed_attenuation"]) 282 except ValueError: 283 rvr_range = float("nan") 284 return rvr_range 285 286 def run_ping_test(self, testcase_params): 287 """Main function to test ping. 288 289 The function sets up the AP in the correct channel and mode 290 configuration and calls get_ping_stats while sweeping attenuation 291 292 Args: 293 testcase_params: dict containing all test parameters 294 Returns: 295 test_result: dict containing ping results and other meta data 296 """ 297 # Prepare results dict 298 test_result = collections.OrderedDict() 299 test_result["test_name"] = self.current_test_name 300 test_result["ap_config"] = self.access_point.ap_settings.copy() 301 test_result["attenuation"] = testcase_params["atten_range"] 302 test_result["fixed_attenuation"] = self.testbed_params[ 303 "fixed_attenuation"][str(testcase_params["channel"])] 304 test_result["rssi_results"] = [] 305 test_result["ping_results"] = [] 306 # Run ping and sweep attenuation as needed 307 zero_counter = 0 308 for atten in testcase_params["atten_range"]: 309 for attenuator in self.attenuators: 310 attenuator.set_atten(atten, strict=False) 311 rssi_future = wputils.get_connected_rssi_nb( 312 self.client_dut, 313 int(testcase_params["ping_duration"] / 2 / 314 self.RSSI_POLL_INTERVAL), self.RSSI_POLL_INTERVAL, 315 testcase_params["ping_duration"] / 2) 316 current_ping_stats = wputils.get_ping_stats( 317 self.ping_server, self.dut_ip, 318 testcase_params["ping_duration"], 319 testcase_params["ping_interval"], testcase_params["ping_size"]) 320 current_rssi = rssi_future.result()["signal_poll_rssi"]["mean"] 321 test_result["rssi_results"].append(current_rssi) 322 if current_ping_stats["connected"]: 323 self.log.info("Attenuation = {0}dB\tPacket Loss = {1}%\t" 324 "Avg RTT = {2:.2f}ms\tRSSI = {3}\t".format( 325 atten, 326 current_ping_stats["packet_loss_percentage"], 327 statistics.mean(current_ping_stats["rtt"]), 328 current_rssi)) 329 if current_ping_stats["packet_loss_percentage"] == 100: 330 zero_counter = zero_counter + 1 331 else: 332 zero_counter = 0 333 else: 334 self.log.info( 335 "Attenuation = {}dB. Disconnected.".format(atten)) 336 zero_counter = zero_counter + 1 337 test_result["ping_results"].append(current_ping_stats.as_dict()) 338 if zero_counter == self.MAX_CONSECUTIVE_ZEROS: 339 self.log.info("Ping loss stable at 100%. Stopping test now.") 340 for idx in range( 341 len(testcase_params["atten_range"]) - 342 len(test_result["ping_results"])): 343 test_result["ping_results"].append( 344 self.DISCONNECTED_PING_RESULT) 345 break 346 return test_result 347 348 def setup_ap(self, testcase_params): 349 """Sets up the access point in the configuration required by the test. 350 351 Args: 352 testcase_params: dict containing AP and other test params 353 """ 354 band = self.access_point.band_lookup_by_channel( 355 testcase_params["channel"]) 356 if "2G" in band: 357 frequency = wutils.WifiEnums.channel_2G_to_freq[ 358 testcase_params["channel"]] 359 else: 360 frequency = wutils.WifiEnums.channel_5G_to_freq[ 361 testcase_params["channel"]] 362 if frequency in wutils.WifiEnums.DFS_5G_FREQUENCIES: 363 self.access_point.set_region(self.testbed_params["DFS_region"]) 364 else: 365 self.access_point.set_region(self.testbed_params["default_region"]) 366 self.access_point.set_channel(band, testcase_params["channel"]) 367 self.access_point.set_bandwidth(band, testcase_params["mode"]) 368 self.log.info("Access Point Configuration: {}".format( 369 self.access_point.ap_settings)) 370 371 def setup_dut(self, testcase_params): 372 """Sets up the DUT in the configuration required by the test. 373 374 Args: 375 testcase_params: dict containing AP and other test params 376 """ 377 band = self.access_point.band_lookup_by_channel( 378 testcase_params["channel"]) 379 wutils.reset_wifi(self.client_dut) 380 self.client_dut.droid.wifiSetCountryCode( 381 self.testclass_params["country_code"]) 382 self.main_network[band]["channel"] = testcase_params["channel"] 383 wutils.wifi_connect( 384 self.client_dut, 385 self.main_network[band], 386 num_of_tries=5, 387 check_connectivity=False) 388 self.dut_ip = self.client_dut.droid.connectivityGetIPv4Addresses( 389 'wlan0')[0] 390 time.sleep(self.MED_SLEEP) 391 392 def setup_ping_test(self, testcase_params): 393 """Function that gets devices ready for the test. 394 395 Args: 396 testcase_params: dict containing test-specific parameters 397 """ 398 # Configure AP 399 self.setup_ap(testcase_params) 400 # Set attenuator to 0 dB 401 for attenuator in self.attenuators: 402 attenuator.set_atten(0, strict=False) 403 # Reset, configure, and connect DUT 404 self.setup_dut(testcase_params) 405 406 def parse_test_params(self, test_name): 407 test_name_params = test_name.split("_") 408 testcase_params = collections.OrderedDict() 409 if "range" in test_name: 410 testcase_params["channel"] = int(test_name_params[3][2:]) 411 testcase_params["mode"] = test_name_params[4] 412 num_atten_steps = int((self.testclass_params["range_atten_stop"] - 413 self.testclass_params["range_atten_start"]) 414 / self.testclass_params["range_atten_step"]) 415 testcase_params["atten_range"] = [ 416 self.testclass_params["range_atten_start"] + 417 x * self.testclass_params["range_atten_step"] 418 for x in range(0, num_atten_steps) 419 ] 420 testcase_params["ping_duration"] = self.testclass_params[ 421 "range_ping_duration"] 422 testcase_params["ping_interval"] = self.testclass_params[ 423 "range_ping_interval"] 424 testcase_params["ping_size"] = self.testclass_params["ping_size"] 425 else: 426 testcase_params["channel"] = int(test_name_params[4][2:]) 427 testcase_params["mode"] = test_name_params[5] 428 testcase_params["atten_range"] = self.testclass_params[ 429 "rtt_test_attenuation"] 430 testcase_params["ping_duration"] = self.testclass_params[ 431 "rtt_ping_duration"] 432 testcase_params["ping_interval"] = self.testclass_params[ 433 "rtt_ping_interval"][test_name_params[1]] 434 testcase_params["ping_size"] = self.testclass_params["ping_size"] 435 return testcase_params 436 437 def _test_ping_rtt(self): 438 """ Function that gets called for each RTT test case 439 440 The function gets called in each RTT test case. The function customizes 441 the RTT test based on the test name of the test that called it 442 """ 443 # Compile test parameters from config and test name 444 testcase_params = self.parse_test_params(self.current_test_name) 445 testcase_params.update(self.testclass_params) 446 # Run ping test 447 self.setup_ping_test(testcase_params) 448 ping_result = self.run_ping_test(testcase_params) 449 # Postprocess results 450 self.process_ping_results(testcase_params, ping_result) 451 self.pass_fail_check_ping_rtt(ping_result) 452 453 def _test_ping_range(self): 454 """ Function that gets called for each range test case 455 456 The function gets called in each range test case. It customizes the 457 range test based on the test name of the test that called it 458 """ 459 # Compile test parameters from config and test name 460 testcase_params = self.parse_test_params(self.current_test_name) 461 testcase_params.update(self.testclass_params) 462 # Run ping test 463 self.setup_ping_test(testcase_params) 464 ping_result = self.run_ping_test(testcase_params) 465 # Postprocess results 466 self.testclass_results.append(ping_result) 467 self.process_ping_results(testcase_params, ping_result) 468 self.pass_fail_check_ping_range(ping_result) 469 470 def test_ping_range_ch1_VHT20(self): 471 self._test_ping_range() 472 473 def test_ping_range_ch6_VHT20(self): 474 self._test_ping_range() 475 476 def test_ping_range_ch11_VHT20(self): 477 self._test_ping_range() 478 479 def test_ping_range_ch36_VHT20(self): 480 self._test_ping_range() 481 482 def test_ping_range_ch36_VHT40(self): 483 self._test_ping_range() 484 485 def test_ping_range_ch36_VHT80(self): 486 self._test_ping_range() 487 488 def test_ping_range_ch40_VHT20(self): 489 self._test_ping_range() 490 491 def test_ping_range_ch44_VHT20(self): 492 self._test_ping_range() 493 494 def test_ping_range_ch44_VHT40(self): 495 self._test_ping_range() 496 497 def test_ping_range_ch48_VHT20(self): 498 self._test_ping_range() 499 500 def test_ping_range_ch149_VHT20(self): 501 self._test_ping_range() 502 503 def test_ping_range_ch149_VHT40(self): 504 self._test_ping_range() 505 506 def test_ping_range_ch149_VHT80(self): 507 self._test_ping_range() 508 509 def test_ping_range_ch153_VHT20(self): 510 self._test_ping_range() 511 512 def test_ping_range_ch157_VHT20(self): 513 self._test_ping_range() 514 515 def test_ping_range_ch157_VHT40(self): 516 self._test_ping_range() 517 518 def test_ping_range_ch161_VHT20(self): 519 self._test_ping_range() 520 521 def test_fast_ping_rtt_ch1_VHT20(self): 522 self._test_ping_rtt() 523 524 def test_slow_ping_rtt_ch1_VHT20(self): 525 self._test_ping_rtt() 526 527 def test_fast_ping_rtt_ch6_VHT20(self): 528 self._test_ping_rtt() 529 530 def test_slow_ping_rtt_ch6_VHT20(self): 531 self._test_ping_rtt() 532 533 def test_fast_ping_rtt_ch11_VHT20(self): 534 self._test_ping_rtt() 535 536 def test_slow_ping_rtt_ch11_VHT20(self): 537 self._test_ping_rtt() 538 539 def test_fast_ping_rtt_ch36_VHT20(self): 540 self._test_ping_rtt() 541 542 def test_slow_ping_rtt_ch36_VHT20(self): 543 self._test_ping_rtt() 544 545 def test_fast_ping_rtt_ch36_VHT40(self): 546 self._test_ping_rtt() 547 548 def test_slow_ping_rtt_ch36_VHT40(self): 549 self._test_ping_rtt() 550 551 def test_fast_ping_rtt_ch36_VHT80(self): 552 self._test_ping_rtt() 553 554 def test_slow_ping_rtt_ch36_VHT80(self): 555 self._test_ping_rtt() 556 557 def test_fast_ping_rtt_ch149_VHT20(self): 558 self._test_ping_rtt() 559 560 def test_slow_ping_rtt_ch149_VHT20(self): 561 self._test_ping_rtt() 562 563 def test_fast_ping_rtt_ch149_VHT40(self): 564 self._test_ping_rtt() 565 566 def test_slow_ping_rtt_ch149_VHT40(self): 567 self._test_ping_rtt() 568 569 def test_fast_ping_rtt_ch149_VHT80(self): 570 self._test_ping_rtt() 571 572 def test_slow_ping_rtt_ch149_VHT80(self): 573 self._test_ping_rtt() 574