1#!/usr/bin/env python3.4 2# 3# Copyright 2017 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the 'License'); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an 'AS IS' BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import collections 18import itertools 19import json 20import logging 21import numpy 22import os 23import time 24from acts import asserts 25from acts import base_test 26from acts import utils 27from acts.controllers import iperf_server as ipf 28from acts.controllers.utils_lib import ssh 29from acts.metrics.loggers.blackbox import BlackboxMappedMetricLogger 30from acts_contrib.test_utils.wifi import ota_chamber 31from acts_contrib.test_utils.wifi import ota_sniffer 32from acts_contrib.test_utils.wifi import wifi_performance_test_utils as wputils 33from acts_contrib.test_utils.wifi.wifi_performance_test_utils.bokeh_figure import BokehFigure 34from acts_contrib.test_utils.wifi import wifi_retail_ap as retail_ap 35from acts_contrib.test_utils.wifi import wifi_test_utils as wutils 36from functools import partial 37 38 39class WifiRvrTest(base_test.BaseTestClass): 40 """Class to test WiFi rate versus range. 41 42 This class implements WiFi rate versus range tests on single AP single STA 43 links. The class setups up the AP in the desired configurations, configures 44 and connects the phone to the AP, and runs iperf throughput test while 45 sweeping attenuation. For an example config file to run this test class see 46 example_connectivity_performance_ap_sta.json. 47 """ 48 49 TEST_TIMEOUT = 6 50 MAX_CONSECUTIVE_ZEROS = 3 51 52 def __init__(self, controllers): 53 base_test.BaseTestClass.__init__(self, controllers) 54 self.testcase_metric_logger = ( 55 BlackboxMappedMetricLogger.for_test_case()) 56 self.testclass_metric_logger = ( 57 BlackboxMappedMetricLogger.for_test_class()) 58 self.publish_testcase_metrics = True 59 60 def setup_class(self): 61 """Initializes common test hardware and parameters. 62 63 This function initializes hardwares and compiles parameters that are 64 common to all tests in this class. 65 """ 66 self.sta_dut = self.android_devices[0] 67 req_params = [ 68 'RetailAccessPoints', 'rvr_test_params', 'testbed_params', 69 'RemoteServer', 'main_network' 70 ] 71 opt_params = ['golden_files_list', 'OTASniffer'] 72 self.unpack_userparams(req_params, opt_params) 73 self.testclass_params = self.rvr_test_params 74 self.num_atten = self.attenuators[0].instrument.num_atten 75 self.iperf_server = self.iperf_servers[0] 76 self.remote_server = ssh.connection.SshConnection( 77 ssh.settings.from_config(self.RemoteServer[0]['ssh_config'])) 78 self.iperf_client = self.iperf_clients[0] 79 self.access_point = retail_ap.create(self.RetailAccessPoints)[0] 80 if hasattr(self, 81 'OTASniffer') and self.testbed_params['sniffer_enable']: 82 try: 83 self.sniffer = ota_sniffer.create(self.OTASniffer)[0] 84 except: 85 self.log.warning('Could not start sniffer. Disabling sniffs.') 86 self.testbed_params['sniffer_enable'] = 0 87 self.log.info('Access Point Configuration: {}'.format( 88 self.access_point.ap_settings)) 89 self.log_path = os.path.join(logging.log_path, 'results') 90 os.makedirs(self.log_path, exist_ok=True) 91 if not hasattr(self, 'golden_files_list'): 92 if 'golden_results_path' in self.testbed_params: 93 self.golden_files_list = [ 94 os.path.join(self.testbed_params['golden_results_path'], 95 file) for file in 96 os.listdir(self.testbed_params['golden_results_path']) 97 ] 98 else: 99 self.log.warning('No golden files found.') 100 self.golden_files_list = [] 101 self.testclass_results = [] 102 103 # Turn WiFi ON 104 if self.testclass_params.get('airplane_mode', 1): 105 for dev in self.android_devices: 106 self.log.info('Turning on airplane mode.') 107 asserts.assert_true(utils.force_airplane_mode(dev, True), 108 'Can not turn on airplane mode.') 109 wutils.reset_wifi(dev) 110 wutils.wifi_toggle_state(dev, True) 111 112 def teardown_test(self): 113 self.iperf_server.stop() 114 115 def teardown_class(self): 116 # Turn WiFi OFF 117 self.access_point.teardown() 118 for dev in self.android_devices: 119 wutils.wifi_toggle_state(dev, False) 120 dev.go_to_sleep() 121 self.process_testclass_results() 122 123 def process_testclass_results(self): 124 """Saves plot with all test results to enable comparison.""" 125 # Plot and save all results 126 plots = collections.OrderedDict() 127 for result in self.testclass_results: 128 plot_id = (result['testcase_params']['channel'], 129 result['testcase_params']['mode']) 130 if plot_id not in plots: 131 plots[plot_id] = BokehFigure( 132 title='Channel {} {} ({})'.format( 133 result['testcase_params']['channel'], 134 result['testcase_params']['mode'], 135 result['testcase_params']['traffic_type']), 136 x_label='Attenuation (dB)', 137 primary_y_label='Throughput (Mbps)') 138 plots[plot_id].add_line(result['total_attenuation'], 139 result['throughput_receive'], 140 result['test_name'].strip('test_rvr_'), 141 hover_text=result['hover_text'], 142 marker='circle') 143 plots[plot_id].add_line(result['total_attenuation'], 144 result['rx_phy_rate'], 145 result['test_name'].strip('test_rvr_') + 146 ' (Rx PHY)', 147 hover_text=result['hover_text'], 148 style='dashed', 149 marker='inverted_triangle') 150 plots[plot_id].add_line(result['total_attenuation'], 151 result['tx_phy_rate'], 152 result['test_name'].strip('test_rvr_') + 153 ' (Tx PHY)', 154 hover_text=result['hover_text'], 155 style='dashed', 156 marker='triangle') 157 158 figure_list = [] 159 for plot_id, plot in plots.items(): 160 plot.generate_figure() 161 figure_list.append(plot) 162 output_file_path = os.path.join(self.log_path, 'results.html') 163 BokehFigure.save_figures(figure_list, output_file_path) 164 165 def pass_fail_check(self, rvr_result): 166 """Check the test result and decide if it passed or failed. 167 168 Checks the RvR test result and compares to a throughput limites for 169 the same configuration. The pass/fail tolerances are provided in the 170 config file. 171 172 Args: 173 rvr_result: dict containing attenuation, throughput and other data 174 """ 175 try: 176 throughput_limits = self.compute_throughput_limits(rvr_result) 177 except: 178 asserts.explicit_pass( 179 'Test passed by default. Golden file not found') 180 181 failure_count = 0 182 for idx, current_throughput in enumerate( 183 rvr_result['throughput_receive']): 184 if (current_throughput < throughput_limits['lower_limit'][idx] 185 or current_throughput > 186 throughput_limits['upper_limit'][idx]): 187 failure_count = failure_count + 1 188 189 # Set test metrics 190 rvr_result['metrics']['failure_count'] = failure_count 191 if self.publish_testcase_metrics: 192 self.testcase_metric_logger.add_metric('failure_count', 193 failure_count) 194 195 # Assert pass or fail 196 if failure_count >= self.testclass_params['failure_count_tolerance']: 197 asserts.fail('Test failed. Found {} points outside limits.'.format( 198 failure_count)) 199 asserts.explicit_pass( 200 'Test passed. Found {} points outside throughput limits.'.format( 201 failure_count)) 202 203 def compute_throughput_limits(self, rvr_result): 204 """Compute throughput limits for current test. 205 206 Checks the RvR test result and compares to a throughput limites for 207 the same configuration. The pass/fail tolerances are provided in the 208 config file. 209 210 Args: 211 rvr_result: dict containing attenuation, throughput and other meta 212 data 213 Returns: 214 throughput_limits: dict containing attenuation and throughput limit data 215 """ 216 test_name = self.current_test_name 217 golden_path = next(file_name for file_name in self.golden_files_list 218 if test_name in file_name) 219 with open(golden_path, 'r') as golden_file: 220 golden_results = json.load(golden_file) 221 golden_attenuation = [ 222 att + golden_results['fixed_attenuation'] 223 for att in golden_results['attenuation'] 224 ] 225 attenuation = [] 226 lower_limit = [] 227 upper_limit = [] 228 for idx, current_throughput in enumerate( 229 rvr_result['throughput_receive']): 230 current_att = rvr_result['attenuation'][idx] + rvr_result[ 231 'fixed_attenuation'] 232 att_distances = [ 233 abs(current_att - golden_att) 234 for golden_att in golden_attenuation 235 ] 236 sorted_distances = sorted(enumerate(att_distances), 237 key=lambda x: x[1]) 238 closest_indeces = [dist[0] for dist in sorted_distances[0:3]] 239 closest_throughputs = [ 240 golden_results['throughput_receive'][index] 241 for index in closest_indeces 242 ] 243 closest_throughputs.sort() 244 245 attenuation.append(current_att) 246 lower_limit.append( 247 max( 248 closest_throughputs[0] - max( 249 self.testclass_params['abs_tolerance'], 250 closest_throughputs[0] * 251 self.testclass_params['pct_tolerance'] / 100), 0)) 252 upper_limit.append(closest_throughputs[-1] + max( 253 self.testclass_params['abs_tolerance'], closest_throughputs[-1] 254 * self.testclass_params['pct_tolerance'] / 100)) 255 throughput_limits = { 256 'attenuation': attenuation, 257 'lower_limit': lower_limit, 258 'upper_limit': upper_limit 259 } 260 return throughput_limits 261 262 def plot_rvr_result(self, rvr_result): 263 """Saves plots and JSON formatted results. 264 265 Args: 266 rvr_result: dict containing attenuation, throughput and other meta 267 data 268 """ 269 # Save output as text file 270 results_file_path = os.path.join( 271 self.log_path, '{}.json'.format(self.current_test_name)) 272 with open(results_file_path, 'w') as results_file: 273 json.dump(wputils.serialize_dict(rvr_result), 274 results_file, 275 indent=4) 276 # Plot and save 277 figure = BokehFigure(title=self.current_test_name, 278 x_label='Attenuation (dB)', 279 primary_y_label='Throughput (Mbps)') 280 try: 281 golden_path = next(file_name 282 for file_name in self.golden_files_list 283 if self.current_test_name in file_name) 284 with open(golden_path, 'r') as golden_file: 285 golden_results = json.load(golden_file) 286 golden_attenuation = [ 287 att + golden_results['fixed_attenuation'] 288 for att in golden_results['attenuation'] 289 ] 290 throughput_limits = self.compute_throughput_limits(rvr_result) 291 shaded_region = { 292 'x_vector': throughput_limits['attenuation'], 293 'lower_limit': throughput_limits['lower_limit'], 294 'upper_limit': throughput_limits['upper_limit'] 295 } 296 figure.add_line(golden_attenuation, 297 golden_results['throughput_receive'], 298 'Golden Results', 299 color='green', 300 marker='circle', 301 shaded_region=shaded_region) 302 except: 303 self.log.warning('ValueError: Golden file not found') 304 305 # Generate graph annotatios 306 rvr_result['hover_text'] = { 307 'llstats': [ 308 'TX MCS = {0} ({1:.1f}%). RX MCS = {2} ({3:.1f}%)'.format( 309 curr_llstats['summary']['common_tx_mcs'], 310 curr_llstats['summary']['common_tx_mcs_freq'] * 100, 311 curr_llstats['summary']['common_rx_mcs'], 312 curr_llstats['summary']['common_rx_mcs_freq'] * 100) 313 for curr_llstats in rvr_result['llstats'] 314 ], 315 'rssi': [ 316 '{0:.2f} [{1:.2f},{2:.2f}]'.format( 317 rssi['signal_poll_rssi'], 318 rssi['chain_0_rssi'], 319 rssi['chain_1_rssi'], 320 ) for rssi in rvr_result['rssi'] 321 ] 322 } 323 324 figure.add_line(rvr_result['total_attenuation'], 325 rvr_result['throughput_receive'], 326 'Measured Throughput', 327 hover_text=rvr_result['hover_text'], 328 color='black', 329 marker='circle') 330 figure.add_line( 331 rvr_result['total_attenuation'][0:len(rvr_result['rx_phy_rate'])], 332 rvr_result['rx_phy_rate'], 333 'Rx PHY Rate', 334 hover_text=rvr_result['hover_text'], 335 color='blue', 336 style='dashed', 337 marker='inverted_triangle') 338 figure.add_line( 339 rvr_result['total_attenuation'][0:len(rvr_result['rx_phy_rate'])], 340 rvr_result['tx_phy_rate'], 341 'Tx PHY Rate', 342 hover_text=rvr_result['hover_text'], 343 color='red', 344 style='dashed', 345 marker='triangle') 346 347 output_file_path = os.path.join( 348 self.log_path, '{}.html'.format(self.current_test_name)) 349 figure.generate_figure(output_file_path) 350 351 def compute_test_metrics(self, rvr_result): 352 # Set test metrics 353 rvr_result['metrics'] = {} 354 rvr_result['metrics']['peak_tput'] = max( 355 rvr_result['throughput_receive']) 356 if self.publish_testcase_metrics: 357 self.testcase_metric_logger.add_metric( 358 'peak_tput', rvr_result['metrics']['peak_tput']) 359 360 test_mode = rvr_result['ap_settings'][rvr_result['testcase_params'] 361 ['band']]['bandwidth'] 362 tput_below_limit = [ 363 tput < 364 self.testclass_params['tput_metric_targets'][test_mode]['high'] 365 for tput in rvr_result['throughput_receive'] 366 ] 367 rvr_result['metrics']['high_tput_range'] = -1 368 for idx in range(len(tput_below_limit)): 369 if all(tput_below_limit[idx:]): 370 if idx == 0: 371 # Throughput was never above limit 372 rvr_result['metrics']['high_tput_range'] = -1 373 else: 374 rvr_result['metrics']['high_tput_range'] = rvr_result[ 375 'total_attenuation'][max(idx, 1) - 1] 376 break 377 if self.publish_testcase_metrics: 378 self.testcase_metric_logger.add_metric( 379 'high_tput_range', rvr_result['metrics']['high_tput_range']) 380 381 tput_below_limit = [ 382 tput < 383 self.testclass_params['tput_metric_targets'][test_mode]['low'] 384 for tput in rvr_result['throughput_receive'] 385 ] 386 for idx in range(len(tput_below_limit)): 387 if all(tput_below_limit[idx:]): 388 rvr_result['metrics']['low_tput_range'] = rvr_result[ 389 'total_attenuation'][max(idx, 1) - 1] 390 break 391 else: 392 rvr_result['metrics']['low_tput_range'] = -1 393 if self.publish_testcase_metrics: 394 self.testcase_metric_logger.add_metric( 395 'low_tput_range', rvr_result['metrics']['low_tput_range']) 396 397 def process_test_results(self, rvr_result): 398 self.plot_rvr_result(rvr_result) 399 self.compute_test_metrics(rvr_result) 400 401 def run_rvr_test(self, testcase_params): 402 """Test function to run RvR. 403 404 The function runs an RvR test in the current device/AP configuration. 405 Function is called from another wrapper function that sets up the 406 testbed for the RvR test 407 408 Args: 409 testcase_params: dict containing test-specific parameters 410 Returns: 411 rvr_result: dict containing rvr_results and meta data 412 """ 413 self.log.info('Start running RvR') 414 # Refresh link layer stats before test 415 llstats_obj = wputils.LinkLayerStats( 416 self.monitored_dut, 417 self.testclass_params.get('monitor_llstats', 1)) 418 zero_counter = 0 419 throughput = [] 420 rx_phy_rate = [] 421 tx_phy_rate = [] 422 llstats = [] 423 rssi = [] 424 for atten in testcase_params['atten_range']: 425 for dev in self.android_devices: 426 if not wputils.health_check(dev, 5, 50): 427 asserts.skip('DUT health check failed. Skipping test.') 428 # Set Attenuation 429 for attenuator in self.attenuators: 430 attenuator.set_atten(atten, strict=False, retry=True) 431 # Refresh link layer stats 432 llstats_obj.update_stats() 433 # Setup sniffer 434 if self.testbed_params['sniffer_enable']: 435 self.sniffer.start_capture( 436 network=testcase_params['test_network'], 437 chan=testcase_params['channel'], 438 bw=testcase_params['bandwidth'], 439 duration=self.testclass_params['iperf_duration'] / 5) 440 # Start iperf session 441 if self.testclass_params.get('monitor_rssi', 1): 442 rssi_future = wputils.get_connected_rssi_nb( 443 self.monitored_dut, 444 self.testclass_params['iperf_duration'] - 1, 445 1, 446 1, 447 interface=self.monitored_interface) 448 self.iperf_server.start(tag=str(atten)) 449 client_output_path = self.iperf_client.start( 450 testcase_params['iperf_server_address'], 451 testcase_params['iperf_args'], str(atten), 452 self.testclass_params['iperf_duration'] + self.TEST_TIMEOUT) 453 server_output_path = self.iperf_server.stop() 454 if self.testclass_params.get('monitor_rssi', 1): 455 rssi_result = rssi_future.result() 456 current_rssi = { 457 'signal_poll_rssi': 458 rssi_result['signal_poll_rssi']['mean'], 459 'chain_0_rssi': rssi_result['chain_0_rssi']['mean'], 460 'chain_1_rssi': rssi_result['chain_1_rssi']['mean'] 461 } 462 else: 463 current_rssi = { 464 'signal_poll_rssi': float('nan'), 465 'chain_0_rssi': float('nan'), 466 'chain_1_rssi': float('nan') 467 } 468 rssi.append(current_rssi) 469 # Stop sniffer 470 if self.testbed_params['sniffer_enable']: 471 self.sniffer.stop_capture(tag=str(atten)) 472 # Parse and log result 473 if testcase_params['use_client_output']: 474 iperf_file = client_output_path 475 else: 476 iperf_file = server_output_path 477 try: 478 iperf_result = ipf.IPerfResult(iperf_file) 479 curr_throughput = numpy.mean(iperf_result.instantaneous_rates[ 480 self.testclass_params['iperf_ignored_interval']:-1] 481 ) * 8 * (1.024**2) 482 except: 483 self.log.warning( 484 'ValueError: Cannot get iperf result. Setting to 0') 485 curr_throughput = 0 486 throughput.append(curr_throughput) 487 llstats_obj.update_stats() 488 curr_llstats = llstats_obj.llstats_incremental.copy() 489 llstats.append(curr_llstats) 490 rx_phy_rate.append(curr_llstats['summary'].get( 491 'mean_rx_phy_rate', 0)) 492 tx_phy_rate.append(curr_llstats['summary'].get( 493 'mean_tx_phy_rate', 0)) 494 self.log.info( 495 ('Throughput at {0:.2f} dB is {1:.2f} Mbps. ' 496 'RSSI = {2:.2f} [{3:.2f}, {4:.2f}].').format( 497 atten, curr_throughput, current_rssi['signal_poll_rssi'], 498 current_rssi['chain_0_rssi'], 499 current_rssi['chain_1_rssi'])) 500 if curr_throughput == 0: 501 zero_counter = zero_counter + 1 502 else: 503 zero_counter = 0 504 if zero_counter == self.MAX_CONSECUTIVE_ZEROS: 505 self.log.info( 506 'Throughput stable at 0 Mbps. Stopping test now.') 507 zero_padding = len( 508 testcase_params['atten_range']) - len(throughput) 509 throughput.extend([0] * zero_padding) 510 rx_phy_rate.extend([0] * zero_padding) 511 tx_phy_rate.extend([0] * zero_padding) 512 break 513 for attenuator in self.attenuators: 514 attenuator.set_atten(0, strict=False, retry=True) 515 # Compile test result and meta data 516 rvr_result = collections.OrderedDict() 517 rvr_result['test_name'] = self.current_test_name 518 rvr_result['testcase_params'] = testcase_params.copy() 519 rvr_result['ap_settings'] = self.access_point.ap_settings.copy() 520 rvr_result['fixed_attenuation'] = self.testbed_params[ 521 'fixed_attenuation'][str(testcase_params['channel'])] 522 rvr_result['attenuation'] = list(testcase_params['atten_range']) 523 rvr_result['total_attenuation'] = [ 524 att + rvr_result['fixed_attenuation'] 525 for att in rvr_result['attenuation'] 526 ] 527 rvr_result['rssi'] = rssi 528 rvr_result['throughput_receive'] = throughput 529 rvr_result['rx_phy_rate'] = rx_phy_rate 530 rvr_result['tx_phy_rate'] = tx_phy_rate 531 rvr_result['llstats'] = llstats 532 return rvr_result 533 534 def setup_ap(self, testcase_params): 535 """Sets up the access point in the configuration required by the test. 536 537 Args: 538 testcase_params: dict containing AP and other test params 539 """ 540 band = self.access_point.band_lookup_by_channel( 541 testcase_params['channel']) 542 if '6G' in band: 543 frequency = wutils.WifiEnums.channel_6G_to_freq[int( 544 testcase_params['channel'].strip('6g'))] 545 else: 546 if testcase_params['channel'] < 13: 547 frequency = wutils.WifiEnums.channel_2G_to_freq[ 548 testcase_params['channel']] 549 else: 550 frequency = wutils.WifiEnums.channel_5G_to_freq[ 551 testcase_params['channel']] 552 if frequency in wutils.WifiEnums.DFS_5G_FREQUENCIES: 553 self.access_point.set_region(self.testbed_params['DFS_region']) 554 else: 555 self.access_point.set_region(self.testbed_params['default_region']) 556 self.access_point.set_channel_and_bandwidth(testcase_params['band'], 557 testcase_params['channel'], 558 testcase_params['mode']) 559 self.log.info('Access Point Configuration: {}'.format( 560 self.access_point.ap_settings)) 561 562 def setup_dut(self, testcase_params): 563 """Sets up the DUT in the configuration required by the test. 564 565 Args: 566 testcase_params: dict containing AP and other test params 567 """ 568 # Turn screen off to preserve battery 569 if self.testbed_params.get('screen_on', 570 False) or self.testclass_params.get( 571 'screen_on', False): 572 self.sta_dut.droid.wakeLockAcquireDim() 573 else: 574 self.sta_dut.go_to_sleep() 575 if (wputils.validate_network(self.sta_dut, 576 testcase_params['test_network']['SSID']) 577 and not self.testclass_params.get('force_reconnect', 0)): 578 self.log.info('Already connected to desired network') 579 else: 580 wutils.wifi_toggle_state(self.sta_dut, False) 581 wutils.set_wifi_country_code(self.sta_dut, 582 self.testclass_params['country_code']) 583 wutils.wifi_toggle_state(self.sta_dut, True) 584 wutils.reset_wifi(self.sta_dut) 585 if self.testbed_params.get('txbf_off', False): 586 wputils.disable_beamforming(self.sta_dut) 587 wutils.set_wifi_country_code(self.sta_dut, 588 self.testclass_params['country_code']) 589 if self.testbed_params['sniffer_enable']: 590 self.sniffer.start_capture( 591 network={'SSID': testcase_params['test_network']['SSID']}, 592 chan=testcase_params['channel'], 593 bw=testcase_params['bandwidth'], 594 duration=180) 595 try: 596 wutils.wifi_connect(self.sta_dut, 597 testcase_params['test_network'], 598 num_of_tries=5, 599 check_connectivity=True) 600 if self.testclass_params.get('num_streams', 2) == 1: 601 wputils.set_nss_capability(self.sta_dut, 1) 602 finally: 603 if self.testbed_params['sniffer_enable']: 604 self.sniffer.stop_capture(tag='connection_setup') 605 606 def setup_rvr_test(self, testcase_params): 607 """Function that gets devices ready for the test. 608 609 Args: 610 testcase_params: dict containing test-specific parameters 611 """ 612 # Configure AP 613 self.setup_ap(testcase_params) 614 # Set attenuator to 0 dB 615 for attenuator in self.attenuators: 616 attenuator.set_atten(0, strict=False, retry=True) 617 # Reset, configure, and connect DUT 618 self.setup_dut(testcase_params) 619 # Wait before running the first wifi test 620 first_test_delay = self.testclass_params.get('first_test_delay', 600) 621 if first_test_delay > 0 and len(self.testclass_results) == 0: 622 self.log.info('Waiting before the first RvR test.') 623 time.sleep(first_test_delay) 624 self.setup_dut(testcase_params) 625 # Get iperf_server address 626 sta_dut_ip = self.sta_dut.droid.connectivityGetIPv4Addresses( 627 'wlan0')[0] 628 if isinstance(self.iperf_server, ipf.IPerfServerOverAdb): 629 testcase_params['iperf_server_address'] = sta_dut_ip 630 else: 631 if self.testbed_params.get('lan_traffic_only', True): 632 testcase_params[ 633 'iperf_server_address'] = wputils.get_server_address( 634 self.remote_server, sta_dut_ip, '255.255.255.0') 635 else: 636 testcase_params[ 637 'iperf_server_address'] = wputils.get_server_address( 638 self.remote_server, sta_dut_ip, 'public') 639 # Set DUT to monitor RSSI and LLStats on 640 self.monitored_dut = self.sta_dut 641 self.monitored_interface = 'wlan0' 642 643 def compile_test_params(self, testcase_params): 644 """Function that completes all test params based on the test name. 645 646 Args: 647 testcase_params: dict containing test-specific parameters 648 """ 649 # Check if test should be skipped based on parameters. 650 wputils.check_skip_conditions(testcase_params, self.sta_dut, 651 self.access_point, 652 getattr(self, 'ota_chamber', None)) 653 654 band = wputils.CHANNEL_TO_BAND_MAP[testcase_params['channel']] 655 start_atten = self.testclass_params['atten_start'].get(band, 0) 656 num_atten_steps = int( 657 (self.testclass_params['atten_stop'] - start_atten) / 658 self.testclass_params['atten_step']) 659 testcase_params['atten_range'] = [ 660 start_atten + x * self.testclass_params['atten_step'] 661 for x in range(0, num_atten_steps) 662 ] 663 band = self.access_point.band_lookup_by_channel( 664 testcase_params['channel']) 665 testcase_params['band'] = band 666 testcase_params['test_network'] = self.main_network[band] 667 if testcase_params['traffic_type'] == 'TCP': 668 testcase_params['iperf_socket_size'] = self.testclass_params.get( 669 'tcp_socket_size', None) 670 testcase_params['iperf_processes'] = self.testclass_params.get( 671 'tcp_processes', 1) 672 elif testcase_params['traffic_type'] == 'UDP': 673 testcase_params['iperf_socket_size'] = self.testclass_params.get( 674 'udp_socket_size', None) 675 testcase_params['iperf_processes'] = self.testclass_params.get( 676 'udp_processes', 1) 677 if (testcase_params['traffic_direction'] == 'DL' 678 and not isinstance(self.iperf_server, ipf.IPerfServerOverAdb) 679 ) or (testcase_params['traffic_direction'] == 'UL' 680 and isinstance(self.iperf_server, ipf.IPerfServerOverAdb)): 681 testcase_params['iperf_args'] = wputils.get_iperf_arg_string( 682 duration=self.testclass_params['iperf_duration'], 683 reverse_direction=1, 684 traffic_type=testcase_params['traffic_type'], 685 socket_size=testcase_params['iperf_socket_size'], 686 num_processes=testcase_params['iperf_processes'], 687 udp_throughput=self.testclass_params['UDP_rates'][ 688 testcase_params['mode']]) 689 testcase_params['use_client_output'] = True 690 else: 691 testcase_params['iperf_args'] = wputils.get_iperf_arg_string( 692 duration=self.testclass_params['iperf_duration'], 693 reverse_direction=0, 694 traffic_type=testcase_params['traffic_type'], 695 socket_size=testcase_params['iperf_socket_size'], 696 num_processes=testcase_params['iperf_processes'], 697 udp_throughput=self.testclass_params['UDP_rates'][ 698 testcase_params['mode']]) 699 testcase_params['use_client_output'] = False 700 return testcase_params 701 702 def _test_rvr(self, testcase_params): 703 """ Function that gets called for each test case 704 705 Args: 706 testcase_params: dict containing test-specific parameters 707 """ 708 # Compile test parameters from config and test name 709 testcase_params = self.compile_test_params(testcase_params) 710 711 # Prepare devices and run test 712 self.setup_rvr_test(testcase_params) 713 rvr_result = self.run_rvr_test(testcase_params) 714 715 # Post-process results 716 self.testclass_results.append(rvr_result) 717 self.process_test_results(rvr_result) 718 self.pass_fail_check(rvr_result) 719 720 def generate_test_cases(self, channels, modes, traffic_types, 721 traffic_directions): 722 """Function that auto-generates test cases for a test class.""" 723 test_cases = [] 724 allowed_configs = { 725 20: [ 726 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 727 116, 132, 140, 149, 153, 157, 161, '6g37', '6g117', '6g213' 728 ], 729 40: [36, 44, 100, 149, 157, '6g37', '6g117', '6g213'], 730 80: [36, 100, 149, '6g37', '6g117', '6g213'], 731 160: [36, '6g37', '6g117', '6g213'] 732 } 733 734 for channel, mode, traffic_type, traffic_direction in itertools.product( 735 channels, modes, traffic_types, traffic_directions): 736 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 737 if channel not in allowed_configs[bandwidth]: 738 continue 739 test_name = 'test_rvr_{}_{}_ch{}_{}'.format( 740 traffic_type, traffic_direction, channel, mode) 741 test_params = collections.OrderedDict( 742 channel=channel, 743 mode=mode, 744 bandwidth=bandwidth, 745 traffic_type=traffic_type, 746 traffic_direction=traffic_direction) 747 setattr(self, test_name, partial(self._test_rvr, test_params)) 748 test_cases.append(test_name) 749 return test_cases 750 751 752class WifiRvr_TCP_Test(WifiRvrTest): 753 754 def __init__(self, controllers): 755 super().__init__(controllers) 756 self.tests = self.generate_test_cases( 757 channels=[ 758 1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117', 759 '6g213' 760 ], 761 modes=['bw20', 'bw40', 'bw80', 'bw160'], 762 traffic_types=['TCP'], 763 traffic_directions=['DL', 'UL']) 764 765 766class WifiRvr_VHT_TCP_Test(WifiRvrTest): 767 768 def __init__(self, controllers): 769 super().__init__(controllers) 770 self.tests = self.generate_test_cases( 771 channels=[1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161], 772 modes=['VHT20', 'VHT40', 'VHT80'], 773 traffic_types=['TCP'], 774 traffic_directions=['DL', 'UL']) 775 776 777class WifiRvr_HE_TCP_Test(WifiRvrTest): 778 779 def __init__(self, controllers): 780 super().__init__(controllers) 781 self.tests = self.generate_test_cases( 782 channels=[ 783 1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117', 784 '6g213' 785 ], 786 modes=['HE20', 'HE40', 'HE80', 'HE160'], 787 traffic_types=['TCP'], 788 traffic_directions=['DL', 'UL']) 789 790 791class WifiRvr_SampleUDP_Test(WifiRvrTest): 792 793 def __init__(self, controllers): 794 super().__init__(controllers) 795 self.tests = self.generate_test_cases( 796 channels=[6, 36, 149, '6g37'], 797 modes=['bw20', 'bw40', 'bw80', 'bw160'], 798 traffic_types=['UDP'], 799 traffic_directions=['DL', 'UL']) 800 801 802class WifiRvr_VHT_SampleUDP_Test(WifiRvrTest): 803 804 def __init__(self, controllers): 805 super().__init__(controllers) 806 self.tests = self.generate_test_cases( 807 channels=[6, 36, 149], 808 modes=['VHT20', 'VHT40', 'VHT80', 'VHT160'], 809 traffic_types=['UDP'], 810 traffic_directions=['DL', 'UL']) 811 812 813class WifiRvr_HE_SampleUDP_Test(WifiRvrTest): 814 815 def __init__(self, controllers): 816 super().__init__(controllers) 817 self.tests = self.generate_test_cases( 818 channels=[6, 36, 149], 819 modes=['HE20', 'HE40', 'HE80', 'HE160', '6g37'], 820 traffic_types=['UDP'], 821 traffic_directions=['DL', 'UL']) 822 823 824class WifiRvr_SampleDFS_Test(WifiRvrTest): 825 826 def __init__(self, controllers): 827 super().__init__(controllers) 828 self.tests = self.generate_test_cases( 829 channels=[64, 100, 116, 132, 140], 830 modes=['bw20', 'bw40', 'bw80'], 831 traffic_types=['TCP'], 832 traffic_directions=['DL', 'UL']) 833 834 835class WifiRvr_SingleChain_TCP_Test(WifiRvrTest): 836 837 def __init__(self, controllers): 838 super().__init__(controllers) 839 self.tests = self.generate_test_cases( 840 channels=[ 841 1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117', 842 '6g213' 843 ], 844 modes=['bw20', 'bw40', 'bw80', 'bw160'], 845 traffic_types=['TCP'], 846 traffic_directions=['DL', 'UL'], 847 chains=[0, 1, '2x2']) 848 849 def setup_dut(self, testcase_params): 850 self.sta_dut = self.android_devices[0] 851 wputils.set_chain_mask(self.sta_dut, testcase_params['chain']) 852 WifiRvrTest.setup_dut(self, testcase_params) 853 854 def generate_test_cases(self, channels, modes, traffic_types, 855 traffic_directions, chains): 856 """Function that auto-generates test cases for a test class.""" 857 test_cases = [] 858 allowed_configs = { 859 20: [ 860 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 861 116, 132, 140, 149, 153, 157, 161, '6g37', '6g117', '6g213' 862 ], 863 40: [36, 44, 100, 149, 157, '6g37', '6g117', '6g213'], 864 80: [36, 100, 149, '6g37', '6g117', '6g213'], 865 160: [36, '6g37', '6g117', '6g213'] 866 } 867 868 for channel, mode, chain, traffic_type, traffic_direction in itertools.product( 869 channels, modes, chains, traffic_types, traffic_directions): 870 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 871 if channel not in allowed_configs[bandwidth]: 872 continue 873 test_name = 'test_rvr_{}_{}_ch{}_{}_ch{}'.format( 874 traffic_type, traffic_direction, channel, mode, chain) 875 test_params = collections.OrderedDict( 876 channel=channel, 877 mode=mode, 878 bandwidth=bandwidth, 879 traffic_type=traffic_type, 880 traffic_direction=traffic_direction, 881 chain=chain) 882 setattr(self, test_name, partial(self._test_rvr, test_params)) 883 test_cases.append(test_name) 884 return test_cases 885 886 887# Over-the air version of RVR tests 888class WifiOtaRvrTest(WifiRvrTest): 889 """Class to test over-the-air RvR 890 891 This class implements measures WiFi RvR tests in an OTA chamber. It enables 892 setting turntable orientation and other chamber parameters to study 893 performance in varying channel conditions 894 """ 895 896 def __init__(self, controllers): 897 base_test.BaseTestClass.__init__(self, controllers) 898 self.testcase_metric_logger = ( 899 BlackboxMappedMetricLogger.for_test_case()) 900 self.testclass_metric_logger = ( 901 BlackboxMappedMetricLogger.for_test_class()) 902 self.publish_testcase_metrics = False 903 904 def setup_class(self): 905 WifiRvrTest.setup_class(self) 906 self.ota_chamber = ota_chamber.create( 907 self.user_params['OTAChamber'])[0] 908 909 def teardown_class(self): 910 WifiRvrTest.teardown_class(self) 911 self.ota_chamber.reset_chamber() 912 913 def extract_test_id(self, testcase_params, id_fields): 914 test_id = collections.OrderedDict( 915 (param, testcase_params.get(param, None)) for param in id_fields) 916 return test_id 917 918 def process_testclass_results(self): 919 """Saves plot with all test results to enable comparison.""" 920 # Plot individual test id results raw data and compile metrics 921 plots = collections.OrderedDict() 922 compiled_data = collections.OrderedDict() 923 for result in self.testclass_results: 924 test_id = tuple( 925 self.extract_test_id(result['testcase_params'], [ 926 'channel', 'mode', 'traffic_type', 'traffic_direction', 927 'chain' 928 ]).items()) 929 if test_id not in plots: 930 # Initialize test id data when not present 931 compiled_data[test_id] = { 932 'throughput': [], 933 'rx_phy_rate': [], 934 'tx_phy_rate': [], 935 'metrics': {} 936 } 937 compiled_data[test_id]['metrics'] = { 938 key: [] 939 for key in result['metrics'].keys() 940 } 941 plots[test_id] = BokehFigure( 942 title='Channel {} {} ({} {})'.format( 943 result['testcase_params']['channel'], 944 result['testcase_params']['mode'], 945 result['testcase_params']['traffic_type'], 946 result['testcase_params']['traffic_direction']), 947 x_label='Attenuation (dB)', 948 primary_y_label='Throughput (Mbps)') 949 test_id_phy = test_id + tuple('PHY') 950 plots[test_id_phy] = BokehFigure( 951 title='Channel {} {} ({} {}) (PHY Rate)'.format( 952 result['testcase_params']['channel'], 953 result['testcase_params']['mode'], 954 result['testcase_params']['traffic_type'], 955 result['testcase_params']['traffic_direction']), 956 x_label='Attenuation (dB)', 957 primary_y_label='PHY Rate (Mbps)') 958 # Compile test id data and metrics 959 compiled_data[test_id]['throughput'].append( 960 result['throughput_receive']) 961 compiled_data[test_id]['rx_phy_rate'].append(result['rx_phy_rate']) 962 compiled_data[test_id]['tx_phy_rate'].append(result['tx_phy_rate']) 963 compiled_data[test_id]['total_attenuation'] = result[ 964 'total_attenuation'] 965 for metric_key, metric_value in result['metrics'].items(): 966 compiled_data[test_id]['metrics'][metric_key].append( 967 metric_value) 968 # Add test id to plots 969 plots[test_id].add_line(result['total_attenuation'], 970 result['throughput_receive'], 971 result['test_name'].strip('test_rvr_'), 972 hover_text=result['hover_text'], 973 width=1, 974 style='dashed', 975 marker='circle') 976 plots[test_id_phy].add_line( 977 result['total_attenuation'], 978 result['rx_phy_rate'], 979 result['test_name'].strip('test_rvr_') + ' Rx PHY Rate', 980 hover_text=result['hover_text'], 981 width=1, 982 style='dashed', 983 marker='inverted_triangle') 984 plots[test_id_phy].add_line( 985 result['total_attenuation'], 986 result['tx_phy_rate'], 987 result['test_name'].strip('test_rvr_') + ' Tx PHY Rate', 988 hover_text=result['hover_text'], 989 width=1, 990 style='dashed', 991 marker='triangle') 992 993 # Compute average RvRs and compute metrics over orientations 994 for test_id, test_data in compiled_data.items(): 995 test_id_dict = dict(test_id) 996 metric_tag = '{}_{}_ch{}_{}'.format( 997 test_id_dict['traffic_type'], 998 test_id_dict['traffic_direction'], test_id_dict['channel'], 999 test_id_dict['mode']) 1000 high_tput_hit_freq = numpy.mean( 1001 numpy.not_equal(test_data['metrics']['high_tput_range'], -1)) 1002 self.testclass_metric_logger.add_metric( 1003 '{}.high_tput_hit_freq'.format(metric_tag), high_tput_hit_freq) 1004 for metric_key, metric_value in test_data['metrics'].items(): 1005 metric_key = '{}.avg_{}'.format(metric_tag, metric_key) 1006 metric_value = numpy.mean(metric_value) 1007 self.testclass_metric_logger.add_metric( 1008 metric_key, metric_value) 1009 test_data['avg_rvr'] = numpy.mean(test_data['throughput'], 0) 1010 test_data['median_rvr'] = numpy.median(test_data['throughput'], 0) 1011 test_data['avg_rx_phy_rate'] = numpy.mean(test_data['rx_phy_rate'], 1012 0) 1013 test_data['avg_tx_phy_rate'] = numpy.mean(test_data['tx_phy_rate'], 1014 0) 1015 plots[test_id].add_line(test_data['total_attenuation'], 1016 test_data['avg_rvr'], 1017 legend='Average Throughput', 1018 marker='circle') 1019 plots[test_id].add_line(test_data['total_attenuation'], 1020 test_data['median_rvr'], 1021 legend='Median Throughput', 1022 marker='square') 1023 test_id_phy = test_id + tuple('PHY') 1024 plots[test_id_phy].add_line(test_data['total_attenuation'], 1025 test_data['avg_rx_phy_rate'], 1026 legend='Average Rx Rate', 1027 marker='inverted_triangle') 1028 plots[test_id_phy].add_line(test_data['total_attenuation'], 1029 test_data['avg_tx_phy_rate'], 1030 legend='Average Tx Rate', 1031 marker='triangle') 1032 1033 figure_list = [] 1034 for plot_id, plot in plots.items(): 1035 plot.generate_figure() 1036 figure_list.append(plot) 1037 output_file_path = os.path.join(self.log_path, 'results.html') 1038 BokehFigure.save_figures(figure_list, output_file_path) 1039 1040 def setup_rvr_test(self, testcase_params): 1041 # Continue test setup 1042 WifiRvrTest.setup_rvr_test(self, testcase_params) 1043 # Set turntable orientation 1044 self.ota_chamber.set_orientation(testcase_params['orientation']) 1045 1046 def generate_test_cases(self, channels, modes, angles, traffic_types, 1047 directions): 1048 test_cases = [] 1049 allowed_configs = { 1050 20: [ 1051 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 1052 116, 132, 140, 149, 153, 157, 161 1053 ], 1054 40: [36, 44, 100, 149, 157], 1055 80: [36, 100, 149], 1056 160: [36, '6g37', '6g117', '6g213'] 1057 } 1058 for channel, mode, angle, traffic_type, direction in itertools.product( 1059 channels, modes, angles, traffic_types, directions): 1060 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 1061 if channel not in allowed_configs[bandwidth]: 1062 continue 1063 testcase_name = 'test_rvr_{}_{}_ch{}_{}_{}deg'.format( 1064 traffic_type, direction, channel, mode, angle) 1065 test_params = collections.OrderedDict(channel=channel, 1066 mode=mode, 1067 bandwidth=bandwidth, 1068 traffic_type=traffic_type, 1069 traffic_direction=direction, 1070 orientation=angle) 1071 setattr(self, testcase_name, partial(self._test_rvr, test_params)) 1072 test_cases.append(testcase_name) 1073 return test_cases 1074 1075 1076class WifiOtaRvr_StandardOrientation_Test(WifiOtaRvrTest): 1077 1078 def __init__(self, controllers): 1079 WifiOtaRvrTest.__init__(self, controllers) 1080 self.tests = self.generate_test_cases( 1081 [1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37'], 1082 ['bw20', 'bw40', 'bw80', 'bw160'], list(range(0, 360, 45)), 1083 ['TCP'], ['DL', 'UL']) 1084 1085 1086class WifiOtaRvr_SampleChannel_Test(WifiOtaRvrTest): 1087 1088 def __init__(self, controllers): 1089 WifiOtaRvrTest.__init__(self, controllers) 1090 self.tests = self.generate_test_cases([6], ['bw20'], 1091 list(range(0, 360, 45)), ['TCP'], 1092 ['DL']) 1093 self.tests.extend( 1094 self.generate_test_cases([36, 149], ['bw80', 'bw160'], 1095 list(range(0, 360, 45)), ['TCP'], ['DL'])) 1096 self.tests.extend( 1097 self.generate_test_cases(['6g37'], ['bw160'], 1098 list(range(0, 360, 45)), ['TCP'], ['DL'])) 1099 1100 1101class WifiOtaRvr_SingleOrientation_Test(WifiOtaRvrTest): 1102 1103 def __init__(self, controllers): 1104 WifiOtaRvrTest.__init__(self, controllers) 1105 self.tests = self.generate_test_cases( 1106 [6, 36, 40, 44, 48, 149, 153, 157, 161, '6g37'], 1107 ['bw20', 'bw40', 'bw80', 'bw160'], [0], ['TCP'], ['DL', 'UL']) 1108 1109 1110class WifiOtaRvr_SingleChain_Test(WifiOtaRvrTest): 1111 1112 def __init__(self, controllers): 1113 WifiOtaRvrTest.__init__(self, controllers) 1114 self.tests = self.generate_test_cases([6], ['bw20'], 1115 list(range(0, 360, 45)), ['TCP'], 1116 ['DL', 'UL'], [0, 1]) 1117 self.tests.extend( 1118 self.generate_test_cases([36, 149], ['bw20', 'bw80', 'bw160'], 1119 list(range(0, 360, 45)), ['TCP'], 1120 ['DL', 'UL'], [0, 1, '2x2'])) 1121 self.tests.extend( 1122 self.generate_test_cases(['6g37'], ['bw20', 'bw80', 'bw160'], 1123 list(range(0, 360, 45)), ['TCP'], 1124 ['DL', 'UL'], [0, 1, '2x2'])) 1125 1126 def setup_dut(self, testcase_params): 1127 self.sta_dut = self.android_devices[0] 1128 wputils.set_chain_mask(self.sta_dut, testcase_params['chain']) 1129 WifiRvrTest.setup_dut(self, testcase_params) 1130 1131 def generate_test_cases(self, channels, modes, angles, traffic_types, 1132 directions, chains): 1133 test_cases = [] 1134 allowed_configs = { 1135 20: [ 1136 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 1137 116, 132, 140, 149, 153, 157, 161 1138 ], 1139 40: [36, 44, 100, 149, 157], 1140 80: [36, 100, 149], 1141 160: [36, '6g37', '6g117', '6g213'] 1142 } 1143 for channel, mode, chain, angle, traffic_type, direction in itertools.product( 1144 channels, modes, chains, angles, traffic_types, directions): 1145 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 1146 if channel not in allowed_configs[bandwidth]: 1147 continue 1148 testcase_name = 'test_rvr_{}_{}_ch{}_{}_ch{}_{}deg'.format( 1149 traffic_type, direction, channel, mode, chain, angle) 1150 test_params = collections.OrderedDict(channel=channel, 1151 mode=mode, 1152 bandwidth=bandwidth, 1153 chain=chain, 1154 traffic_type=traffic_type, 1155 traffic_direction=direction, 1156 orientation=angle) 1157 setattr(self, testcase_name, partial(self._test_rvr, test_params)) 1158 test_cases.append(testcase_name) 1159 return test_cases 1160