1#!/usr/bin/python2 2 3"""Unit tests for the perf_uploader.py module. 4 5""" 6 7import json 8import os 9import tempfile 10import unittest 11 12import common 13from autotest_lib.tko import models as tko_models 14from autotest_lib.tko.perf_upload import perf_uploader 15 16 17class test_aggregate_iterations(unittest.TestCase): 18 """Tests for the aggregate_iterations function.""" 19 20 _PERF_ITERATION_DATA = { 21 '1': [ 22 { 23 'description': 'metric1', 24 'value': 1, 25 'stddev': 0.0, 26 'units': 'units1', 27 'higher_is_better': True, 28 'graph': None 29 }, 30 { 31 'description': 'metric2', 32 'value': 10, 33 'stddev': 0.0, 34 'units': 'units2', 35 'higher_is_better': True, 36 'graph': 'graph1', 37 }, 38 { 39 'description': 'metric2', 40 'value': 100, 41 'stddev': 1.7, 42 'units': 'units3', 43 'higher_is_better': False, 44 'graph': 'graph2', 45 } 46 ], 47 '2': [ 48 { 49 'description': 'metric1', 50 'value': 2, 51 'stddev': 0.0, 52 'units': 'units1', 53 'higher_is_better': True, 54 'graph': None, 55 }, 56 { 57 'description': 'metric2', 58 'value': 20, 59 'stddev': 0.0, 60 'units': 'units2', 61 'higher_is_better': True, 62 'graph': 'graph1', 63 }, 64 { 65 'description': 'metric2', 66 'value': 200, 67 'stddev': 21.2, 68 'units': 'units3', 69 'higher_is_better': False, 70 'graph': 'graph2', 71 } 72 ], 73 } 74 75 76 def setUp(self): 77 """Sets up for each test case.""" 78 self._perf_values = [] 79 for iter_num, iter_data in self._PERF_ITERATION_DATA.iteritems(): 80 self._perf_values.append( 81 tko_models.perf_value_iteration(iter_num, iter_data)) 82 83 84 85class test_json_config_file_sanity(unittest.TestCase): 86 """Sanity tests for the JSON-formatted presentation config file.""" 87 88 def test_parse_json(self): 89 """Verifies _parse_config_file function.""" 90 perf_uploader._parse_config_file( 91 perf_uploader._PRESENTATION_CONFIG_FILE) 92 93 def test_proper_config(self): 94 """Verifies configs have either autotest_name or autotest_regex.""" 95 json_obj = [] 96 try: 97 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 98 json_obj = json.load(fp) 99 except: 100 self.fail('Presentation config file could not be parsed as JSON.') 101 102 for entry in json_obj: 103 if 'autotest_name' not in entry and 'autotest_regex' not in entry: 104 self.fail('Missing autotest_name or autotest_regex field for ' 105 'test %s.' % entry) 106 107 108 def test_proper_json(self): 109 """Verifies the file can be parsed as proper JSON.""" 110 try: 111 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 112 json.load(fp) 113 except: 114 self.fail('Presentation config file could not be parsed as JSON.') 115 116 117 def test_required_master_name(self): 118 """Verifies that master name must be specified.""" 119 json_obj = [] 120 try: 121 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 122 json_obj = json.load(fp) 123 except: 124 self.fail('Presentation config file could not be parsed as JSON.') 125 126 for entry in json_obj: 127 if not 'master_name' in entry: 128 self.fail('Missing master field for test %s.' % 129 entry['autotest_name']) 130 131class test_get_image_board_name(unittest.TestCase): 132 """Sanity tests for retrieving the image board name.""" 133 def test_normal_platform(self): 134 """Verify image board name is equal to the platform in normal image.""" 135 platform = 'veyron_jerry' 136 image = 'veyron_jerry-release/R78-12428.0.0' 137 self.assertEqual(perf_uploader._get_image_board_name(platform, image), 138 'veyron_jerry') 139 140 def test_empty_platform(self): 141 """Sanity Verify image board name is equal to the platform.""" 142 platform = '' 143 image = '-release/R78-12428.0.0' 144 self.assertEqual(perf_uploader._get_image_board_name(platform, image), 145 '') 146 147 def test_specifc_image_suffix_found(self): 148 """Verify image board name is reflecting the running image.""" 149 platform = 'veyron_jerry' 150 image = 'veyron_jerry-kernelnext-release/R78-12419.0.0' 151 self.assertEqual(perf_uploader._get_image_board_name(platform, image), 152 'veyron_jerry-kernelnext') 153 image = 'veyron_jerry-arcnext-release/R78-12419.0.0' 154 self.assertEqual(perf_uploader._get_image_board_name(platform, image), 155 'veyron_jerry-arcnext') 156 image = 'veyron_jerry-arcvm-release/R78-12419.0.0' 157 self.assertEqual(perf_uploader._get_image_board_name(platform, image), 158 'veyron_jerry-arcvm') 159 160 161class test_gather_presentation_info(unittest.TestCase): 162 """Tests for the gather_presentation_info function.""" 163 164 _PRESENT_INFO = { 165 'test_name': { 166 'master_name': 'new_master_name', 167 'dashboard_test_name': 'new_test_name', 168 } 169 } 170 171 _PRESENT_INFO_MISSING_MASTER = { 172 'test_name': { 173 'dashboard_test_name': 'new_test_name', 174 } 175 } 176 177 _PRESENT_INFO_REGEX = { 178 'test_name.*': { 179 'master_name': 'new_master_name', 180 'dashboard_test_name': 'new_test_name', 181 } 182 } 183 184 _PRESENT_INFO_COLLISION = { 185 'test_name.*': { 186 'master_name': 'new_master_name', 187 'dashboard_test_name': 'new_test_name', 188 }, 189 'test_name-test.*': { 190 'master_name': 'new_master_name', 191 'dashboard_test_name': 'new_test_name', 192 }, 193 } 194 195 def test_test_selection_collision(self): 196 """Verifies error when multiple entry refers to the same test.""" 197 try: 198 result = perf_uploader._gather_presentation_info( 199 self._PRESENT_INFO_COLLISION, 'test_name-test-23') 200 self.fail('PerfUploadingError is expected if more than one entry ' 201 'refer to the same test.') 202 except perf_uploader.PerfUploadingError: 203 return 204 205 def test_test_name_regex_specified(self): 206 """Verifies gathers presentation info for regex search correctly""" 207 for test_name in ['test_name.arm.7.1', 'test_name.x86.7.1']: 208 result = perf_uploader._gather_presentation_info( 209 self._PRESENT_INFO, 'test_name_P') 210 self.assertTrue( 211 all([key in result for key in 212 ['test_name', 'master_name']]), 213 msg='Unexpected keys in resulting dictionary: %s' % result) 214 self.assertEqual(result['master_name'], 'new_master_name', 215 msg='Unexpected "master_name" value: %s' % 216 result['master_name']) 217 self.assertEqual(result['test_name'], 'new_test_name', 218 msg='Unexpected "test_name" value: %s' % 219 result['test_name']) 220 221 def test_test_name_specified(self): 222 """Verifies gathers presentation info correctly.""" 223 result = perf_uploader._gather_presentation_info( 224 self._PRESENT_INFO, 'test_name') 225 self.assertTrue( 226 all([key in result for key in 227 ['test_name', 'master_name']]), 228 msg='Unexpected keys in resulting dictionary: %s' % result) 229 self.assertEqual(result['master_name'], 'new_master_name', 230 msg='Unexpected "master_name" value: %s' % 231 result['master_name']) 232 self.assertEqual(result['test_name'], 'new_test_name', 233 msg='Unexpected "test_name" value: %s' % 234 result['test_name']) 235 236 237 def test_test_name_not_specified(self): 238 """Verifies exception raised if test is not there.""" 239 self.assertRaises( 240 perf_uploader.PerfUploadingError, 241 perf_uploader._gather_presentation_info, 242 self._PRESENT_INFO, 'other_test_name') 243 244 245 def test_master_not_specified(self): 246 """Verifies exception raised if master is not there.""" 247 self.assertRaises( 248 perf_uploader.PerfUploadingError, 249 perf_uploader._gather_presentation_info, 250 self._PRESENT_INFO_MISSING_MASTER, 'test_name') 251 252 253class test_parse_and_gather_presentation(unittest.TestCase): 254 """Tests for _parse_config_file and then_gather_presentation_info.""" 255 _AUTOTEST_NAME_CONFIG = """[{ 256 "autotest_name": "test.test.VM", 257 "master_name": "ChromeOSPerf" 258 }]""" 259 260 _AUTOTEST_REGEX_CONFIG = r"""[{ 261 "autotest_regex": "test\\.test\\.VM.*", 262 "master_name": "ChromeOSPerf" 263 }]""" 264 265 def setUp(self): 266 _, self._temp_path = tempfile.mkstemp() 267 268 def tearDown(self): 269 os.remove(self._temp_path) 270 271 def test_autotest_name_is_matched(self): 272 """Verifies that autotest name is matched to the test properly.""" 273 with open(self._temp_path, 'w') as f: 274 f.write(self._AUTOTEST_NAME_CONFIG) 275 config = perf_uploader._parse_config_file(self._temp_path) 276 test_name = 'test.test.VM' 277 result = perf_uploader._gather_presentation_info(config, test_name) 278 self.assertEqual(result, { 279 'test_name': test_name, 280 'master_name': 'ChromeOSPerf' 281 }) 282 283 def test_autotest_name_is_exact_matched(self): 284 """Verifies that autotest name is exact matched to the test properly.""" 285 with open(self._temp_path, 'w') as f: 286 f.write(self._AUTOTEST_NAME_CONFIG) 287 config = perf_uploader._parse_config_file(self._temp_path) 288 test_name = 'test.test.VM.test' 289 try: 290 perf_uploader._gather_presentation_info(config, test_name) 291 self.fail( 292 'PerfUploadingError is expected for %s. autotest_name should ' 293 'be exactly matched.' % test_name) 294 except perf_uploader.PerfUploadingError: 295 return 296 297 def test_autotest_name_is_escaped(self): 298 """Verifies that autotest name is escaped properly.""" 299 with open(self._temp_path, 'w') as f: 300 f.write(self._AUTOTEST_NAME_CONFIG) 301 config = perf_uploader._parse_config_file(self._temp_path) 302 try: 303 test_name = 'test.testkVM' 304 result = perf_uploader._gather_presentation_info( 305 config, test_name) 306 self.fail( 307 'PerfUploadingError is expected for %s. autotest_name should ' 308 'be escaped' % test_name) 309 except perf_uploader.PerfUploadingError: 310 return 311 312 def test_autotest_regex_is_matched(self): 313 """Verifies that autotest regex is matched to the test properly.""" 314 with open(self._temp_path, 'w') as f: 315 f.write(self._AUTOTEST_REGEX_CONFIG) 316 config = perf_uploader._parse_config_file(self._temp_path) 317 for test_name in ['test.test.VM1', 'test.test.VMTest']: 318 result = perf_uploader._gather_presentation_info(config, test_name) 319 self.assertEqual(result, { 320 'test_name': test_name, 321 'master_name': 'ChromeOSPerf' 322 }) 323 324 def test_autotest_regex_is_not_matched(self): 325 """Verifies that autotest regex is matched to the test properly.""" 326 with open(self._temp_path, 'w') as f: 327 f.write(self._AUTOTEST_REGEX_CONFIG) 328 config = perf_uploader._parse_config_file(self._temp_path) 329 for test_name in ['testktest.VM', 'test.testkVM', 'test.test\VM']: 330 try: 331 result = perf_uploader._gather_presentation_info( 332 config, test_name) 333 self.fail('PerfUploadingError is expected for %s' % test_name) 334 except perf_uploader.PerfUploadingError: 335 return 336 337class test_get_id_from_version(unittest.TestCase): 338 """Tests for the _get_id_from_version function.""" 339 340 def test_correctly_formatted_versions(self): 341 """Verifies that the expected ID is returned when input is OK.""" 342 chrome_version = '27.0.1452.2' 343 cros_version = '27.3906.0.0' 344 # 1452.2 + 3906.0.0 345 # --> 01452 + 002 + 03906 + 000 + 00 346 # --> 14520020390600000 347 self.assertEqual( 348 14520020390600000, 349 perf_uploader._get_id_from_version( 350 chrome_version, cros_version)) 351 352 chrome_version = '25.10.1000.0' 353 cros_version = '25.1200.0.0' 354 # 1000.0 + 1200.0.0 355 # --> 01000 + 000 + 01200 + 000 + 00 356 # --> 10000000120000000 357 self.assertEqual( 358 10000000120000000, 359 perf_uploader._get_id_from_version( 360 chrome_version, cros_version)) 361 362 def test_returns_none_when_given_invalid_input(self): 363 """Checks the return value when invalid input is given.""" 364 chrome_version = '27.0' 365 cros_version = '27.3906.0.0' 366 self.assertIsNone(perf_uploader._get_id_from_version( 367 chrome_version, cros_version)) 368 369 370class test_get_version_numbers(unittest.TestCase): 371 """Tests for the _get_version_numbers function.""" 372 373 def test_with_valid_versions(self): 374 """Checks the version numbers used when data is formatted as expected.""" 375 self.assertEqual( 376 ('34.5678.9.0', '34.5.678.9'), 377 perf_uploader._get_version_numbers( 378 { 379 'CHROME_VERSION': '34.5.678.9', 380 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 381 })) 382 383 def test_with_missing_version_raises_error(self): 384 """Checks that an error is raised when a version is missing.""" 385 with self.assertRaises(perf_uploader.PerfUploadingError): 386 perf_uploader._get_version_numbers( 387 { 388 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 389 }) 390 391 def test_with_unexpected_version_format_raises_error(self): 392 """Checks that an error is raised when there's a rN suffix.""" 393 with self.assertRaises(perf_uploader.PerfUploadingError): 394 perf_uploader._get_version_numbers( 395 { 396 'CHROME_VERSION': '34.5.678.9', 397 'CHROMEOS_RELEASE_VERSION': '5678.9.0r1', 398 }) 399 400 def test_with_valid_release_milestone(self): 401 """Checks the version numbers used when data is formatted as expected.""" 402 self.assertEqual( 403 ('54.5678.9.0', '34.5.678.9'), 404 perf_uploader._get_version_numbers( 405 { 406 'CHROME_VERSION': '34.5.678.9', 407 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 408 'CHROMEOS_RELEASE_CHROME_MILESTONE': '54', 409 })) 410 411 412class test_format_for_upload(unittest.TestCase): 413 """Tests for the format_for_upload function.""" 414 415 _PERF_DATA = { 416 "charts": { 417 "metric1": { 418 "summary": { 419 "improvement_direction": "down", 420 "type": "scalar", 421 "units": "msec", 422 "value": 2.7, 423 } 424 }, 425 "metric2": { 426 "summary": { 427 "improvement_direction": "up", 428 "type": "scalar", 429 "units": "frames_per_sec", 430 "value": 101.35, 431 } 432 } 433 }, 434 } 435 _PRESENT_INFO = { 436 'master_name': 'new_master_name', 437 'test_name': 'new_test_name', 438 } 439 440 def setUp(self): 441 self._perf_data = self._PERF_DATA 442 443 def _verify_result_string(self, actual_result, expected_result): 444 """Verifies a JSON string matches the expected result. 445 446 This function compares JSON objects rather than strings, because of 447 possible floating-point values that need to be compared using 448 assertAlmostEqual(). 449 450 @param actual_result: The candidate JSON string. 451 @param expected_result: The reference JSON string that the candidate 452 must match. 453 454 """ 455 actual = json.loads(actual_result) 456 expected = json.loads(expected_result) 457 458 def ordered(obj): 459 """Return the sorted obj.""" 460 if isinstance(obj, dict): 461 return sorted((k, ordered(v)) for k, v in obj.items()) 462 if isinstance(obj, list): 463 return sorted(ordered(x) for x in obj) 464 else: 465 return obj 466 fail_msg = 'Unexpected result string: %s' % actual_result 467 self.assertEqual(ordered(expected), ordered(actual), msg=fail_msg) 468 469 470 def test_format_for_upload(self): 471 """Verifies format_for_upload generates correct json data.""" 472 result = perf_uploader._format_for_upload( 473 'platform', '25.1200.0.0', '25.10.1000.0', 'WINKY E2A-F2K-Q35', 474 'test_machine', self._perf_data, self._PRESENT_INFO, 475 '52926644-username/hostname') 476 expected_result_string = ( 477 '{"versions": {' 478 '"cros_version": "25.1200.0.0",' 479 '"chrome_version": "25.10.1000.0"' 480 '},' 481 '"point_id": 10000000120000000,' 482 '"bot": "cros-platform",' 483 '"chart_data": {' 484 '"charts": {' 485 '"metric2": {' 486 '"summary": {' 487 '"units": "frames_per_sec",' 488 '"type": "scalar",' 489 '"value": 101.35,' 490 '"improvement_direction": "up"' 491 '}' 492 '},' 493 '"metric1": {' 494 '"summary": {' 495 '"units": "msec",' 496 '"type": "scalar",' 497 '"value": 2.7,' 498 '"improvement_direction": "down"}' 499 '}' 500 '}' 501 '},' 502 '"master": "new_master_name",' 503 '"supplemental": {' 504 '"hardware_identifier": "WINKY E2A-F2K-Q35",' 505 '"jobname": "52926644-username/hostname",' 506 '"hardware_hostname": "test_machine",' 507 '"default_rev": "r_cros_version"}' 508 '}') 509 self._verify_result_string(result['data'], expected_result_string) 510 511 512if __name__ == '__main__': 513 unittest.main() 514