1#!/usr/bin/env vpython3 2# Copyright 2020 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from typing import Iterable, Optional 7import unittest 8from unittest import mock 9 10from unexpected_passes_common import builders 11from unexpected_passes_common import constants 12from unexpected_passes_common import data_types 13from unexpected_passes_common import expectations 14from unexpected_passes_common import queries 15from unexpected_passes_common import unittest_utils as uu 16 17# Protected access is allowed for unittests. 18# pylint: disable=protected-access 19 20class HelperMethodUnittest(unittest.TestCase): 21 def testStripPrefixFromBuildIdValidId(self) -> None: 22 self.assertEqual(queries._StripPrefixFromBuildId('build-1'), '1') 23 24 def testStripPrefixFromBuildIdInvalidId(self) -> None: 25 with self.assertRaises(AssertionError): 26 queries._StripPrefixFromBuildId('build1') 27 with self.assertRaises(AssertionError): 28 queries._StripPrefixFromBuildId('build-1-2') 29 30 def testConvertActualResultToExpectationFileFormatAbort(self) -> None: 31 self.assertEqual( 32 queries._ConvertActualResultToExpectationFileFormat('ABORT'), 'Timeout') 33 34 35class BigQueryQuerierInitUnittest(unittest.TestCase): 36 37 def testInvalidNumSamples(self): 38 """Tests that the number of samples is validated.""" 39 with self.assertRaises(AssertionError): 40 uu.CreateGenericQuerier(num_samples=-1) 41 42 def testDefaultSamples(self): 43 """Tests that the number of samples is set to a default if not provided.""" 44 querier = uu.CreateGenericQuerier(num_samples=0) 45 self.assertGreater(querier._num_samples, 0) 46 47 48class GetBuilderGroupedQueryResultsUnittest(unittest.TestCase): 49 50 def setUp(self): 51 builders.ClearInstance() 52 expectations.ClearInstance() 53 uu.RegisterGenericBuildersImplementation() 54 uu.RegisterGenericExpectationsImplementation() 55 self._querier = uu.CreateGenericQuerier() 56 57 def testUnknownBuilderType(self): 58 """Tests behavior when an unknown builder type is provided.""" 59 with self.assertRaisesRegex(RuntimeError, 'Unknown builder type unknown'): 60 for _ in self._querier.GetBuilderGroupedQueryResults('unknown', False): 61 pass 62 63 def testQueryRouting(self): 64 """Tests that the correct query is used based on inputs.""" 65 with mock.patch.object(self._querier, 66 '_GetPublicCiQuery', 67 return_value='public_ci') as public_ci_mock: 68 with mock.patch.object(self._querier, 69 '_GetInternalCiQuery', 70 return_value='internal_ci') as internal_ci_mock: 71 with mock.patch.object(self._querier, 72 '_GetPublicTryQuery', 73 return_value='public_try') as public_try_mock: 74 with mock.patch.object( 75 self._querier, 76 '_GetInternalTryQuery', 77 return_value='internal_try') as internal_try_mock: 78 all_mocks = [ 79 public_ci_mock, 80 internal_ci_mock, 81 public_try_mock, 82 internal_try_mock, 83 ] 84 inputs = [ 85 (constants.BuilderTypes.CI, False, public_ci_mock), 86 (constants.BuilderTypes.CI, True, internal_ci_mock), 87 (constants.BuilderTypes.TRY, False, public_try_mock), 88 (constants.BuilderTypes.TRY, True, internal_try_mock), 89 ] 90 for builder_type, internal_status, called_mock in inputs: 91 for _ in self._querier.GetBuilderGroupedQueryResults( 92 builder_type, internal_status): 93 pass 94 for m in all_mocks: 95 if m == called_mock: 96 m.assert_called_once() 97 else: 98 m.assert_not_called() 99 for m in all_mocks: 100 m.reset_mock() 101 102 def testNoResults(self): 103 """Tests functionality if the query returns no results.""" 104 returned_builders = [] 105 with self.assertLogs(level='WARNING') as log_manager: 106 with mock.patch.object(self._querier, 107 '_GetPublicCiQuery', 108 return_value=''): 109 for builder_name, _, _ in self._querier.GetBuilderGroupedQueryResults( 110 constants.BuilderTypes.CI, False): 111 returned_builders.append(builder_name) 112 for message in log_manager.output: 113 if ('Did not get any results for builder type ci and internal status ' 114 'False. Depending on where tests are run and how frequently ' 115 'trybots are used for submission, this may be benign') in message: 116 break 117 else: 118 self.fail('Did not find expected log message: %s' % log_manager.output) 119 self.assertEqual(len(returned_builders), 0) 120 121 def testHappyPath(self): 122 """Tests functionality in the happy path.""" 123 self._querier.query_results = [ 124 uu.FakeQueryResult(builder_name='builder_a', 125 id_='build-a', 126 test_id='test_a', 127 status='PASS', 128 typ_tags=['linux', 'unknown_tag'], 129 step_name='step_a'), 130 uu.FakeQueryResult(builder_name='builder_b', 131 id_='build-b', 132 test_id='test_b', 133 status='FAIL', 134 typ_tags=['win'], 135 step_name='step_b'), 136 ] 137 138 expected_results = [ 139 ('builder_a', 140 [data_types.BaseResult('test_a', ('linux', ), 'Pass', 'step_a', 141 'a')], None), 142 ('builder_b', 143 [data_types.BaseResult('test_b', ('win', ), 'Failure', 'step_b', 144 'b')], None), 145 ] 146 147 results = [] 148 with mock.patch.object(self._querier, '_GetPublicCiQuery', return_value=''): 149 for builder_name, result_list, expectation_files in ( 150 self._querier.GetBuilderGroupedQueryResults(constants.BuilderTypes.CI, 151 False)): 152 results.append((builder_name, result_list, expectation_files)) 153 154 self.assertEqual(results, expected_results) 155 156 def testHappyPathWithExpectationFiles(self): 157 """Tests functionality in the happy path with expectation files provided.""" 158 self._querier.query_results = [ 159 uu.FakeQueryResult(builder_name='builder_a', 160 id_='build-a', 161 test_id='test_a', 162 status='PASS', 163 typ_tags=['linux', 'unknown_tag'], 164 step_name='step_a'), 165 uu.FakeQueryResult(builder_name='builder_b', 166 id_='build-b', 167 test_id='test_b', 168 status='FAIL', 169 typ_tags=['win'], 170 step_name='step_b'), 171 ] 172 173 expected_results = [ 174 ('builder_a', 175 [data_types.BaseResult('test_a', ('linux', ), 'Pass', 'step_a', 176 'a')], list(set(['ef_a']))), 177 ('builder_b', 178 [data_types.BaseResult('test_b', ('win', ), 'Failure', 'step_b', 179 'b')], list(set(['ef_b', 'ef_c']))), 180 ] 181 182 results = [] 183 with mock.patch.object(self._querier, 184 '_GetRelevantExpectationFilesForQueryResult', 185 side_effect=(['ef_a'], ['ef_b', 'ef_c'])): 186 with mock.patch.object(self._querier, 187 '_GetPublicCiQuery', 188 return_value=''): 189 for builder_name, result_list, expectation_files in ( 190 self._querier.GetBuilderGroupedQueryResults( 191 constants.BuilderTypes.CI, False)): 192 results.append((builder_name, result_list, expectation_files)) 193 194 self.assertEqual(results, expected_results) 195 196 197class FillExpectationMapForBuildersUnittest(unittest.TestCase): 198 def setUp(self) -> None: 199 self._querier = uu.CreateGenericQuerier() 200 201 expectations.ClearInstance() 202 uu.RegisterGenericExpectationsImplementation() 203 204 def testErrorOnMixedBuilders(self) -> None: 205 """Tests that providing builders of mixed type is an error.""" 206 builders_to_fill = [ 207 data_types.BuilderEntry('ci_builder', constants.BuilderTypes.CI, False), 208 data_types.BuilderEntry('try_builder', constants.BuilderTypes.TRY, 209 False) 210 ] 211 with self.assertRaises(AssertionError): 212 self._querier.FillExpectationMapForBuilders( 213 data_types.TestExpectationMap({}), builders_to_fill) 214 215 def _runValidResultsTest(self, keep_unmatched_results: bool) -> None: 216 self._querier = uu.CreateGenericQuerier( 217 keep_unmatched_results=keep_unmatched_results) 218 219 public_results = [ 220 uu.FakeQueryResult(builder_name='matched_builder', 221 id_='build-build_id', 222 test_id='foo', 223 status='PASS', 224 typ_tags=['win'], 225 step_name='step_name'), 226 uu.FakeQueryResult(builder_name='unmatched_builder', 227 id_='build-build_id', 228 test_id='bar', 229 status='PASS', 230 typ_tags=[], 231 step_name='step_name'), 232 uu.FakeQueryResult(builder_name='extra_builder', 233 id_='build-build_id', 234 test_id='foo', 235 status='PASS', 236 typ_tags=['win'], 237 step_name='step_name'), 238 ] 239 240 internal_results = [ 241 uu.FakeQueryResult(builder_name='matched_internal', 242 id_='build-build_id', 243 test_id='foo', 244 status='PASS', 245 typ_tags=['win'], 246 step_name='step_name_internal'), 247 uu.FakeQueryResult(builder_name='unmatched_internal', 248 id_='build-build_id', 249 test_id='bar', 250 status='PASS', 251 typ_tags=[], 252 step_name='step_name_internal'), 253 ] 254 255 builders_to_fill = [ 256 data_types.BuilderEntry('matched_builder', constants.BuilderTypes.CI, 257 False), 258 data_types.BuilderEntry('unmatched_builder', constants.BuilderTypes.CI, 259 False), 260 data_types.BuilderEntry('matched_internal', constants.BuilderTypes.CI, 261 True), 262 data_types.BuilderEntry('unmatched_internal', constants.BuilderTypes.CI, 263 True), 264 ] 265 266 expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure') 267 expectation_map = data_types.TestExpectationMap({ 268 'foo': 269 data_types.ExpectationBuilderMap({ 270 expectation: 271 data_types.BuilderStepMap(), 272 }), 273 }) 274 275 def PublicSideEffect(): 276 self._querier.query_results = public_results 277 return '' 278 279 def InternalSideEffect(): 280 self._querier.query_results = internal_results 281 return '' 282 283 with self.assertLogs(level='WARNING') as log_manager: 284 with mock.patch.object(self._querier, 285 '_GetPublicCiQuery', 286 side_effect=PublicSideEffect) as public_mock: 287 with mock.patch.object(self._querier, 288 '_GetInternalCiQuery', 289 side_effect=InternalSideEffect) as internal_mock: 290 unmatched_results = self._querier.FillExpectationMapForBuilders( 291 expectation_map, builders_to_fill) 292 public_mock.assert_called_once() 293 internal_mock.assert_called_once() 294 295 for message in log_manager.output: 296 if ('Did not find a matching builder for name extra_builder and ' 297 'internal status False. This is normal if the builder is no longer ' 298 'running tests (e.g. it was experimental).') in message: 299 break 300 else: 301 self.fail('Did not find expected log message') 302 303 stats = data_types.BuildStats() 304 stats.AddPassedBuild(frozenset(['win'])) 305 expected_expectation_map = { 306 'foo': { 307 expectation: { 308 'chromium/ci:matched_builder': { 309 'step_name': stats, 310 }, 311 'chrome/ci:matched_internal': { 312 'step_name_internal': stats, 313 }, 314 }, 315 }, 316 } 317 self.assertEqual(expectation_map, expected_expectation_map) 318 if keep_unmatched_results: 319 self.assertEqual( 320 unmatched_results, { 321 'chromium/ci:unmatched_builder': [ 322 data_types.Result('bar', [], 'Pass', 'step_name', 'build_id'), 323 ], 324 'chrome/ci:unmatched_internal': [ 325 data_types.Result('bar', [], 'Pass', 'step_name_internal', 326 'build_id'), 327 ], 328 }) 329 else: 330 self.assertEqual(unmatched_results, {}) 331 332 def testValidResultsKeepUnmatched(self) -> None: 333 """Tests behavior w/ valid results and keeping unmatched results.""" 334 self._runValidResultsTest(True) 335 336 def testValidResultsDoNotKeepUnmatched(self) -> None: 337 """Tests behavior w/ valid results and not keeping unmatched results.""" 338 self._runValidResultsTest(False) 339 340 341class ProcessRowsForBuilderUnittest(unittest.TestCase): 342 343 def setUp(self): 344 self._querier = uu.CreateGenericQuerier() 345 346 def testHappyPathWithExpectationFiles(self): 347 """Tests functionality along the happy path with expectation files.""" 348 349 def SideEffect(row: queries.QueryResult) -> Optional[Iterable[str]]: 350 if row.step_name == 'step_a1': 351 return ['ef_a1'] 352 if row.step_name == 'step_a2': 353 return ['ef_a2'] 354 if row.step_name == 'step_b': 355 return ['ef_b1', 'ef_b2'] 356 raise RuntimeError('Unexpected row') 357 358 rows = [ 359 uu.FakeQueryResult(builder_name='unused', 360 id_='build-a', 361 test_id='test_a', 362 status='PASS', 363 typ_tags=['linux', 'unknown_tag'], 364 step_name='step_a1'), 365 uu.FakeQueryResult(builder_name='unused', 366 id_='build-a', 367 test_id='test_a', 368 status='FAIL', 369 typ_tags=['linux', 'unknown_tag'], 370 step_name='step_a2'), 371 uu.FakeQueryResult(builder_name='unused', 372 id_='build-b', 373 test_id='test_b', 374 status='FAIL', 375 typ_tags=['win'], 376 step_name='step_b'), 377 ] 378 379 # Reversed order is expected since results are popped. 380 expected_results = [ 381 data_types.BaseResult(test='test_b', 382 tags=['win'], 383 actual_result='Failure', 384 step='step_b', 385 build_id='b'), 386 data_types.BaseResult(test='test_a', 387 tags=['linux'], 388 actual_result='Failure', 389 step='step_a2', 390 build_id='a'), 391 data_types.BaseResult(test='test_a', 392 tags=['linux'], 393 actual_result='Pass', 394 step='step_a1', 395 build_id='a'), 396 ] 397 398 with mock.patch.object(self._querier, 399 '_GetRelevantExpectationFilesForQueryResult', 400 side_effect=SideEffect): 401 results, expectation_files = self._querier._ProcessRowsForBuilder(rows) 402 self.assertEqual(results, expected_results) 403 self.assertEqual(len(expectation_files), len(set(expectation_files))) 404 self.assertEqual(set(expectation_files), 405 set(['ef_a1', 'ef_a2', 'ef_b1', 'ef_b2'])) 406 407 def testHappyPathNoneExpectation(self): 408 """Tests functionality along the happy path with a None expectation file.""" 409 410 # A single None expectation file should cause the resulting return value to 411 # become None. 412 def SideEffect(row: queries.QueryResult) -> Optional[Iterable[str]]: 413 if row.step_name == 'step_a1': 414 return ['ef_a1'] 415 if row.step_name == 'step_a2': 416 return ['ef_a2'] 417 return None 418 419 rows = [ 420 uu.FakeQueryResult(builder_name='unused', 421 id_='build-a', 422 test_id='test_a', 423 status='PASS', 424 typ_tags=['linux', 'unknown_tag'], 425 step_name='step_a1'), 426 uu.FakeQueryResult(builder_name='unused', 427 id_='build-a', 428 test_id='test_a', 429 status='FAIL', 430 typ_tags=['linux', 'unknown_tag'], 431 step_name='step_a2'), 432 uu.FakeQueryResult(builder_name='unused', 433 id_='build-b', 434 test_id='test_b', 435 status='FAIL', 436 typ_tags=['win'], 437 step_name='step_b'), 438 ] 439 440 # Reversed order is expected since results are popped. 441 expected_results = [ 442 data_types.BaseResult(test='test_b', 443 tags=['win'], 444 actual_result='Failure', 445 step='step_b', 446 build_id='b'), 447 data_types.BaseResult(test='test_a', 448 tags=['linux'], 449 actual_result='Failure', 450 step='step_a2', 451 build_id='a'), 452 data_types.BaseResult(test='test_a', 453 tags=['linux'], 454 actual_result='Pass', 455 step='step_a1', 456 build_id='a'), 457 ] 458 459 with mock.patch.object(self._querier, 460 '_GetRelevantExpectationFilesForQueryResult', 461 side_effect=SideEffect): 462 results, expectation_files = self._querier._ProcessRowsForBuilder(rows) 463 self.assertEqual(results, expected_results) 464 self.assertEqual(expectation_files, None) 465 466 def testHappyPathSkippedResult(self): 467 """Tests functionality along the happy path with a skipped result.""" 468 469 def SideEffect(row: queries.QueryResult) -> bool: 470 if row.step_name == 'step_b': 471 return True 472 return False 473 474 rows = [ 475 uu.FakeQueryResult(builder_name='unused', 476 id_='build-a', 477 test_id='test_a', 478 status='PASS', 479 typ_tags=['linux', 'unknown_tag'], 480 step_name='step_a1'), 481 uu.FakeQueryResult(builder_name='unused', 482 id_='build-a', 483 test_id='test_a', 484 status='FAIL', 485 typ_tags=['linux', 'unknown_tag'], 486 step_name='step_a2'), 487 uu.FakeQueryResult(builder_name='unused', 488 id_='build-b', 489 test_id='test_b', 490 status='FAIL', 491 typ_tags=['win'], 492 step_name='step_b'), 493 ] 494 495 # Reversed order is expected since results are popped. 496 expected_results = [ 497 data_types.BaseResult(test='test_a', 498 tags=['linux'], 499 actual_result='Failure', 500 step='step_a2', 501 build_id='a'), 502 data_types.BaseResult(test='test_a', 503 tags=['linux'], 504 actual_result='Pass', 505 step='step_a1', 506 build_id='a'), 507 ] 508 509 with mock.patch.object(self._querier, 510 '_ShouldSkipOverResult', 511 side_effect=SideEffect): 512 results, expectation_files = self._querier._ProcessRowsForBuilder(rows) 513 self.assertEqual(results, expected_results) 514 self.assertEqual(expectation_files, None) 515 516 517if __name__ == '__main__': 518 unittest.main(verbosity=2) 519