1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Common file shared by test_push of autotest and skylab. 6 7autotest: site_utils/test_push.py 8skylab: venv/skylab_staging/test_push.py 9""" 10 11import collections 12import re 13 14# Dictionary of test results keyed by test name regular expression. 15EXPECTED_TEST_RESULTS = {'^SERVER_JOB$': 'GOOD', 16 # This is related to dummy_Fail/control.dependency. 17 'dummy_Fail.dependency$': 'TEST_NA', 18 'login_LoginSuccess.*': 'GOOD', 19 'provision_AutoUpdate.double': 'GOOD', 20 'dummy_Pass.*': 'GOOD', 21 'dummy_Fail.Fail$': 'FAIL', 22 'dummy_Fail.Error$': 'ERROR', 23 'dummy_Fail.Warn$': 'WARN', 24 'dummy_Fail.NAError$': 'TEST_NA', 25 'dummy_Fail.Crash$': 'GOOD', 26 'autotest_SyncCount$': 'GOOD', 27 } 28 29EXPECTED_TEST_RESULTS_DUMMY = {'^SERVER_JOB$': 'GOOD', 30 'dummy_Pass.*': 'GOOD', 31 'dummy_Fail.Fail': 'FAIL', 32 'dummy_Fail.Warn': 'WARN', 33 'dummy_Fail.Crash': 'GOOD', 34 'dummy_Fail.Error': 'ERROR', 35 'dummy_Fail.NAError': 'TEST_NA',} 36 37EXPECTED_TEST_RESULTS_POWERWASH = {'platform_Powerwash': 'GOOD', 38 'SERVER_JOB': 'GOOD'} 39 40_TestPushErrors = collections.namedtuple( 41 '_TestPushErrors', 42 [ 43 'mismatch_errors', 44 'unknown_tests', 45 'missing_tests', 46 ] 47) 48 49 50def summarize_push(test_views, expected_results, ignored_tests=[]): 51 """Summarize the test push errors.""" 52 test_push_errors = _match_test_results(test_views, expected_results, 53 ignored_tests) 54 return _generate_push_summary(test_push_errors) 55 56 57def _match_test_results(test_views, expected_results, ignored_tests): 58 """Match test results with expected results. 59 60 @param test_views: A dictionary of test status keyed by test name, e.g., 61 {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'} 62 @param expected_results: A dictionary of test name to expected test result. 63 64 @return: A _TestPushErrors tuple. 65 """ 66 mismatch_errors = [] 67 unknown_tests = [] 68 found_keys = set() 69 for test_name, test_status in test_views.iteritems(): 70 test_found = False 71 for test_name_pattern, expected_result in expected_results.items(): 72 if re.search(test_name_pattern, test_name): 73 test_found = True 74 found_keys.add(test_name_pattern) 75 if (expected_result != test_status and 76 _is_significant(test_name, ignored_tests)): 77 error = ('%s Expected: [%s], Actual: [%s]' % 78 (test_name, expected_result, test_status)) 79 mismatch_errors.append(error) 80 81 if not test_found and _is_significant(test_name, ignored_tests): 82 unknown_tests.append(test_name) 83 84 missing_tests = set(expected_results.keys()) - found_keys 85 missing_tests = [t for t in missing_tests 86 if _is_significant(t, ignored_tests)] 87 return _TestPushErrors(mismatch_errors=mismatch_errors, 88 unknown_tests=unknown_tests, 89 missing_tests=missing_tests) 90 91 92def _is_significant(test, ignored_tests_patterns): 93 return all([test not in m for m in ignored_tests_patterns]) 94 95 96def _generate_push_summary(test_push_errors): 97 """Generate a list of summary based on the test_push results.""" 98 summary = [] 99 if test_push_errors.mismatch_errors: 100 summary.append(('Results of %d test(s) do not match expected ' 101 'values:') % len(test_push_errors.mismatch_errors)) 102 summary.extend(test_push_errors.mismatch_errors) 103 summary.append('\n') 104 105 if test_push_errors.unknown_tests: 106 summary.append('%d test(s) are not expected to be run:' % 107 len(test_push_errors.unknown_tests)) 108 summary.extend(test_push_errors.unknown_tests) 109 summary.append('\n') 110 111 if test_push_errors.missing_tests: 112 summary.append('%d test(s) are missing from the results:' % 113 len(test_push_errors.missing_tests)) 114 summary.extend(test_push_errors.missing_tests) 115 summary.append('\n') 116 117 return summary 118