• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Lint as: python2, python3
2# Copyright 2018 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Common file shared by test_push of autotest and skylab.
7
8autotest: site_utils/test_push.py
9skylab: venv/skylab_staging/test_push.py
10"""
11
12from __future__ import absolute_import
13from __future__ import division
14from __future__ import print_function
15import collections
16import re
17import six
18
19# Dictionary of test results keyed by test name regular expression.
20EXPECTED_TEST_RESULTS = {'^SERVER_JOB$':                 ['GOOD'],
21                         # This is related to dummy_Fail/control.dependency.
22                         'dummy_Fail.dependency$':       ['TEST_NA'],
23                         'login_LoginSuccess.*':         ['GOOD'],
24                         'dummy_Pass.*':                 ['GOOD'],
25                         'dummy_Fail.Fail$':             ['FAIL'],
26                         'dummy_Fail.Error$':            ['ERROR'],
27                         'dummy_Fail.Warn$':             ['WARN'],
28                         'dummy_Fail.NAError$':          ['TEST_NA'],
29                         'dummy_Fail.Crash$':            ['GOOD'],
30                         }
31
32EXPECTED_TEST_RESULTS_DUMMY = {'^SERVER_JOB$':       ['GOOD'],
33                               'dummy_Pass.*':       ['GOOD'],
34                               'dummy_Fail.Fail':    ['FAIL'],
35                               'dummy_Fail.Warn':    ['WARN'],
36                               'dummy_Fail.Crash':   ['GOOD'],
37                               'dummy_Fail.Error':   ['ERROR'],
38                               'dummy_Fail.NAError': ['TEST_NA'],
39                               }
40
41EXPECTED_TEST_RESULTS_POWERWASH = {'platform_Powerwash': ['GOOD'],
42                                   'SERVER_JOB':         ['GOOD'],
43                                   }
44
45_TestPushErrors = collections.namedtuple(
46        '_TestPushErrors',
47        [
48                'mismatch_errors',
49                'unknown_tests',
50                'missing_tests',
51        ]
52)
53
54
55def summarize_push(test_views, expected_results, ignored_tests=[]):
56    """Summarize the test push errors."""
57    test_push_errors = _match_test_results(test_views, expected_results,
58                                           ignored_tests)
59    return _generate_push_summary(test_push_errors)
60
61
62def _match_test_results(test_views, expected_results, ignored_tests):
63    """Match test results with expected results.
64
65    @param test_views: A defaultdict where keys are test names and values are
66                       lists of test statuses, e.g.,
67                       {'dummy_Fail.Error': ['ERROR', 'ERROR],
68                        'dummy_Fail.NAError': ['TEST_NA'],
69                        'dummy_Fail.RetrySuccess': ['ERROR', 'GOOD'],
70                        }
71    @param expected_results: A dictionary of test name to expected test result.
72                             Has the same format as test_views.
73    @param ignored_tests: A list of test name patterns. Any mismatch between
74                          test results and expected test results that matches
75                          one these patterns is ignored.
76
77    @return: A _TestPushErrors tuple.
78    """
79    mismatch_errors = []
80    unknown_tests = []
81    found_keys = set()
82    for test_name, test_status_list in six.iteritems(test_views):
83        test_found = False
84        for test_name_pattern, expected_result in expected_results.items():
85            if re.search(test_name_pattern, test_name):
86                test_found = True
87                found_keys.add(test_name_pattern)
88                if (sorted(expected_result) != sorted(test_status_list) and
89                    _is_significant(test_name, ignored_tests)):
90                    error = ('%s Expected: %s, Actual: %s' %
91                             (test_name, expected_result, test_status_list))
92                    mismatch_errors.append(error)
93
94        if not test_found and _is_significant(test_name, ignored_tests):
95            unknown_tests.append(test_name)
96
97    missing_tests = set(expected_results.keys()) - found_keys
98    missing_tests = [t for t in missing_tests
99                     if _is_significant(t, ignored_tests)]
100    return _TestPushErrors(mismatch_errors=mismatch_errors,
101                           unknown_tests=unknown_tests,
102                           missing_tests=missing_tests)
103
104
105def _is_significant(test, ignored_tests_patterns):
106    return all([test not in m for m in ignored_tests_patterns])
107
108
109def _generate_push_summary(test_push_errors):
110    """Generate a list of summary based on the test_push results."""
111    summary = []
112    if test_push_errors.mismatch_errors:
113        summary.append(('Results of %d test(s) do not match expected '
114                        'values:') % len(test_push_errors.mismatch_errors))
115        summary.extend(test_push_errors.mismatch_errors)
116        summary.append('\n')
117
118    if test_push_errors.unknown_tests:
119        summary.append('%d test(s) are not expected to be run:' %
120                       len(test_push_errors.unknown_tests))
121        summary.extend(test_push_errors.unknown_tests)
122        summary.append('\n')
123
124    if test_push_errors.missing_tests:
125        summary.append('%d test(s) are missing from the results:' %
126                       len(test_push_errors.missing_tests))
127        summary.extend(test_push_errors.missing_tests)
128        summary.append('\n')
129
130    return summary
131