• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2import json
3import optparse
4import os
5import sys
6
7from webkitpy.common.host import Host
8from webkitpy.layout_tests.port import platform_options, configuration_options
9
10
11def main(argv):
12    parser = optparse.OptionParser(usage='%prog [path-to-results.json]')
13    parser.add_option('--failures', action='store_true',
14                      help='show failing tests')
15    parser.add_option('--flakes', action='store_true',
16                      help='show flaky tests')
17    parser.add_option('--expected', action='store_true',
18                      help='include expected results along with unexpected')
19    parser.add_option('--passes', action='store_true',
20                      help='show passing tests')
21    parser.add_option('--ignored-failures-path', action='store',
22                      help='ignore failures seen in a previous run')
23    parser.add_options(platform_options())
24    parser.add_options(configuration_options())
25    options, args = parser.parse_args(argv)
26
27    host = Host()
28    if args:
29        if args[0] == '-':
30            txt = sys.stdin.read()
31        elif os.path.exists(args[0]):
32            with open(args[0], 'r') as fp:
33                txt = fp.read()
34        else:
35            print >> sys.stderr, "file not found: %s" % args[0]
36            sys.exit(1)
37    else:
38        txt = host.filesystem.read_text_file(host.filesystem.join(host.port_factory.get(options=options).results_directory(), 'full_results.json'))
39
40    if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
41        txt = txt[12:-2]  # ignore optional JSONP wrapper
42    results = json.loads(txt)
43
44    passes, failures, flakes = decode_results(results, options.expected)
45
46    tests_to_print = []
47    if options.passes:
48        tests_to_print += passes.keys()
49    if options.failures:
50        tests_to_print += failures.keys()
51    if options.flakes:
52        tests_to_print += flakes.keys()
53    print "\n".join(sorted(tests_to_print))
54
55    if options.ignored_failures_path:
56        with open(options.ignored_failures_path, 'r') as fp:
57            txt = fp.read()
58        if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
59            txt = txt[12:-2]  # ignore optional JSONP wrapper
60        results = json.loads(txt)
61        _, ignored_failures, _ = decode_results(results, options.expected)
62        new_failures = set(failures.keys()) - set(ignored_failures.keys())
63        if new_failures:
64            print "New failures:"
65            print "\n".join(sorted(new_failures))
66            print
67        if ignored_failures:
68            print "Ignored failures:"
69            print "\n".join(sorted(ignored_failures.keys()))
70        if new_failures:
71            return 1
72        return 0
73
74
75def decode_results(results, include_expected=False):
76    tests = convert_trie_to_flat_paths(results['tests'])
77    failures = {}
78    flakes = {}
79    passes = {}
80    for (test, result) in tests.iteritems():
81        if include_expected or result.get('is_unexpected'):
82            actual_results = result['actual'].split()
83            expected_results = result['expected'].split()
84            if len(actual_results) > 1:
85                if actual_results[1] in expected_results:
86                    flakes[test] = actual_results[0]
87                else:
88                    # We report the first failure type back, even if the second
89                    # was more severe.
90                    failures[test] = actual_results[0]
91            elif actual_results[0] == 'PASS':
92                passes[test] = result
93            else:
94                failures[test] = actual_results[0]
95
96    return (passes, failures, flakes)
97
98
99def convert_trie_to_flat_paths(trie, prefix=None):
100    # Cloned from webkitpy.layout_tests.layout_package.json_results_generator
101    # so that this code can stand alone.
102    result = {}
103    for name, data in trie.iteritems():
104        if prefix:
105            name = prefix + "/" + name
106
107        if len(data) and not "actual" in data and not "expected" in data:
108            result.update(convert_trie_to_flat_paths(data, name))
109        else:
110            result[name] = data
111
112    return result
113
114
115if __name__ ==  '__main__':
116    sys.exit(main(sys.argv[1:]))
117