• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# Copyright 2016 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# pylint: disable-msg=W0311
7
8import argparse
9import json
10import os
11
12
13gpu_list = [
14    #'pinetrail',
15    'sandybridge',
16    'ivybridge',
17    'baytrail',
18    'haswell',
19    'broadwell',
20    'braswell',
21    'skylake',
22    'broxton',
23    'mali-t604',
24    'mali-t628',
25    'mali-t760',
26    'mali-t860',
27    'rogue',
28    'tegra',
29]
30
31_PROBLEM_STATUS = ['Fail', 'Flaky']
32_UNKNOWN_STATUS = ['NotSupported', 'Skipped', 'Unknown', None]
33
34status_dict = {
35    'Fail': 'FAIL ',
36    'Flaky': 'flaky',
37    'Pass': '  +  ',
38    'NotSupported': ' --- ',
39    'Skipped': ' === ',
40    'QualityWarning': 'qw   ',
41    'CompatibilityWarning': 'cw   ',
42    'Unknown': ' ??? ',
43    None: ' n/a ',
44}
45
46def load_expectation_dict(json_file):
47  data = {}
48  if os.path.isfile(json_file):
49    with open(json_file, 'r') as f:
50      text = f.read()
51      data = json.loads(text)
52  return data
53
54
55def load_expectations(json_file):
56  data = load_expectation_dict(json_file)
57  expectations = {}
58  # Convert from dictionary of lists to dictionary of sets.
59  for key in data:
60    expectations[key] = set(data[key])
61  return expectations
62
63
64def get_problem_count(dict, gpu):
65  if gpu in dict:
66    if not dict[gpu]:
67      return None
68    count = 0
69    for status in dict[gpu]:
70      if status not in _UNKNOWN_STATUS:
71        count = count + len((dict[gpu])[status])
72    # If every result has an unknown status then don't return a count.
73    if count < 1:
74      return None
75    count = 0
76    # Return counts of truly problematic statuses.
77    for status in _PROBLEM_STATUS:
78      if status in dict[gpu]:
79        count = count + len((dict[gpu])[status])
80  else:
81    print 'Warning: %s not found in dict!' % gpu
82  return count
83
84
85def get_problem_tests(dict):
86  tests = set([])
87  for gpu in dict:
88    for status in _PROBLEM_STATUS:
89      if status in dict[gpu]:
90        tests = tests.union((dict[gpu])[status])
91  return sorted(list(tests))
92
93
94def get_test_result(dict, test):
95  for key in dict:
96    if test in dict[key]:
97      return key
98  return None
99
100
101argparser = argparse.ArgumentParser(
102    description='Create a matrix of failing tests per GPU.')
103argparser.add_argument('interface',
104                       default='gles2',
105                       help='Interface for matrix (gles2, gles3, gles31).')
106args = argparser.parse_args()
107status = '%s-master.txt.json' % args.interface
108
109dict = {}
110for gpu in gpu_list:
111  filename = 'expectations/%s/%s' % (gpu, status)
112  dict[gpu] = load_expectations(filename)
113
114tests = get_problem_tests(dict)
115
116print 'Legend:'
117for key in status_dict:
118  print '%s  -->  %s' % (status_dict[key], key)
119print
120
121offset = ''
122for gpu in gpu_list:
123  print '%s%s' % (offset, gpu)
124  offset = '%s    |    ' % offset
125print offset
126
127text_count = ''
128text_del = ''
129for gpu in gpu_list:
130  problem_count = get_problem_count(dict, gpu)
131  if problem_count is None:
132    text_count = '%s  %s  ' % (text_count, status_dict[problem_count])
133  else:
134    text_count = '%s%5d    ' % (text_count, problem_count)
135  text_del = '%s=========' % text_del
136text_count = '%s  Total failure count (Fail + Flaky)' % text_count
137print text_del
138print text_count
139print text_del
140
141for test in tests:
142  text = ''
143  for gpu in gpu_list:
144    result = get_test_result(dict[gpu], test)
145    status = status_dict[result]
146    text = '%s  %s  ' % (text, status)
147  text = '%s  %s' % (text, test)
148  print text
149
150print text_del
151print '%s repeated' % text_count
152print text_del
153
154