• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python2
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4"""Script to  compare a baseline results file to a new results file."""
5
6from __future__ import print_function
7
8__author__ = 'raymes@google.com (Raymes Khoury)'
9
10import sys
11from cros_utils import logger
12from cros_utils import html_tools
13
14PASS = 'pass'
15FAIL = 'fail'
16NOT_EXECUTED = 'not executed'
17
18
19class ResultsReport(object):
20  """Class for holding report results."""
21
22  def __init__(self, report, num_tests_executed, num_passes, num_failures,
23               num_regressions):
24    self.report = report
25    self.num_tests_executed = num_tests_executed
26    self.num_passes = num_passes
27    self.num_failures = num_failures
28    self.num_regressions = num_regressions
29
30  def GetReport(self):
31    return self.report
32
33  def GetNumExecuted(self):
34    return self.num_tests_executed
35
36  def GetNumPasses(self):
37    return self.num_passes
38
39  def GetNumFailures(self):
40    return self.num_failures
41
42  def GetNumRegressions(self):
43    return self.num_regressions
44
45  def GetSummary(self):
46    summary = 'Tests executed: %s\n' % str(self.num_tests_executed)
47    summary += 'Tests Passing: %s\n' % str(self.num_passes)
48    summary += 'Tests Failing: %s\n' % str(self.num_failures)
49    summary += 'Regressions: %s\n' % str(self.num_regressions)
50    return summary
51
52
53def Usage():
54  print('Usage: %s baseline_results new_results' % sys.argv[0])
55  sys.exit(1)
56
57
58def ParseResults(results_filename):
59  results = []
60  try:
61    results_file = open(results_filename, 'rb')
62    for line in results_file:
63      if line.strip() != '':
64        results.append(line.strip().split('\t'))
65    results_file.close()
66  except IOError:
67    logger.GetLogger().LogWarning('Could not open results file: ' +
68                                  results_filename)
69  return results
70
71
72def GenerateResultsReport(baseline_file, new_result_file):
73  baseline_results = ParseResults(baseline_file)
74  new_results = ParseResults(new_result_file)
75
76  test_status = {}
77
78  for new_result in new_results:
79    new_test_name = new_result[0]
80    new_test_result = new_result[1]
81    test_status[new_test_name] = (new_test_result, NOT_EXECUTED)
82
83  for baseline_result in baseline_results:
84    baseline_test_name = baseline_result[0]
85    baseline_test_result = baseline_result[1]
86    if baseline_test_name in test_status:
87      new_test_result = test_status[baseline_test_name][0]
88      test_status[baseline_test_name] = (new_test_result, baseline_test_result)
89    else:
90      test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result)
91
92  regressions = []
93  for result in test_status.keys():
94    if test_status[result][0] != test_status[result][1]:
95      regressions.append(result)
96
97  num_tests_executed = len(new_results)
98  num_regressions = len(regressions)
99  num_passes = 0
100  num_failures = 0
101  for result in new_results:
102    if result[1] == PASS:
103      num_passes += 1
104    else:
105      num_failures += 1
106
107  report = html_tools.GetPageHeader('Test Summary')
108  report += html_tools.GetHeader('Test Summary')
109  report += html_tools.GetListHeader()
110  report += html_tools.GetListItem('Tests executed: ' + str(num_tests_executed))
111  report += html_tools.GetListItem('Passes: ' + str(num_passes))
112  report += html_tools.GetListItem('Failures: ' + str(num_failures))
113  report += html_tools.GetListItem('Regressions: ' + str(num_regressions))
114  report += html_tools.GetListFooter()
115  report += html_tools.GetHeader('Regressions', 2)
116  report += html_tools.GetTableHeader(['Test name', 'Expected result',
117                                       'Actual result'])
118
119  for regression in regressions:
120    report += html_tools.GetTableRow([regression[:150], test_status[regression][
121        1], test_status[regression][0]])
122    report += '\n'
123  report += html_tools.GetTableFooter()
124  report += html_tools.GetHeader('All Tests', 2)
125  report += html_tools.GetTableHeader(['Test name', 'Expected result',
126                                       'Actual result'])
127  for result in test_status.keys():
128    report += html_tools.GetTableRow([result[:150], test_status[result][1],
129                                      test_status[result][0]])
130    report += '\n'
131  report += html_tools.GetTableFooter()
132  report += html_tools.GetFooter()
133  return ResultsReport(report, num_tests_executed, num_passes, num_failures,
134                       num_regressions)
135
136
137def Main(argv):
138  if len(argv) < 2:
139    Usage()
140
141  print(GenerateResultsReport(argv[1], argv[2])[0])
142
143
144if __name__ == '__main__':
145  Main(sys.argv)
146