• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# (C) Copyright IBM Corp. 2006
3# Author: Dustin Kirkland <dustin.kirkland@us.ibm.com>
4# Description:
5#  Input:  Two or more files containing results from different executions of
6#          the LTP. The input can either be file names or the url location
7#          of the ltp.results file.
8#  Output: A report on the following:
9#          - The total number of tests executed in each run
10#          - The testname, sequence number, and output of each run
11#            where the results of those runs differ
12#  Return:
13#          0 if all runs had identical results
14#          Non-zero if results differ, or bad input
15
16
17import sys, re
18from autotest_lib.client.common_lib import utils
19
20def usage():
21    """Show script help text."""
22    print "\nUsage: \n\
23ltp-diff results1 results2 ... locationN \n\
24Note: location[1,2,N] may be local files or URLs of LTP results\n"
25    sys.exit(1)
26
27def get_results(results_files):
28    """
29    Download the results if needed.
30    Return results of each run in a numerically-indexed dictionary
31    of dictionaries keyed on testnames.
32    Return dictionary keyed on unique testnames across all runs.
33
34    @param results_files: list of result-file names (or url locations).
35    """
36    r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:')
37    i = 0
38    runs = {}
39    testnames = {}
40    for file in results_files:
41        runs[i] = {}
42        try:
43            fh = utils.urlopen(file)
44            results = fh.readlines()
45            fh.close()
46        except:
47            print "ERROR: reading results resource [%s]" % (file)
48            usage()
49        for line in results:
50            try:
51                s = r.match(line)
52                testname = s.group(1)
53                status = s.group(2)
54                runs[i][testname] = status
55                testnames[testname] = 1
56            except:
57                pass
58        i += 1
59    return (runs, testnames)
60
61
62
63def compare_results(runs):
64    """
65    Loop through all testnames alpahbetically.
66    Print any testnames with differing results across runs.
67    Return 1 if any test results across runs differ.
68    Return 0 if all test results match.
69
70    @param runs: dict of test status from get_results().
71    """
72    rc = 0
73    print "LTP Test Results to Compare"
74    for i in range(len(runs)):
75        print "  Run[%d]: %d" % (i, len(runs[i].keys()))
76    print ""
77    header = 0
78    all_testnames = testnames.keys()
79    all_testnames.sort()
80    for testname in all_testnames:
81        differ = 0
82        for i in range(1,len(runs)):
83            # Must handle testcases that executed in one run
84            # but not another by setting status to "null"
85            if not runs[i].has_key(testname):
86                runs[i][testname] = "null"
87            if not runs[i-1].has_key(testname):
88                runs[i-1][testname] = "null"
89            # Check for the results inconsistencies
90            if runs[i][testname] != runs[i-1][testname]:
91                differ = 1
92        if differ:
93            if header == 0:
94                # Print the differences header only once
95                print "Tests with Inconsistent Results across Runs"
96                print "  %-35s:\t%s" % ("Testname,Sequence", "Run Results")
97                header = 1
98
99            # Print info if results differ
100            rc = 1
101            testname_cleaned = re.sub('\s+', ',', testname)
102            print "  %-35s:\t" % (testname_cleaned),
103            all_results = ""
104            for i in range(len(runs)):
105                all_results += runs[i][testname]
106                if i+1<len(runs):
107                    all_results += "/"
108            print all_results
109    if rc == 0:
110        print "All LTP results are identical"
111    return rc
112
113
114########
115# Main #
116########
117sys.argv.pop(0)
118if (len(sys.argv) < 2):
119    usage()
120(runs, testnames) = get_results(sys.argv)
121rc = compare_results(runs)
122sys.exit(rc)
123