• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Given a previous good compile narrow down miscompiles.
4# Expects two directories named "before" and "after" each containing a set of
5# assembly or object files where the "after" version is assumed to be broken.
6# You also have to provide a script called "link_test". It is called with a list
7# of files which should be linked together and result tested. "link_test" should
8# returns with exitcode 0 if the linking and testing succeeded.
9#
10# abtest.py operates by taking all files from the "before" directory and
11# in each step replacing one of them with a file from the "bad" directory.
12#
13# Additionally you can perform the same steps with a single .s file. In this
14# mode functions are identified by "# -- Begin FunctionName" and
15# "# -- End FunctionName" markers. The abtest.py then takes all functions from
16# the file in the "before" directory and replaces one function with the
17# corresponding function from the "bad" file in each step.
18#
19# Example usage to identify miscompiled files:
20#    1. Create a link_test script, make it executable. Simple Example:
21#          clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"
22#    2. Run the script to figure out which files are miscompiled:
23#       > ./abtest.py
24#       somefile.s: ok
25#       someotherfile.s: skipped: same content
26#       anotherfile.s: failed: './link_test' exitcode != 0
27#       ...
28# Example usage to identify miscompiled functions inside a file:
29#    3. First you have to mark begin and end of the functions.
30#       The script comes with some examples called mark_xxx.py.
31#       Unfortunately this is very specific to your environment and it is likely
32#       that you have to write a custom version for your environment.
33#       > for i in before/*.s after/*.s; do mark_xxx.py $i; done
34#    4. Run the tests on a single file (assuming before/file.s and
35#       after/file.s exist)
36#       > ./abtest.py file.s
37#       funcname1 [0/XX]: ok
38#       funcname2 [1/XX]: ok
39#       funcname3 [2/XX]: skipped: same content
40#       funcname4 [3/XX]: failed: './link_test' exitcode != 0
41#       ...
42from fnmatch import filter
43from sys import stderr
44import argparse
45import filecmp
46import os
47import subprocess
48import sys
49
50LINKTEST="./link_test"
51ESCAPE="\033[%sm"
52BOLD=ESCAPE % "1"
53RED=ESCAPE % "31"
54NORMAL=ESCAPE % "0"
55FAILED=RED+"failed"+NORMAL
56
57def find(dir, file_filter=None):
58    files = [walkdir[0]+"/"+file for walkdir in os.walk(dir) for file in walkdir[2]]
59    if file_filter != None:
60        files = filter(files, file_filter)
61    return files
62
63def error(message):
64    stderr.write("Error: %s\n" % (message,))
65
66def warn(message):
67    stderr.write("Warning: %s\n" % (message,))
68
69def extract_functions(file):
70    functions = []
71    in_function = None
72    for line in open(file):
73        if line.startswith("# -- Begin  "):
74            if in_function != None:
75                warn("Missing end of function %s" % (in_function,))
76            funcname = line[12:-1]
77            in_function = funcname
78            text = line
79        elif line.startswith("# -- End  "):
80            function_name = line[10:-1]
81            if in_function != function_name:
82                warn("End %s does not match begin %s" % (function_name, in_function))
83            else:
84                text += line
85                functions.append( (in_function, text) )
86            in_function = None
87        elif in_function != None:
88            text += line
89    return functions
90
91def replace_function(file, function, replacement, dest):
92    out = open(dest, "w")
93    skip = False
94    found = False
95    in_function = None
96    for line in open(file):
97        if line.startswith("# -- Begin  "):
98            if in_function != None:
99                warn("Missing end of function %s" % (in_function,))
100            funcname = line[12:-1]
101            in_function = funcname
102            if in_function == function:
103                out.write(replacement)
104                skip = True
105        elif line.startswith("# -- End  "):
106            function_name = line[10:-1]
107            if in_function != function_name:
108                warn("End %s does not match begin %s" % (function_name, in_function))
109            in_function = None
110            if skip:
111                skip = False
112                continue
113        if not skip:
114            out.write(line)
115
116def announce_test(name):
117    stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
118    stderr.flush()
119
120def announce_result(result, info):
121    stderr.write(result)
122    if info != "":
123        stderr.write(": %s" % info)
124    stderr.write("\n")
125    stderr.flush()
126
127def testrun(files):
128    linkline="%s %s" % (LINKTEST, " ".join(files),)
129    res = subprocess.call(linkline, shell=True)
130    if res != 0:
131        announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST)
132        return False
133    else:
134        announce_result("ok", "")
135        return True
136
137def check_files():
138    """Check files mode"""
139    for i in range(0, len(NO_PREFIX)):
140        f = NO_PREFIX[i]
141        b=baddir+"/"+f
142        if b not in BAD_FILES:
143            warn("There is no corresponding file to '%s' in %s" \
144                 % (gooddir+"/"+f, baddir))
145            continue
146
147        announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX)))
148
149        # combine files (everything from good except f)
150        testfiles=[]
151        skip=False
152        for c in NO_PREFIX:
153            badfile = baddir+"/"+c
154            goodfile = gooddir+"/"+c
155            if c == f:
156                testfiles.append(badfile)
157                if filecmp.cmp(goodfile, badfile):
158                    announce_result("skipped", "same content")
159                    skip = True
160                    break
161            else:
162                testfiles.append(goodfile)
163        if skip:
164            continue
165        testrun(testfiles)
166
167def check_functions_in_file(base, goodfile, badfile):
168    functions = extract_functions(goodfile)
169    if len(functions) == 0:
170        warn("Couldn't find any function in %s, missing annotations?" % (goodfile,))
171        return
172    badfunctions = dict(extract_functions(badfile))
173    if len(functions) == 0:
174        warn("Couldn't find any function in %s, missing annotations?" % (badfile,))
175        return
176
177    COMBINED="/tmp/combined.s"
178    i = 0
179    for (func,func_text) in functions:
180        announce_test(func + " [%s/%s]" % (i+1, len(functions)))
181        i+=1
182        if func not in badfunctions:
183            warn("Function '%s' missing from bad file" % func)
184            continue
185        if badfunctions[func] == func_text:
186            announce_result("skipped", "same content")
187            continue
188        replace_function(goodfile, func, badfunctions[func], COMBINED)
189        testfiles=[]
190        for c in NO_PREFIX:
191            if c == base:
192                testfiles.append(COMBINED)
193                continue
194            testfiles.append(gooddir + "/" + c)
195
196        testrun(testfiles)
197
198parser = argparse.ArgumentParser()
199parser.add_argument('--a', dest='dir_a', default='before')
200parser.add_argument('--b', dest='dir_b', default='after')
201parser.add_argument('--insane', help='Skip sanity check', action='store_true')
202parser.add_argument('file', metavar='file', nargs='?')
203config = parser.parse_args()
204
205gooddir=config.dir_a
206baddir=config.dir_b
207
208BAD_FILES=find(baddir, "*")
209GOOD_FILES=find(gooddir, "*")
210NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES])
211
212# "Checking whether build environment is sane ..."
213if not config.insane:
214    announce_test("sanity check")
215    if not os.access(LINKTEST, os.X_OK):
216        error("Expect '%s' to be present and executable" % (LINKTEST,))
217        exit(1)
218
219    res = testrun(GOOD_FILES)
220    if not res:
221        # "build environment is grinning and holding a spatula. Guess not."
222        linkline="%s %s" % (LINKTEST, " ".join(GOOD_FILES),)
223        stderr.write("\n%s\n\n" % linkline)
224        stderr.write("Returned with exitcode != 0\n")
225        sys.exit(1)
226
227if config.file is not None:
228    # File exchange mode
229    goodfile = gooddir+"/"+config.file
230    badfile = baddir+"/"+config.file
231    check_functions_in_file(config.file, goodfile, badfile)
232else:
233    # Function exchange mode
234    check_files()
235