3 # Given a previous good compile narrow down miscompiles.
4 # Expects two directories named "before" and "after" each containing a set of
5 # assembly or object files where the "after" version is assumed to be broken.
6 # You also have to provide a script called "link_test". It is called with a list
7 # of files which should be linked together and result tested. "link_test" should
8 # returns with exitcode 0 if the linking and testing succeeded.
10 # abtest.py operates by taking all files from the "before" directory and
11 # in each step replacing one of them with a file from the "bad" directory.
13 # Additionally you can perform the same steps with a single .s file. In this
14 # mode functions are identified by " -- Begin function FunctionName" and
15 # " -- End function" markers. The abtest.py then takes all
16 # function from the file in the "before" directory and replaces one function
17 # with the corresponding function from the "bad" file in each step.
19 # Example usage to identify miscompiled files:
20 # 1. Create a link_test script, make it executable. Simple Example:
21 # clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"
22 # 2. Run the script to figure out which files are miscompiled:
25 # someotherfile.s: skipped: same content
26 # anotherfile.s: failed: './link_test' exitcode != 0
28 # Example usage to identify miscompiled functions inside a file:
29 # 3. Run the tests on a single file (assuming before/file.s and
31 # > ./abtest.py file.s
32 # funcname1 [0/XX]: ok
33 # funcname2 [1/XX]: ok
34 # funcname3 [2/XX]: skipped: same content
35 # funcname4 [3/XX]: failed: './link_test' exitcode != 0
37 from fnmatch import filter
38 from sys import stderr
45 LINKTEST="./link_test"
50 FAILED=RED+"failed"+NORMAL
52 def find(dir, file_filter=None):
53 files = [walkdir[0]+"/"+file for walkdir in os.walk(dir) for file in walkdir[2]]
54 if file_filter != None:
55 files = filter(files, file_filter)
59 stderr.write("Error: %s\n" % (message,))
62 stderr.write("Warning: %s\n" % (message,))
64 def extract_functions(file):
67 for line in open(file):
68 marker = line.find(" -- Begin function ")
70 if in_function != None:
71 warn("Missing end of function %s" % (in_function,))
72 funcname = line[marker + 19:-1]
73 in_function = funcname
77 marker = line.find(" -- End function")
80 functions.append( (in_function, text) )
84 if in_function != None:
88 def replace_function(file, function, replacement, dest):
93 for line in open(file):
94 marker = line.find(" -- Begin function ")
96 if in_function != None:
97 warn("Missing end of function %s" % (in_function,))
98 funcname = line[marker + 19:-1]
99 in_function = funcname
100 if in_function == function:
101 out.write(replacement)
104 marker = line.find(" -- End function")
114 def announce_test(name):
115 stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
118 def announce_result(result, info):
121 stderr.write(": %s" % info)
126 linkline="%s %s" % (LINKTEST, " ".join(files),)
127 res = subprocess.call(linkline, shell=True)
129 announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST)
132 announce_result("ok", "")
136 """Check files mode"""
137 for i in range(0, len(NO_PREFIX)):
140 if b not in BAD_FILES:
141 warn("There is no corresponding file to '%s' in %s" \
142 % (gooddir+"/"+f, baddir))
145 announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX)))
147 # combine files (everything from good except f)
151 badfile = baddir+"/"+c
152 goodfile = gooddir+"/"+c
154 testfiles.append(badfile)
155 if filecmp.cmp(goodfile, badfile):
156 announce_result("skipped", "same content")
160 testfiles.append(goodfile)
165 def check_functions_in_file(base, goodfile, badfile):
166 functions = extract_functions(goodfile)
167 if len(functions) == 0:
168 warn("Couldn't find any function in %s, missing annotations?" % (goodfile,))
170 badfunctions = dict(extract_functions(badfile))
171 if len(functions) == 0:
172 warn("Couldn't find any function in %s, missing annotations?" % (badfile,))
175 COMBINED="/tmp/combined.s"
177 for (func,func_text) in functions:
178 announce_test(func + " [%s/%s]" % (i+1, len(functions)))
180 if func not in badfunctions:
181 warn("Function '%s' missing from bad file" % func)
183 if badfunctions[func] == func_text:
184 announce_result("skipped", "same content")
186 replace_function(goodfile, func, badfunctions[func], COMBINED)
190 testfiles.append(COMBINED)
192 testfiles.append(gooddir + "/" + c)
196 parser = argparse.ArgumentParser()
197 parser.add_argument('--a', dest='dir_a', default='before')
198 parser.add_argument('--b', dest='dir_b', default='after')
199 parser.add_argument('--insane', help='Skip sanity check', action='store_true')
200 parser.add_argument('file', metavar='file', nargs='?')
201 config = parser.parse_args()
206 BAD_FILES=find(baddir, "*")
207 GOOD_FILES=find(gooddir, "*")
208 NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES])
210 # "Checking whether build environment is sane ..."
211 if not config.insane:
212 announce_test("sanity check")
213 if not os.access(LINKTEST, os.X_OK):
214 error("Expect '%s' to be present and executable" % (LINKTEST,))
217 res = testrun(GOOD_FILES)
219 # "build environment is grinning and holding a spatula. Guess not."
220 linkline="%s %s" % (LINKTEST, " ".join(GOOD_FILES),)
221 stderr.write("\n%s\n\n" % linkline)
222 stderr.write("Returned with exitcode != 0\n")
225 if config.file is not None:
227 goodfile = gooddir+"/"+config.file
228 badfile = baddir+"/"+config.file
229 check_functions_in_file(config.file, goodfile, badfile)
231 # Function exchange mode