• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Test harness for chromium clang tools."""
7
8import difflib
9import glob
10import json
11import os
12import os.path
13import shutil
14import subprocess
15import sys
16
17
18def _RunGit(args):
19  if sys.platform == 'win32':
20    args = ['git.bat'] + args
21  else:
22    args = ['git'] + args
23  subprocess.check_call(args)
24
25
26def _GenerateCompileCommands(files, include_paths):
27  """Returns a JSON string containing a compilation database for the input."""
28  # Note: in theory, backslashes in the compile DB should work but the tools
29  # that write compile DBs and the tools that read them don't agree on the
30  # escaping convention: https://llvm.org/bugs/show_bug.cgi?id=19687
31  files = [f.replace('\\', '/') for f in files]
32  include_path_flags = ' '.join('-I %s' % include_path.replace('\\', '/')
33                                for include_path in include_paths)
34  return json.dumps([{'directory': '.',
35                      'command': 'clang++ -std=c++11 -fsyntax-only %s -c %s' % (
36                          include_path_flags, f),
37                      'file': f} for f in files], indent=2)
38
39
40def _NumberOfTestsToString(tests):
41  """Returns an English describing the number of tests."""
42  return '%d test%s' % (tests, 's' if tests != 1 else '')
43
44
45def _RunToolAndApplyEdits(tools_clang_scripts_directory,
46                          tool_to_test,
47                          test_directory_for_tool,
48                          actual_files):
49  try:
50    # Stage the test files in the git index. If they aren't staged, then
51    # run_tool.py will skip them when applying replacements.
52    args = ['add']
53    args.extend(actual_files)
54    _RunGit(args)
55
56    # Launch the following pipeline:
57    #     run_tool.py ... | extract_edits.py | apply_edits.py ...
58    args = ['python',
59            os.path.join(tools_clang_scripts_directory, 'run_tool.py'),
60            tool_to_test,
61            test_directory_for_tool]
62    args.extend(actual_files)
63    run_tool = subprocess.Popen(args, stdout=subprocess.PIPE)
64
65    args = ['python',
66            os.path.join(tools_clang_scripts_directory, 'extract_edits.py')]
67    extract_edits = subprocess.Popen(args, stdin=run_tool.stdout,
68                                     stdout=subprocess.PIPE)
69
70    args = ['python',
71            os.path.join(tools_clang_scripts_directory, 'apply_edits.py'),
72            test_directory_for_tool]
73    apply_edits = subprocess.Popen(args, stdin=extract_edits.stdout,
74                                   stdout=subprocess.PIPE)
75
76    # Wait for the pipeline to finish running + check exit codes.
77    stdout, _ = apply_edits.communicate()
78    for process in [run_tool, extract_edits, apply_edits]:
79      process.wait()
80      if process.returncode != 0:
81        print "Failure while running the tool."
82        return process.returncode
83
84    # Reformat the resulting edits via: git cl format.
85    args = ['cl', 'format']
86    args.extend(actual_files)
87    _RunGit(args)
88
89    return 0
90
91  finally:
92    # No matter what, unstage the git changes we made earlier to avoid polluting
93    # the index.
94    args = ['reset', '--quiet', 'HEAD']
95    args.extend(actual_files)
96    _RunGit(args)
97
98
99def main(argv):
100  if len(argv) < 1:
101    print 'Usage: test_tool.py <clang tool>'
102    print '  <clang tool> is the clang tool to be tested.'
103    sys.exit(1)
104
105  tool_to_test = argv[0]
106  print '\nTesting %s\n' % tool_to_test
107  tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__))
108  tools_clang_directory = os.path.dirname(tools_clang_scripts_directory)
109  test_directory_for_tool = os.path.join(
110      tools_clang_directory, tool_to_test, 'tests')
111  compile_database = os.path.join(test_directory_for_tool,
112                                  'compile_commands.json')
113  source_files = glob.glob(os.path.join(test_directory_for_tool,
114                                        '*-original.cc'))
115  actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc'])
116                  for source_file in source_files]
117  expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.cc'])
118                    for source_file in source_files]
119  include_paths = []
120  include_paths.append(
121      os.path.realpath(os.path.join(tools_clang_directory, '../..')))
122  # Many gtest and gmock headers expect to have testing/gtest/include and/or
123  # testing/gmock/include in the include search path.
124  include_paths.append(
125      os.path.realpath(os.path.join(tools_clang_directory,
126                                    '../..',
127                                    'testing/gtest/include')))
128  include_paths.append(
129      os.path.realpath(os.path.join(tools_clang_directory,
130                                    '../..',
131                                    'testing/gmock/include')))
132
133  if len(actual_files) == 0:
134    print 'Tool "%s" does not have compatible test files.' % tool_to_test
135    return 1
136
137  # Set up the test environment.
138  for source, actual in zip(source_files, actual_files):
139    shutil.copyfile(source, actual)
140  # Generate a temporary compilation database to run the tool over.
141  with open(compile_database, 'w') as f:
142    f.write(_GenerateCompileCommands(actual_files, include_paths))
143
144  # Run the tool.
145  exitcode = _RunToolAndApplyEdits(tools_clang_scripts_directory, tool_to_test,
146                                   test_directory_for_tool, actual_files)
147  if (exitcode != 0):
148    return exitcode
149
150  # Compare actual-vs-expected results.
151  passed = 0
152  failed = 0
153  for expected, actual in zip(expected_files, actual_files):
154    print '[ RUN      ] %s' % os.path.relpath(actual)
155    expected_output = actual_output = None
156    with open(expected, 'r') as f:
157      expected_output = f.readlines()
158    with open(actual, 'r') as f:
159      actual_output = f.readlines()
160    if actual_output != expected_output:
161      failed += 1
162      for line in difflib.unified_diff(expected_output, actual_output,
163                                       fromfile=os.path.relpath(expected),
164                                       tofile=os.path.relpath(actual)):
165        sys.stdout.write(line)
166      print '[  FAILED  ] %s' % os.path.relpath(actual)
167      # Don't clean up the file on failure, so the results can be referenced
168      # more easily.
169      continue
170    print '[       OK ] %s' % os.path.relpath(actual)
171    passed += 1
172    os.remove(actual)
173
174  if failed == 0:
175    os.remove(compile_database)
176
177  print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files))
178  if passed > 0:
179    print '[  PASSED  ] %s.' % _NumberOfTestsToString(passed)
180  if failed > 0:
181    print '[  FAILED  ] %s.' % _NumberOfTestsToString(failed)
182    return 1
183
184
185if __name__ == '__main__':
186  sys.exit(main(sys.argv[1:]))
187