• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2#                     The LLVM Compiler Infrastructure
3#
4# This file is distributed under the University of Illinois Open Source
5# License. See LICENSE.TXT for details.
6""" This module is responsible to run the analyzer commands. """
7
8import re
9import os
10import os.path
11import tempfile
12import functools
13import subprocess
14import logging
15from libscanbuild.compilation import classify_source, compiler_language
16from libscanbuild.clang import get_version, get_arguments
17from libscanbuild.shell import decode
18
19__all__ = ['run']
20
21# To have good results from static analyzer certain compiler options shall be
22# omitted. The compiler flag filtering only affects the static analyzer run.
23#
24# Keys are the option name, value number of options to skip
25IGNORED_FLAGS = {
26    '-c': 0,  # compile option will be overwritten
27    '-fsyntax-only': 0,  # static analyzer option will be overwritten
28    '-o': 1,  # will set up own output file
29    # flags below are inherited from the perl implementation.
30    '-g': 0,
31    '-save-temps': 0,
32    '-install_name': 1,
33    '-exported_symbols_list': 1,
34    '-current_version': 1,
35    '-compatibility_version': 1,
36    '-init': 1,
37    '-e': 1,
38    '-seg1addr': 1,
39    '-bundle_loader': 1,
40    '-multiply_defined': 1,
41    '-sectorder': 3,
42    '--param': 1,
43    '--serialize-diagnostics': 1
44}
45
46
47def require(required):
48    """ Decorator for checking the required values in state.
49
50    It checks the required attributes in the passed state and stop when
51    any of those is missing. """
52
53    def decorator(function):
54        @functools.wraps(function)
55        def wrapper(*args, **kwargs):
56            for key in required:
57                if key not in args[0]:
58                    raise KeyError('{0} not passed to {1}'.format(
59                        key, function.__name__))
60
61            return function(*args, **kwargs)
62
63        return wrapper
64
65    return decorator
66
67
68@require(['command',  # entry from compilation database
69          'directory',  # entry from compilation database
70          'file',  # entry from compilation database
71          'clang',  # clang executable name (and path)
72          'direct_args',  # arguments from command line
73          'force_debug',  # kill non debug macros
74          'output_dir',  # where generated report files shall go
75          'output_format',  # it's 'plist' or 'html' or both
76          'output_failures'])  # generate crash reports or not
77def run(opts):
78    """ Entry point to run (or not) static analyzer against a single entry
79    of the compilation database.
80
81    This complex task is decomposed into smaller methods which are calling
82    each other in chain. If the analyzis is not possibe the given method
83    just return and break the chain.
84
85    The passed parameter is a python dictionary. Each method first check
86    that the needed parameters received. (This is done by the 'require'
87    decorator. It's like an 'assert' to check the contract between the
88    caller and the called method.) """
89
90    try:
91        command = opts.pop('command')
92        command = command if isinstance(command, list) else decode(command)
93        logging.debug("Run analyzer against '%s'", command)
94        opts.update(classify_parameters(command))
95
96        return arch_check(opts)
97    except Exception:
98        logging.error("Problem occured during analyzis.", exc_info=1)
99        return None
100
101
102@require(['clang', 'directory', 'flags', 'file', 'output_dir', 'language',
103          'error_type', 'error_output', 'exit_code'])
104def report_failure(opts):
105    """ Create report when analyzer failed.
106
107    The major report is the preprocessor output. The output filename generated
108    randomly. The compiler output also captured into '.stderr.txt' file.
109    And some more execution context also saved into '.info.txt' file. """
110
111    def extension(opts):
112        """ Generate preprocessor file extension. """
113
114        mapping = {'objective-c++': '.mii', 'objective-c': '.mi', 'c++': '.ii'}
115        return mapping.get(opts['language'], '.i')
116
117    def destination(opts):
118        """ Creates failures directory if not exits yet. """
119
120        name = os.path.join(opts['output_dir'], 'failures')
121        if not os.path.isdir(name):
122            os.makedirs(name)
123        return name
124
125    error = opts['error_type']
126    (handle, name) = tempfile.mkstemp(suffix=extension(opts),
127                                      prefix='clang_' + error + '_',
128                                      dir=destination(opts))
129    os.close(handle)
130    cwd = opts['directory']
131    cmd = get_arguments([opts['clang'], '-fsyntax-only', '-E'] +
132                        opts['flags'] + [opts['file'], '-o', name], cwd)
133    logging.debug('exec command in %s: %s', cwd, ' '.join(cmd))
134    subprocess.call(cmd, cwd=cwd)
135    # write general information about the crash
136    with open(name + '.info.txt', 'w') as handle:
137        handle.write(opts['file'] + os.linesep)
138        handle.write(error.title().replace('_', ' ') + os.linesep)
139        handle.write(' '.join(cmd) + os.linesep)
140        handle.write(' '.join(os.uname()) + os.linesep)
141        handle.write(get_version(opts['clang']))
142        handle.close()
143    # write the captured output too
144    with open(name + '.stderr.txt', 'w') as handle:
145        handle.writelines(opts['error_output'])
146        handle.close()
147    # return with the previous step exit code and output
148    return {
149        'error_output': opts['error_output'],
150        'exit_code': opts['exit_code']
151    }
152
153
154@require(['clang', 'directory', 'flags', 'direct_args', 'file', 'output_dir',
155          'output_format'])
156def run_analyzer(opts, continuation=report_failure):
157    """ It assembles the analysis command line and executes it. Capture the
158    output of the analysis and returns with it. If failure reports are
159    requested, it calls the continuation to generate it. """
160
161    def output():
162        """ Creates output file name for reports. """
163        if opts['output_format'] in {'plist', 'plist-html'}:
164            (handle, name) = tempfile.mkstemp(prefix='report-',
165                                              suffix='.plist',
166                                              dir=opts['output_dir'])
167            os.close(handle)
168            return name
169        return opts['output_dir']
170
171    cwd = opts['directory']
172    cmd = get_arguments([opts['clang'], '--analyze'] + opts['direct_args'] +
173                        opts['flags'] + [opts['file'], '-o', output()],
174                        cwd)
175    logging.debug('exec command in %s: %s', cwd, ' '.join(cmd))
176    child = subprocess.Popen(cmd,
177                             cwd=cwd,
178                             universal_newlines=True,
179                             stdout=subprocess.PIPE,
180                             stderr=subprocess.STDOUT)
181    output = child.stdout.readlines()
182    child.stdout.close()
183    # do report details if it were asked
184    child.wait()
185    if opts.get('output_failures', False) and child.returncode:
186        error_type = 'crash' if child.returncode & 127 else 'other_error'
187        opts.update({
188            'error_type': error_type,
189            'error_output': output,
190            'exit_code': child.returncode
191        })
192        return continuation(opts)
193    # return the output for logging and exit code for testing
194    return {'error_output': output, 'exit_code': child.returncode}
195
196
197@require(['flags', 'force_debug'])
198def filter_debug_flags(opts, continuation=run_analyzer):
199    """ Filter out nondebug macros when requested. """
200
201    if opts.pop('force_debug'):
202        # lazy implementation just append an undefine macro at the end
203        opts.update({'flags': opts['flags'] + ['-UNDEBUG']})
204
205    return continuation(opts)
206
207
208@require(['file', 'directory'])
209def set_file_path_relative(opts, continuation=filter_debug_flags):
210    """ Set source file path to relative to the working directory.
211
212    The only purpose of this function is to pass the SATestBuild.py tests. """
213
214    opts.update({'file': os.path.relpath(opts['file'], opts['directory'])})
215
216    return continuation(opts)
217
218
219@require(['language', 'compiler', 'file', 'flags'])
220def language_check(opts, continuation=set_file_path_relative):
221    """ Find out the language from command line parameters or file name
222    extension. The decision also influenced by the compiler invocation. """
223
224    accepted = frozenset({
225        'c', 'c++', 'objective-c', 'objective-c++', 'c-cpp-output',
226        'c++-cpp-output', 'objective-c-cpp-output'
227    })
228
229    # language can be given as a parameter...
230    language = opts.pop('language')
231    compiler = opts.pop('compiler')
232    # ... or find out from source file extension
233    if language is None and compiler is not None:
234        language = classify_source(opts['file'], compiler == 'c')
235
236    if language is None:
237        logging.debug('skip analysis, language not known')
238        return None
239    elif language not in accepted:
240        logging.debug('skip analysis, language not supported')
241        return None
242    else:
243        logging.debug('analysis, language: %s', language)
244        opts.update({'language': language,
245                     'flags': ['-x', language] + opts['flags']})
246        return continuation(opts)
247
248
249@require(['arch_list', 'flags'])
250def arch_check(opts, continuation=language_check):
251    """ Do run analyzer through one of the given architectures. """
252
253    disabled = frozenset({'ppc', 'ppc64'})
254
255    received_list = opts.pop('arch_list')
256    if received_list:
257        # filter out disabled architectures and -arch switches
258        filtered_list = [a for a in received_list if a not in disabled]
259        if filtered_list:
260            # There should be only one arch given (or the same multiple
261            # times). If there are multiple arch are given and are not
262            # the same, those should not change the pre-processing step.
263            # But that's the only pass we have before run the analyzer.
264            current = filtered_list.pop()
265            logging.debug('analysis, on arch: %s', current)
266
267            opts.update({'flags': ['-arch', current] + opts['flags']})
268            return continuation(opts)
269        else:
270            logging.debug('skip analysis, found not supported arch')
271            return None
272    else:
273        logging.debug('analysis, on default arch')
274        return continuation(opts)
275
276
277def classify_parameters(command):
278    """ Prepare compiler flags (filters some and add others) and take out
279    language (-x) and architecture (-arch) flags for future processing. """
280
281    result = {
282        'flags': [],  # the filtered compiler flags
283        'arch_list': [],  # list of architecture flags
284        'language': None,  # compilation language, None, if not specified
285        'compiler': compiler_language(command)  # 'c' or 'c++'
286    }
287
288    # iterate on the compile options
289    args = iter(command[1:])
290    for arg in args:
291        # take arch flags into a separate basket
292        if arg == '-arch':
293            result['arch_list'].append(next(args))
294        # take language
295        elif arg == '-x':
296            result['language'] = next(args)
297        # parameters which looks source file are not flags
298        elif re.match(r'^[^-].+', arg) and classify_source(arg):
299            pass
300        # ignore some flags
301        elif arg in IGNORED_FLAGS:
302            count = IGNORED_FLAGS[arg]
303            for _ in range(count):
304                next(args)
305        # we don't care about extra warnings, but we should suppress ones
306        # that we don't want to see.
307        elif re.match(r'^-W.+', arg) and not re.match(r'^-Wno-.+', arg):
308            pass
309        # and consider everything else as compilation flag.
310        else:
311            result['flags'].append(arg)
312
313    return result
314