• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os
2import platform
3import re
4import subprocess
5import sys
6
7import lit.util
8from lit.llvm.subst import FindTool
9from lit.llvm.subst import ToolSubst
10
11
12class LLVMConfig(object):
13
14    def __init__(self, lit_config, config):
15        self.lit_config = lit_config
16        self.config = config
17
18        features = config.available_features
19
20        self.use_lit_shell = False
21        # Tweak PATH for Win32 to decide to use bash.exe or not.
22        if sys.platform == 'win32':
23            # For tests that require Windows to run.
24            features.add('system-windows')
25
26            # Seek sane tools in directories and set to $PATH.
27            path = self.lit_config.getToolsPath(config.lit_tools_dir,
28                                                config.environment['PATH'],
29                                                ['cmp.exe', 'grep.exe', 'sed.exe'])
30            if path is not None:
31                self.with_environment('PATH', path, append_path=True)
32            # Many tools behave strangely if these environment variables aren't set.
33            self.with_system_environment(['SystemDrive', 'SystemRoot', 'TEMP', 'TMP'])
34            self.use_lit_shell = True
35
36        # Choose between lit's internal shell pipeline runner and a real shell.  If
37        # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
38        lit_shell_env = os.environ.get('LIT_USE_INTERNAL_SHELL')
39        if lit_shell_env:
40            self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
41
42        if not self.use_lit_shell:
43            features.add('shell')
44
45        # Running on Darwin OS
46        if platform.system() == 'Darwin':
47            # FIXME: lld uses the first, other projects use the second.
48            # We should standardize on the former.
49            features.add('system-linker-mach-o')
50            features.add('system-darwin')
51        elif platform.system() == 'Windows':
52            # For tests that require Windows to run.
53            features.add('system-windows')
54        elif platform.system() == 'Linux':
55            features.add('system-linux')
56        elif platform.system() in ['FreeBSD']:
57            features.add('system-freebsd')
58        elif platform.system() == 'NetBSD':
59            features.add('system-netbsd')
60        elif platform.system() == 'AIX':
61            features.add('system-aix')
62        elif platform.system() == 'SunOS':
63            features.add('system-solaris')
64
65        # Native compilation: host arch == default triple arch
66        # Both of these values should probably be in every site config (e.g. as
67        # part of the standard header.  But currently they aren't)
68        host_triple = getattr(config, 'host_triple', None)
69        target_triple = getattr(config, 'target_triple', None)
70        if host_triple and host_triple == target_triple:
71            features.add('native')
72
73        # Sanitizers.
74        sanitizers = getattr(config, 'llvm_use_sanitizer', '')
75        sanitizers = frozenset(x.lower() for x in sanitizers.split(';'))
76        if 'address' in sanitizers:
77            features.add('asan')
78        if 'memory' in sanitizers or 'memorywithorigins' in sanitizers:
79            features.add('msan')
80        if 'undefined' in sanitizers:
81            features.add('ubsan')
82
83        have_zlib = getattr(config, 'have_zlib', None)
84        if have_zlib:
85            features.add('zlib')
86
87        # Check if we should run long running tests.
88        long_tests = lit_config.params.get('run_long_tests', None)
89        if lit.util.pythonize_bool(long_tests):
90            features.add('long_tests')
91
92        if target_triple:
93            if re.match(r'^x86_64.*-apple', target_triple):
94                features.add('x86_64-apple')
95                host_cxx = getattr(config, 'host_cxx', None)
96                if 'address' in sanitizers and self.get_clang_has_lsan(host_cxx, target_triple):
97                    self.with_environment(
98                        'ASAN_OPTIONS', 'detect_leaks=1', append_path=True)
99            if re.match(r'^x86_64.*-linux', target_triple):
100                features.add('x86_64-linux')
101            if re.match(r'^i.86.*', target_triple):
102                features.add('target-x86')
103            elif re.match(r'^x86_64.*', target_triple):
104                features.add('target-x86_64')
105            elif re.match(r'^aarch64.*', target_triple):
106                features.add('target-aarch64')
107            elif re.match(r'^arm.*', target_triple):
108                features.add('target-arm')
109
110        use_gmalloc = lit_config.params.get('use_gmalloc', None)
111        if lit.util.pythonize_bool(use_gmalloc):
112            # Allow use of an explicit path for gmalloc library.
113            # Will default to '/usr/lib/libgmalloc.dylib' if not set.
114            gmalloc_path_str = lit_config.params.get('gmalloc_path',
115                                                     '/usr/lib/libgmalloc.dylib')
116            if gmalloc_path_str is not None:
117                self.with_environment(
118                    'DYLD_INSERT_LIBRARIES', gmalloc_path_str)
119
120    def with_environment(self, variable, value, append_path=False):
121        if append_path:
122            # For paths, we should be able to take a list of them and process all
123            # of them.
124            paths_to_add = value
125            if lit.util.is_string(paths_to_add):
126                paths_to_add = [paths_to_add]
127
128            def norm(x):
129                return os.path.normcase(os.path.normpath(x))
130
131            current_paths = self.config.environment.get(variable, None)
132            if current_paths:
133                current_paths = current_paths.split(os.path.pathsep)
134                paths = [norm(p) for p in current_paths]
135            else:
136                paths = []
137
138            # If we are passed a list [a b c], then iterating this list forwards
139            # and adding each to the beginning would result in b c a.  So we
140            # need to iterate in reverse to end up with the original ordering.
141            for p in reversed(paths_to_add):
142                # Move it to the front if it already exists, otherwise insert it at the
143                # beginning.
144                p = norm(p)
145                try:
146                    paths.remove(p)
147                except ValueError:
148                    pass
149                paths = [p] + paths
150            value = os.pathsep.join(paths)
151        self.config.environment[variable] = value
152
153    def with_system_environment(self, variables, append_path=False):
154        if lit.util.is_string(variables):
155            variables = [variables]
156        for v in variables:
157            value = os.environ.get(v)
158            if value:
159                self.with_environment(v, value, append_path)
160
161    def clear_environment(self, variables):
162        for name in variables:
163            if name in self.config.environment:
164                del self.config.environment[name]
165
166    def get_process_output(self, command):
167        try:
168            cmd = subprocess.Popen(
169                command, stdout=subprocess.PIPE,
170                stderr=subprocess.PIPE, env=self.config.environment)
171            stdout, stderr = cmd.communicate()
172            stdout = lit.util.to_string(stdout)
173            stderr = lit.util.to_string(stderr)
174            return (stdout, stderr)
175        except OSError:
176            self.lit_config.fatal('Could not run process %s' % command)
177
178    def feature_config(self, features):
179        # Ask llvm-config about the specified feature.
180        arguments = [x for (x, _) in features]
181        config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
182
183        output, _ = self.get_process_output([config_path] + arguments)
184        lines = output.split('\n')
185
186        for (feature_line, (_, patterns)) in zip(lines, features):
187            # We should have either a callable or a dictionary.  If it's a
188            # dictionary, grep each key against the output and use the value if
189            # it matches.  If it's a callable, it does the entire translation.
190            if callable(patterns):
191                features_to_add = patterns(feature_line)
192                self.config.available_features.update(features_to_add)
193            else:
194                for (re_pattern, feature) in patterns.items():
195                    if re.search(re_pattern, feature_line):
196                        self.config.available_features.add(feature)
197
198    # Note that when substituting %clang_cc1 also fill in the include directory of
199    # the builtin headers. Those are part of even a freestanding environment, but
200    # Clang relies on the driver to locate them.
201    def get_clang_builtin_include_dir(self, clang):
202        # FIXME: Rather than just getting the version, we should have clang print
203        # out its resource dir here in an easy to scrape form.
204        clang_dir, _ = self.get_process_output(
205            [clang, '-print-file-name=include'])
206
207        if not clang_dir:
208            self.lit_config.fatal(
209                "Couldn't find the include dir for Clang ('%s')" % clang)
210
211        clang_dir = clang_dir.strip()
212        if sys.platform in ['win32'] and not self.use_lit_shell:
213            # Don't pass dosish path separator to msys bash.exe.
214            clang_dir = clang_dir.replace('\\', '/')
215        # Ensure the result is an ascii string, across Python2.5+ - Python3.
216        return clang_dir
217
218    # On macOS, LSan is only supported on clang versions 5 and higher
219    def get_clang_has_lsan(self, clang, triple):
220        if not clang:
221            self.lit_config.warning(
222                'config.host_cxx is unset but test suite is configured to use sanitizers.')
223            return False
224
225        clang_binary = clang.split()[0]
226        version_string, _ = self.get_process_output(
227            [clang_binary, '--version'])
228        if not 'clang' in version_string:
229            self.lit_config.warning(
230                "compiler '%s' does not appear to be clang, " % clang_binary +
231                'but test suite is configured to use sanitizers.')
232            return False
233
234        if re.match(r'.*-linux', triple):
235            return True
236
237        if re.match(r'^x86_64.*-apple', triple):
238            version_regex = re.search(r'version ([0-9]+)\.([0-9]+).([0-9]+)', version_string)
239            major_version_number = int(version_regex.group(1))
240            minor_version_number = int(version_regex.group(2))
241            patch_version_number = int(version_regex.group(3))
242            if ('Apple LLVM' in version_string) or ('Apple clang' in version_string):
243                # Apple clang doesn't yet support LSan
244                return False
245            else:
246                return major_version_number >= 5
247
248        return False
249
250    def make_itanium_abi_triple(self, triple):
251        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
252        if not m:
253            self.lit_config.fatal(
254                "Could not turn '%s' into Itanium ABI triple" % triple)
255        if m.group(3).lower() != 'windows':
256            # All non-windows triples use the Itanium ABI.
257            return triple
258        return m.group(1) + '-' + m.group(2) + '-' + m.group(3) + '-gnu'
259
260    def make_msabi_triple(self, triple):
261        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
262        if not m:
263            self.lit_config.fatal(
264                "Could not turn '%s' into MS ABI triple" % triple)
265        isa = m.group(1).lower()
266        vendor = m.group(2).lower()
267        os = m.group(3).lower()
268        if os == 'windows' and re.match(r'.*-msvc$', triple):
269            # If the OS is windows and environment is msvc, we're done.
270            return triple
271        if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
272            # For x86 ISAs, adjust the OS.
273            return isa + '-' + vendor + '-windows-msvc'
274        # -msvc is not supported for non-x86 targets; use a default.
275        return 'i686-pc-windows-msvc'
276
277    def add_tool_substitutions(self, tools, search_dirs=None):
278        if not search_dirs:
279            search_dirs = [self.config.llvm_tools_dir]
280
281        if lit.util.is_string(search_dirs):
282            search_dirs = [search_dirs]
283
284        tools = [x if isinstance(x, ToolSubst) else ToolSubst(x)
285                 for x in tools]
286
287        search_dirs = os.pathsep.join(search_dirs)
288        substitutions = []
289
290        for tool in tools:
291            match = tool.resolve(self, search_dirs)
292
293            # Either no match occurred, or there was an unresolved match that
294            # is ignored.
295            if not match:
296                continue
297
298            subst_key, tool_pipe, command = match
299
300            # An unresolved match occurred that can't be ignored.  Fail without
301            # adding any of the previously-discovered substitutions.
302            if not command:
303                return False
304
305            substitutions.append((subst_key, tool_pipe + command))
306
307        self.config.substitutions.extend(substitutions)
308        return True
309
310    def use_default_substitutions(self):
311        tool_patterns = [
312            ToolSubst('FileCheck', unresolved='fatal'),
313            # Handle these specially as they are strings searched for during testing.
314            ToolSubst(r'\| \bcount\b', command=FindTool(
315                'count'), verbatim=True, unresolved='fatal'),
316            ToolSubst(r'\| \bnot\b', command=FindTool('not'), verbatim=True, unresolved='fatal')]
317
318        self.config.substitutions.append(('%python', '"%s"' % (sys.executable)))
319
320        self.add_tool_substitutions(
321            tool_patterns, [self.config.llvm_tools_dir])
322
323    def use_llvm_tool(self, name, search_env=None, required=False, quiet=False):
324        """Find the executable program 'name', optionally using the specified
325        environment variable as an override before searching the
326        configuration's PATH."""
327        # If the override is specified in the environment, use it without
328        # validation.
329        if search_env:
330            tool = self.config.environment.get(search_env)
331            if tool:
332                return tool
333
334        # Otherwise look in the path.
335        tool = lit.util.which(name, self.config.environment['PATH'])
336
337        if required and not tool:
338            message = "couldn't find '{}' program".format(name)
339            if search_env:
340                message = message + \
341                    ', try setting {} in your environment'.format(search_env)
342            self.lit_config.fatal(message)
343
344        if tool:
345            tool = os.path.normpath(tool)
346            if not self.lit_config.quiet and not quiet:
347                self.lit_config.note('using {}: {}'.format(name, tool))
348        return tool
349
350    def use_clang(self, additional_tool_dirs=[], additional_flags=[], required=True):
351        """Configure the test suite to be able to invoke clang.
352
353        Sets up some environment variables important to clang, locates a
354        just-built or installed clang, and add a set of standard
355        substitutions useful to any test suite that makes use of clang.
356
357        """
358        # Clear some environment variables that might affect Clang.
359        #
360        # This first set of vars are read by Clang, but shouldn't affect tests
361        # that aren't specifically looking for these features, or are required
362        # simply to run the tests at all.
363        #
364        # FIXME: Should we have a tool that enforces this?
365
366        # safe_env_vars = ('TMPDIR', 'TEMP', 'TMP', 'USERPROFILE', 'PWD',
367        #                  'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET',
368        #                  'VCINSTALLDIR', 'VC100COMNTOOLS', 'VC90COMNTOOLS',
369        #                  'VC80COMNTOOLS')
370        possibly_dangerous_env_vars = ['COMPILER_PATH', 'RC_DEBUG_OPTIONS',
371                                       'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH',
372                                       'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH',
373                                       'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH',
374                                       'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING',
375                                       'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX',
376                                       'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS',
377                                       'LIBCLANG_RESOURCE_USAGE',
378                                       'LIBCLANG_CODE_COMPLETION_LOGGING']
379        # Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it.
380        if platform.system() != 'Windows':
381            possibly_dangerous_env_vars.append('INCLUDE')
382
383        self.clear_environment(possibly_dangerous_env_vars)
384
385        # Tweak the PATH to include the tools dir and the scripts dir.
386        # Put Clang first to avoid LLVM from overriding out-of-tree clang builds.
387        exe_dir_props = [self.config.name.lower() + '_tools_dir', 'clang_tools_dir', 'llvm_tools_dir']
388        paths = [getattr(self.config, pp) for pp in exe_dir_props
389                 if getattr(self.config, pp, None)]
390        paths = additional_tool_dirs + paths
391        self.with_environment('PATH', paths, append_path=True)
392
393        lib_dir_props = [self.config.name.lower() + '_libs_dir', 'clang_libs_dir', 'llvm_shlib_dir', 'llvm_libs_dir']
394        paths = [getattr(self.config, pp) for pp in lib_dir_props
395                 if getattr(self.config, pp, None)]
396
397        self.with_environment('LD_LIBRARY_PATH', paths, append_path=True)
398
399        shl = getattr(self.config, 'llvm_shlib_dir', None)
400        pext = getattr(self.config, 'llvm_plugin_ext', None)
401        if shl:
402            self.config.substitutions.append(('%llvmshlibdir', shl))
403        if pext:
404            self.config.substitutions.append(('%pluginext', pext))
405
406        # Discover the 'clang' and 'clangcc' to use.
407        self.config.clang = self.use_llvm_tool(
408            'clang', search_env='CLANG', required=required)
409        if self.config.clang:
410          self.config.available_features.add('clang')
411          builtin_include_dir = self.get_clang_builtin_include_dir(self.config.clang)
412          tool_substitutions = [
413              ToolSubst('%clang', command=self.config.clang, extra_args=additional_flags),
414              ToolSubst('%clang_analyze_cc1', command='%clang_cc1', extra_args=['-analyze', '%analyze', '-setup-static-analyzer']+additional_flags),
415              ToolSubst('%clang_cc1', command=self.config.clang, extra_args=['-cc1', '-internal-isystem', builtin_include_dir, '-nostdsysteminc']+additional_flags),
416              ToolSubst('%clang_cpp', command=self.config.clang, extra_args=['--driver-mode=cpp']+additional_flags),
417              ToolSubst('%clang_cl', command=self.config.clang, extra_args=['--driver-mode=cl']+additional_flags),
418              ToolSubst('%clangxx', command=self.config.clang, extra_args=['--driver-mode=g++']+additional_flags),
419              ]
420          self.add_tool_substitutions(tool_substitutions)
421          self.config.substitutions.append(
422              ('%resource_dir', builtin_include_dir))
423
424        self.config.substitutions.append(('%itanium_abi_triple',
425                                          self.make_itanium_abi_triple(self.config.target_triple)))
426        self.config.substitutions.append(('%ms_abi_triple',
427                                          self.make_msabi_triple(self.config.target_triple)))
428
429        # The host triple might not be set, at least if we're compiling clang from
430        # an already installed llvm.
431        if self.config.host_triple and self.config.host_triple != '@LLVM_HOST_TRIPLE@':
432            self.config.substitutions.append(('%target_itanium_abi_host_triple',
433                                              '--target=%s' % self.make_itanium_abi_triple(self.config.host_triple)))
434        else:
435            self.config.substitutions.append(
436                ('%target_itanium_abi_host_triple', ''))
437
438        # FIXME: Find nicer way to prohibit this.
439        self.config.substitutions.append(
440            (' clang ', """\"*** Do not use 'clang' in tests, use '%clang'. ***\""""))
441        self.config.substitutions.append(
442            (r' clang\+\+ ', """\"*** Do not use 'clang++' in tests, use '%clangxx'. ***\""""))
443        self.config.substitutions.append(
444            (' clang-cc ',
445             """\"*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***\""""))
446        self.config.substitutions.append(
447            (' clang-cl ',
448             """\"*** Do not use 'clang-cl' in tests, use '%clang_cl'. ***\""""))
449        self.config.substitutions.append(
450            (' clang -cc1 -analyze ',
451             """\"*** Do not use 'clang -cc1 -analyze' in tests, use '%clang_analyze_cc1'. ***\""""))
452        self.config.substitutions.append(
453            (' clang -cc1 ',
454             """\"*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***\""""))
455        self.config.substitutions.append(
456            (' %clang-cc1 ',
457             """\"*** invalid substitution, use '%clang_cc1'. ***\""""))
458        self.config.substitutions.append(
459            (' %clang-cpp ',
460             """\"*** invalid substitution, use '%clang_cpp'. ***\""""))
461        self.config.substitutions.append(
462            (' %clang-cl ',
463             """\"*** invalid substitution, use '%clang_cl'. ***\""""))
464
465    def use_lld(self, additional_tool_dirs=[], required=True):
466        """Configure the test suite to be able to invoke lld.
467
468        Sets up some environment variables important to lld, locates a
469        just-built or installed lld, and add a set of standard
470        substitutions useful to any test suite that makes use of lld.
471
472        """
473
474        # Tweak the PATH to include the tools dir and the scripts dir.
475        exe_dir_props = [self.config.name.lower() + '_tools_dir', 'lld_tools_dir', 'llvm_tools_dir']
476        paths = [getattr(self.config, pp) for pp in exe_dir_props
477                 if getattr(self.config, pp, None)]
478        paths = additional_tool_dirs + paths
479        self.with_environment('PATH', paths, append_path=True)
480
481        lib_dir_props = [self.config.name.lower() + '_libs_dir', 'lld_libs_dir', 'llvm_libs_dir']
482        paths = [getattr(self.config, pp) for pp in lib_dir_props
483                 if getattr(self.config, pp, None)]
484
485        self.with_environment('LD_LIBRARY_PATH', paths, append_path=True)
486
487        # Discover the 'clang' and 'clangcc' to use.
488
489        ld_lld = self.use_llvm_tool('ld.lld', required=required)
490        lld_link = self.use_llvm_tool('lld-link', required=required)
491        ld64_lld = self.use_llvm_tool('ld64.lld', required=required)
492        wasm_ld = self.use_llvm_tool('wasm-ld', required=required)
493
494        was_found = ld_lld and lld_link and ld64_lld and wasm_ld
495        tool_substitutions = []
496        if ld_lld:
497            tool_substitutions.append(ToolSubst(r'ld\.lld', command=ld_lld))
498        if lld_link:
499            tool_substitutions.append(ToolSubst('lld-link', command=lld_link))
500        if ld64_lld:
501            tool_substitutions.append(ToolSubst(r'ld64\.lld', command=ld64_lld))
502        if wasm_ld:
503            tool_substitutions.append(ToolSubst('wasm-ld', command=wasm_ld))
504        self.add_tool_substitutions(tool_substitutions)
505        return was_found
506