1#===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7#===----------------------------------------------------------------------===## 8 9import lit 10import lit.formats 11import os 12import pipes 13import re 14import shutil 15 16def _getTempPaths(test): 17 """ 18 Return the values to use for the %T and %t substitutions, respectively. 19 20 The difference between this and Lit's default behavior is that we guarantee 21 that %T is a path unique to the test being run. 22 """ 23 tmpDir, _ = lit.TestRunner.getTempPaths(test) 24 _, testName = os.path.split(test.getExecPath()) 25 tmpDir = os.path.join(tmpDir, testName + '.dir') 26 tmpBase = os.path.join(tmpDir, 't') 27 return tmpDir, tmpBase 28 29def _checkBaseSubstitutions(substitutions): 30 substitutions = [s for (s, _) in substitutions] 31 for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{flags}', '%{exec}']: 32 assert s in substitutions, "Required substitution {} was not provided".format(s) 33 34def parseScript(test, preamble): 35 """ 36 Extract the script from a test, with substitutions applied. 37 38 Returns a list of commands ready to be executed. 39 40 - test 41 The lit.Test to parse. 42 43 - preamble 44 A list of commands to perform before any command in the test. 45 These commands can contain unexpanded substitutions, but they 46 must not be of the form 'RUN:' -- they must be proper commands 47 once substituted. 48 """ 49 # Get the default substitutions 50 tmpDir, tmpBase = _getTempPaths(test) 51 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 52 53 # Check base substitutions and add the %{build} and %{run} convenience substitutions 54 _checkBaseSubstitutions(substitutions) 55 substitutions.append(('%{build}', '%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe')) 56 substitutions.append(('%{run}', '%{exec} %t.exe')) 57 58 # Parse the test file, including custom directives 59 additionalCompileFlags = [] 60 fileDependencies = [] 61 parsers = [ 62 lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:', 63 lit.TestRunner.ParserKind.LIST, 64 initial_value=fileDependencies), 65 lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', 66 lit.TestRunner.ParserKind.LIST, 67 initial_value=additionalCompileFlags) 68 ] 69 70 # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first 71 # class support for conditional keywords in Lit, which would allow evaluating arbitrary 72 # Lit boolean expressions instead. 73 for feature in test.config.available_features: 74 parser = lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS({}):'.format(feature), 75 lit.TestRunner.ParserKind.LIST, 76 initial_value=additionalCompileFlags) 77 parsers.append(parser) 78 79 scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, 80 require_script=not preamble) 81 if isinstance(scriptInTest, lit.Test.Result): 82 return scriptInTest 83 84 script = [] 85 86 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 87 # that file to the execution directory. Execute the copy from %S to allow 88 # relative paths from the test directory. 89 for dep in fileDependencies: 90 script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)] 91 script += preamble 92 script += scriptInTest 93 94 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 95 substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}' 96 else (s, x) for (s, x) in substitutions] 97 98 # Perform substitutions in the script itself. 99 script = lit.TestRunner.applySubstitutions(script, substitutions, 100 recursion_limit=test.config.recursiveExpansionLimit) 101 102 return script 103 104 105class CxxStandardLibraryTest(lit.formats.TestFormat): 106 """ 107 Lit test format for the C++ Standard Library conformance test suite. 108 109 This test format is based on top of the ShTest format -- it basically 110 creates a shell script performing the right operations (compile/link/run) 111 based on the extension of the test file it encounters. It supports files 112 with the following extensions: 113 114 FOO.pass.cpp - Compiles, links and runs successfully 115 FOO.pass.mm - Same as .pass.cpp, but for Objective-C++ 116 117 FOO.compile.pass.cpp - Compiles successfully, link and run not attempted 118 FOO.compile.pass.mm - Same as .compile.pass.cpp, but for Objective-C++ 119 FOO.compile.fail.cpp - Does not compile successfully 120 121 FOO.link.pass.cpp - Compiles and links successfully, run not attempted 122 FOO.link.pass.mm - Same as .link.pass.cpp, but for Objective-C++ 123 FOO.link.fail.cpp - Compiles successfully, but fails to link 124 125 FOO.sh.<anything> - A builtin Lit Shell test 126 127 FOO.verify.cpp - Compiles with clang-verify. This type of test is 128 automatically marked as UNSUPPORTED if the compiler 129 does not support Clang-verify. 130 131 FOO.fail.cpp - Compiled with clang-verify if clang-verify is 132 supported, and equivalent to a .compile.fail.cpp 133 test otherwise. This is supported only for backwards 134 compatibility with the test suite. 135 136 137 Substitution requirements 138 =============================== 139 The test format operates by assuming that each test's configuration provides 140 the following substitutions, which it will reuse in the shell scripts it 141 constructs: 142 %{cxx} - A command that can be used to invoke the compiler 143 %{compile_flags} - Flags to use when compiling a test case 144 %{link_flags} - Flags to use when linking a test case 145 %{flags} - Flags to use either when compiling or linking a test case 146 %{exec} - A command to prefix the execution of executables 147 148 Note that when building an executable (as opposed to only compiling a source 149 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 150 in the same command line. In other words, the test format doesn't perform 151 separate compilation and linking steps in this case. 152 153 154 Additional supported directives 155 =============================== 156 In addition to everything that's supported in Lit ShTests, this test format 157 also understands the following directives inside test files: 158 159 // FILE_DEPENDENCIES: file, directory, /path/to/file 160 161 This directive expresses that the test requires the provided files 162 or directories in order to run. An example is a test that requires 163 some test input stored in a data file. When a test file contains 164 such a directive, this test format will collect them and copy them 165 to the directory represented by %T. The intent is that %T contains 166 all the inputs necessary to run the test, such that e.g. execution 167 on a remote host can be done by simply copying %T to the host. 168 169 // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3 170 171 This directive will cause the provided flags to be added to the 172 %{compile_flags} substitution for the test that contains it. This 173 allows adding special compilation flags without having to use a 174 .sh.cpp test, which would be more powerful but perhaps overkill. 175 176 177 Additional provided substitutions and features 178 ============================================== 179 The test format will define the following substitutions for use inside tests: 180 181 %{build} 182 Expands to a command-line that builds the current source 183 file with the %{flags}, %{compile_flags} and %{link_flags} 184 substitutions, and that produces an executable named %t.exe. 185 186 %{run} 187 Equivalent to `%{exec} %t.exe`. This is intended to be used 188 in conjunction with the %{build} substitution. 189 """ 190 def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): 191 SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', 192 '[.]compile[.]pass[.]cpp$', '[.]compile[.]pass[.]mm$', 193 '[.]compile[.]fail[.]cpp$', 194 '[.]link[.]pass[.]cpp$', '[.]link[.]pass[.]mm$', 195 '[.]link[.]fail[.]cpp$', 196 '[.]sh[.][^.]+$', 197 '[.]verify[.]cpp$', 198 '[.]fail[.]cpp$'] 199 sourcePath = testSuite.getSourcePath(pathInSuite) 200 for filename in os.listdir(sourcePath): 201 # Ignore dot files and excluded tests. 202 if filename.startswith('.') or filename in localConfig.excludes: 203 continue 204 205 filepath = os.path.join(sourcePath, filename) 206 if not os.path.isdir(filepath): 207 if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]): 208 yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig) 209 210 def execute(self, test, litConfig): 211 VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0' 212 supportsVerify = 'verify-support' in test.config.available_features 213 filename = test.path_in_suite[-1] 214 215 if re.search('[.]sh[.][^.]+$', filename): 216 steps = [ ] # The steps are already in the script 217 return self._executeShTest(test, litConfig, steps) 218 elif filename.endswith('.compile.pass.cpp') or filename.endswith('.compile.pass.mm'): 219 steps = [ 220 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 221 ] 222 return self._executeShTest(test, litConfig, steps) 223 elif filename.endswith('.compile.fail.cpp'): 224 steps = [ 225 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 226 ] 227 return self._executeShTest(test, litConfig, steps) 228 elif filename.endswith('.link.pass.cpp') or filename.endswith('.link.pass.mm'): 229 steps = [ 230 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 231 ] 232 return self._executeShTest(test, litConfig, steps) 233 elif filename.endswith('.link.fail.cpp'): 234 steps = [ 235 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 236 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe" 237 ] 238 return self._executeShTest(test, litConfig, steps) 239 elif filename.endswith('.verify.cpp'): 240 if not supportsVerify: 241 return lit.Test.Result(lit.Test.UNSUPPORTED, 242 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(test.getFullName())) 243 steps = [ 244 # Note: Use -Wno-error to make sure all diagnostics are not treated as errors, 245 # which doesn't make sense for clang-verify tests. 246 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 247 ] 248 return self._executeShTest(test, litConfig, steps) 249 # Make sure to check these ones last, since they will match other 250 # suffixes above too. 251 elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'): 252 steps = [ 253 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 254 "%dbg(EXECUTED AS) %{exec} %t.exe" 255 ] 256 return self._executeShTest(test, litConfig, steps) 257 # This is like a .verify.cpp test when clang-verify is supported, 258 # otherwise it's like a .compile.fail.cpp test. This is only provided 259 # for backwards compatibility with the test suite. 260 elif filename.endswith('.fail.cpp'): 261 if supportsVerify: 262 steps = [ 263 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 264 ] 265 else: 266 steps = [ 267 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 268 ] 269 return self._executeShTest(test, litConfig, steps) 270 else: 271 return lit.Test.Result(lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)) 272 273 def _executeShTest(self, test, litConfig, steps): 274 if test.config.unsupported: 275 return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported') 276 277 script = parseScript(test, steps) 278 if isinstance(script, lit.Test.Result): 279 return script 280 281 if litConfig.noExecute: 282 return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS) 283 else: 284 _, tmpBase = _getTempPaths(test) 285 useExternalSh = False 286 return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase) 287