1# ===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7# ===----------------------------------------------------------------------===## 8 9import lit 10import lit.formats 11import os 12import re 13 14 15def _getTempPaths(test): 16 """ 17 Return the values to use for the %T and %t substitutions, respectively. 18 19 The difference between this and Lit's default behavior is that we guarantee 20 that %T is a path unique to the test being run. 21 """ 22 tmpDir, _ = lit.TestRunner.getTempPaths(test) 23 _, testName = os.path.split(test.getExecPath()) 24 tmpDir = os.path.join(tmpDir, testName + ".dir") 25 tmpBase = os.path.join(tmpDir, "t") 26 return tmpDir, tmpBase 27 28 29def _checkBaseSubstitutions(substitutions): 30 substitutions = [s for (s, _) in substitutions] 31 for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]: 32 assert s in substitutions, "Required substitution {} was not provided".format(s) 33 34def _executeScriptInternal(test, litConfig, commands): 35 """ 36 Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands) 37 38 TODO: This really should be easier to access from Lit itself 39 """ 40 parsedCommands = parseScript(test, preamble=commands) 41 42 _, tmpBase = _getTempPaths(test) 43 execDir = os.path.dirname(test.getExecPath()) 44 try: 45 res = lit.TestRunner.executeScriptInternal( 46 test, litConfig, tmpBase, parsedCommands, execDir, debug=False 47 ) 48 except lit.TestRunner.ScriptFatal as e: 49 res = ("", str(e), 127, None) 50 (out, err, exitCode, timeoutInfo) = res 51 52 return (out, err, exitCode, timeoutInfo, parsedCommands) 53 54 55def parseScript(test, preamble): 56 """ 57 Extract the script from a test, with substitutions applied. 58 59 Returns a list of commands ready to be executed. 60 61 - test 62 The lit.Test to parse. 63 64 - preamble 65 A list of commands to perform before any command in the test. 66 These commands can contain unexpanded substitutions, but they 67 must not be of the form 'RUN:' -- they must be proper commands 68 once substituted. 69 """ 70 # Get the default substitutions 71 tmpDir, tmpBase = _getTempPaths(test) 72 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 73 74 # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions 75 # 76 # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as 77 # errors, which doesn't make sense for clang-verify tests because we may want to check 78 # for specific warning diagnostics. 79 _checkBaseSubstitutions(substitutions) 80 substitutions.append( 81 ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe") 82 ) 83 substitutions.append( 84 ( 85 "%{verify}", 86 "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0", 87 ) 88 ) 89 substitutions.append(("%{run}", "%{exec} %t.exe")) 90 91 # Parse the test file, including custom directives 92 additionalCompileFlags = [] 93 fileDependencies = [] 94 parsers = [ 95 lit.TestRunner.IntegratedTestKeywordParser( 96 "FILE_DEPENDENCIES:", 97 lit.TestRunner.ParserKind.LIST, 98 initial_value=fileDependencies, 99 ), 100 lit.TestRunner.IntegratedTestKeywordParser( 101 "ADDITIONAL_COMPILE_FLAGS:", 102 lit.TestRunner.ParserKind.SPACE_LIST, 103 initial_value=additionalCompileFlags, 104 ), 105 ] 106 107 # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first 108 # class support for conditional keywords in Lit, which would allow evaluating arbitrary 109 # Lit boolean expressions instead. 110 for feature in test.config.available_features: 111 parser = lit.TestRunner.IntegratedTestKeywordParser( 112 "ADDITIONAL_COMPILE_FLAGS({}):".format(feature), 113 lit.TestRunner.ParserKind.SPACE_LIST, 114 initial_value=additionalCompileFlags, 115 ) 116 parsers.append(parser) 117 118 scriptInTest = lit.TestRunner.parseIntegratedTestScript( 119 test, additional_parsers=parsers, require_script=not preamble 120 ) 121 if isinstance(scriptInTest, lit.Test.Result): 122 return scriptInTest 123 124 script = [] 125 126 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 127 # that file to the execution directory. Execute the copy from %S to allow 128 # relative paths from the test directory. 129 for dep in fileDependencies: 130 script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)] 131 script += preamble 132 script += scriptInTest 133 134 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 135 substitutions = [ 136 (s, x + " " + " ".join(additionalCompileFlags)) 137 if s == "%{compile_flags}" 138 else (s, x) 139 for (s, x) in substitutions 140 ] 141 142 # Perform substitutions in the script itself. 143 script = lit.TestRunner.applySubstitutions( 144 script, substitutions, recursion_limit=test.config.recursiveExpansionLimit 145 ) 146 147 return script 148 149 150class CxxStandardLibraryTest(lit.formats.FileBasedTest): 151 """ 152 Lit test format for the C++ Standard Library conformance test suite. 153 154 This test format is based on top of the ShTest format -- it basically 155 creates a shell script performing the right operations (compile/link/run) 156 based on the extension of the test file it encounters. It supports files 157 with the following extensions: 158 159 FOO.pass.cpp - Compiles, links and runs successfully 160 FOO.pass.mm - Same as .pass.cpp, but for Objective-C++ 161 162 FOO.compile.pass.cpp - Compiles successfully, link and run not attempted 163 FOO.compile.pass.mm - Same as .compile.pass.cpp, but for Objective-C++ 164 FOO.compile.fail.cpp - Does not compile successfully 165 166 FOO.link.pass.cpp - Compiles and links successfully, run not attempted 167 FOO.link.pass.mm - Same as .link.pass.cpp, but for Objective-C++ 168 FOO.link.fail.cpp - Compiles successfully, but fails to link 169 170 FOO.sh.<anything> - A builtin Lit Shell test 171 172 FOO.gen.<anything> - A .sh test that generates one or more Lit tests on the 173 fly. Executing this test must generate one or more files 174 as expected by LLVM split-file, and each generated file 175 leads to a separate Lit test that runs that file as 176 defined by the test format. This can be used to generate 177 multiple Lit tests from a single source file, which is 178 useful for testing repetitive properties in the library. 179 Be careful not to abuse this since this is not a replacement 180 for usual code reuse techniques. 181 182 FOO.verify.cpp - Compiles with clang-verify. This type of test is 183 automatically marked as UNSUPPORTED if the compiler 184 does not support Clang-verify. 185 186 187 Substitution requirements 188 =============================== 189 The test format operates by assuming that each test's configuration provides 190 the following substitutions, which it will reuse in the shell scripts it 191 constructs: 192 %{cxx} - A command that can be used to invoke the compiler 193 %{compile_flags} - Flags to use when compiling a test case 194 %{link_flags} - Flags to use when linking a test case 195 %{flags} - Flags to use either when compiling or linking a test case 196 %{exec} - A command to prefix the execution of executables 197 198 Note that when building an executable (as opposed to only compiling a source 199 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 200 in the same command line. In other words, the test format doesn't perform 201 separate compilation and linking steps in this case. 202 203 204 Additional supported directives 205 =============================== 206 In addition to everything that's supported in Lit ShTests, this test format 207 also understands the following directives inside test files: 208 209 // FILE_DEPENDENCIES: file, directory, /path/to/file 210 211 This directive expresses that the test requires the provided files 212 or directories in order to run. An example is a test that requires 213 some test input stored in a data file. When a test file contains 214 such a directive, this test format will collect them and copy them 215 to the directory represented by %T. The intent is that %T contains 216 all the inputs necessary to run the test, such that e.g. execution 217 on a remote host can be done by simply copying %T to the host. 218 219 // ADDITIONAL_COMPILE_FLAGS: flag1 flag2 flag3 220 221 This directive will cause the provided flags to be added to the 222 %{compile_flags} substitution for the test that contains it. This 223 allows adding special compilation flags without having to use a 224 .sh.cpp test, which would be more powerful but perhaps overkill. 225 226 227 Additional provided substitutions and features 228 ============================================== 229 The test format will define the following substitutions for use inside tests: 230 231 %{build} 232 Expands to a command-line that builds the current source 233 file with the %{flags}, %{compile_flags} and %{link_flags} 234 substitutions, and that produces an executable named %t.exe. 235 236 %{verify} 237 Expands to a command-line that builds the current source 238 file with the %{flags} and %{compile_flags} substitutions 239 and enables clang-verify. This can be used to write .sh.cpp 240 tests that use clang-verify. Note that this substitution can 241 only be used when the 'verify-support' feature is available. 242 243 %{run} 244 Equivalent to `%{exec} %t.exe`. This is intended to be used 245 in conjunction with the %{build} substitution. 246 """ 247 248 def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig): 249 SUPPORTED_SUFFIXES = [ 250 "[.]pass[.]cpp$", 251 "[.]pass[.]mm$", 252 "[.]compile[.]pass[.]cpp$", 253 "[.]compile[.]pass[.]mm$", 254 "[.]compile[.]fail[.]cpp$", 255 "[.]link[.]pass[.]cpp$", 256 "[.]link[.]pass[.]mm$", 257 "[.]link[.]fail[.]cpp$", 258 "[.]sh[.][^.]+$", 259 "[.]gen[.][^.]+$", 260 "[.]verify[.]cpp$", 261 "[.]fail[.]cpp$", 262 ] 263 264 sourcePath = testSuite.getSourcePath(pathInSuite) 265 filename = os.path.basename(sourcePath) 266 267 # Ignore dot files, excluded tests and tests with an unsupported suffix 268 hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES]) 269 if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename): 270 return 271 272 # If this is a generated test, run the generation step and add 273 # as many Lit tests as necessary. 274 if re.search('[.]gen[.][^.]+$', filename): 275 for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig): 276 yield test 277 else: 278 yield lit.Test.Test(testSuite, pathInSuite, localConfig) 279 280 def execute(self, test, litConfig): 281 supportsVerify = "verify-support" in test.config.available_features 282 filename = test.path_in_suite[-1] 283 284 if re.search("[.]sh[.][^.]+$", filename): 285 steps = [] # The steps are already in the script 286 return self._executeShTest(test, litConfig, steps) 287 elif filename.endswith(".compile.pass.cpp") or filename.endswith( 288 ".compile.pass.mm" 289 ): 290 steps = [ 291 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 292 ] 293 return self._executeShTest(test, litConfig, steps) 294 elif filename.endswith(".compile.fail.cpp"): 295 steps = [ 296 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 297 ] 298 return self._executeShTest(test, litConfig, steps) 299 elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"): 300 steps = [ 301 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 302 ] 303 return self._executeShTest(test, litConfig, steps) 304 elif filename.endswith(".link.fail.cpp"): 305 steps = [ 306 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 307 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe", 308 ] 309 return self._executeShTest(test, litConfig, steps) 310 elif filename.endswith(".verify.cpp"): 311 if not supportsVerify: 312 return lit.Test.Result( 313 lit.Test.UNSUPPORTED, 314 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format( 315 test.getFullName() 316 ), 317 ) 318 steps = ["%dbg(COMPILED WITH) %{verify}"] 319 return self._executeShTest(test, litConfig, steps) 320 # Make sure to check these ones last, since they will match other 321 # suffixes above too. 322 elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"): 323 steps = [ 324 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 325 "%dbg(EXECUTED AS) %{exec} %t.exe", 326 ] 327 return self._executeShTest(test, litConfig, steps) 328 else: 329 return lit.Test.Result( 330 lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename) 331 ) 332 333 def _executeShTest(self, test, litConfig, steps): 334 if test.config.unsupported: 335 return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported") 336 337 script = parseScript(test, steps) 338 if isinstance(script, lit.Test.Result): 339 return script 340 341 if litConfig.noExecute: 342 return lit.Test.Result( 343 lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS 344 ) 345 else: 346 _, tmpBase = _getTempPaths(test) 347 useExternalSh = False 348 return lit.TestRunner._runShTest( 349 test, litConfig, useExternalSh, script, tmpBase 350 ) 351 352 def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): 353 generator = lit.Test.Test(testSuite, pathInSuite, localConfig) 354 355 # Make sure we have a directory to execute the generator test in 356 generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) 357 os.makedirs(generatorExecDir, exist_ok=True) 358 359 # Run the generator test 360 steps = [] # Steps must already be in the script 361 (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) 362 if exitCode != 0: 363 raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") 364 365 # Split the generated output into multiple files and generate one test for each file 366 for subfile, content in self._splitFile(out): 367 generatedFile = testSuite.getExecPath(pathInSuite + (subfile,)) 368 os.makedirs(os.path.dirname(generatedFile), exist_ok=True) 369 with open(generatedFile, 'w') as f: 370 f.write(content) 371 yield lit.Test.Test(testSuite, (generatedFile,), localConfig) 372 373 def _splitFile(self, input): 374 DELIM = r'^(//|#)---(.+)' 375 lines = input.splitlines() 376 currentFile = None 377 thisFileContent = [] 378 for line in lines: 379 match = re.match(DELIM, line) 380 if match: 381 if currentFile is not None: 382 yield (currentFile, '\n'.join(thisFileContent)) 383 currentFile = match.group(2).strip() 384 thisFileContent = [] 385 assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}" 386 thisFileContent.append(line) 387 if currentFile is not None: 388 yield (currentFile, '\n'.join(thisFileContent)) 389