1import errno 2import os 3import time 4 5import lit.Test # pylint: disable=import-error 6import lit.TestRunner # pylint: disable=import-error 7import lit.util # pylint: disable=import-error 8 9from libcxx.test.executor import LocalExecutor as LocalExecutor 10import libcxx.util 11 12 13class LibcxxTestFormat(object): 14 """ 15 Custom test format handler for use with the test format use by libc++. 16 17 Tests fall into two categories: 18 FOO.pass.cpp - Executable test which should compile, run, and exit with 19 code 0. 20 FOO.fail.cpp - Negative test case which is expected to fail compilation. 21 FOO.sh.cpp - A test that uses LIT's ShTest format. 22 """ 23 24 def __init__(self, cxx, use_verify_for_fail, execute_external, 25 executor, exec_env): 26 self.cxx = cxx 27 self.use_verify_for_fail = use_verify_for_fail 28 self.execute_external = execute_external 29 self.executor = executor 30 self.exec_env = dict(exec_env) 31 32 # TODO: Move this into lit's FileBasedTest 33 def getTestsInDirectory(self, testSuite, path_in_suite, 34 litConfig, localConfig): 35 source_path = testSuite.getSourcePath(path_in_suite) 36 for filename in os.listdir(source_path): 37 # Ignore dot files and excluded tests. 38 if filename.startswith('.') or filename in localConfig.excludes: 39 continue 40 41 filepath = os.path.join(source_path, filename) 42 if not os.path.isdir(filepath): 43 if any([filename.endswith(ext) 44 for ext in localConfig.suffixes]): 45 yield lit.Test.Test(testSuite, path_in_suite + (filename,), 46 localConfig) 47 48 def execute(self, test, lit_config): 49 while True: 50 try: 51 return self._execute(test, lit_config) 52 except OSError as oe: 53 if oe.errno != errno.ETXTBSY: 54 raise 55 time.sleep(0.1) 56 57 def _execute(self, test, lit_config): 58 name = test.path_in_suite[-1] 59 is_sh_test = name.endswith('.sh.cpp') 60 is_pass_test = name.endswith('.pass.cpp') 61 is_fail_test = name.endswith('.fail.cpp') 62 63 if test.config.unsupported: 64 return (lit.Test.UNSUPPORTED, 65 "A lit.local.cfg marked this unsupported") 66 67 res = lit.TestRunner.parseIntegratedTestScript( 68 test, require_script=is_sh_test) 69 # Check if a result for the test was returned. If so return that 70 # result. 71 if isinstance(res, lit.Test.Result): 72 return res 73 if lit_config.noExecute: 74 return lit.Test.Result(lit.Test.PASS) 75 # res is not an instance of lit.test.Result. Expand res into its parts. 76 script, tmpBase, execDir = res 77 # Check that we don't have run lines on tests that don't support them. 78 if not is_sh_test and len(script) != 0: 79 lit_config.fatal('Unsupported RUN line found in test %s' % name) 80 81 # Dispatch the test based on its suffix. 82 if is_sh_test: 83 if not isinstance(self.executor, LocalExecutor): 84 # We can't run ShTest tests with a executor yet. 85 # For now, bail on trying to run them 86 return lit.Test.UNSUPPORTED, 'ShTest format not yet supported' 87 return lit.TestRunner._runShTest(test, lit_config, 88 self.execute_external, script, 89 tmpBase, execDir) 90 elif is_fail_test: 91 return self._evaluate_fail_test(test) 92 elif is_pass_test: 93 return self._evaluate_pass_test(test, tmpBase, execDir, lit_config) 94 else: 95 # No other test type is supported 96 assert False 97 98 def _clean(self, exec_path): # pylint: disable=no-self-use 99 libcxx.util.cleanFile(exec_path) 100 101 def _evaluate_pass_test(self, test, tmpBase, execDir, lit_config): 102 source_path = test.getSourcePath() 103 exec_path = tmpBase + '.exe' 104 object_path = tmpBase + '.o' 105 # Create the output directory if it does not already exist. 106 lit.util.mkdir_p(os.path.dirname(tmpBase)) 107 try: 108 # Compile the test 109 cmd, out, err, rc = self.cxx.compileLinkTwoSteps( 110 source_path, out=exec_path, object_file=object_path, 111 cwd=execDir) 112 compile_cmd = cmd 113 if rc != 0: 114 report = libcxx.util.makeReport(cmd, out, err, rc) 115 report += "Compilation failed unexpectedly!" 116 return lit.Test.FAIL, report 117 # Run the test 118 local_cwd = os.path.dirname(source_path) 119 env = None 120 if self.exec_env: 121 env = self.exec_env 122 # TODO: Only list actually needed files in file_deps. 123 # Right now we just mark all of the .dat files in the same 124 # directory as dependencies, but it's likely less than that. We 125 # should add a `// FILE-DEP: foo.dat` to each test to track this. 126 data_files = [os.path.join(local_cwd, f) 127 for f in os.listdir(local_cwd) if f.endswith('.dat')] 128 cmd, out, err, rc = self.executor.run(exec_path, [exec_path], 129 local_cwd, data_files, env) 130 if rc != 0: 131 report = libcxx.util.makeReport(cmd, out, err, rc) 132 report = "Compiled With: %s\n%s" % (compile_cmd, report) 133 report += "Compiled test failed unexpectedly!" 134 return lit.Test.FAIL, report 135 return lit.Test.PASS, '' 136 finally: 137 # Note that cleanup of exec_file happens in `_clean()`. If you 138 # override this, cleanup is your reponsibility. 139 libcxx.util.cleanFile(object_path) 140 self._clean(exec_path) 141 142 def _evaluate_fail_test(self, test): 143 source_path = test.getSourcePath() 144 with open(source_path, 'r') as f: 145 contents = f.read() 146 verify_tags = ['expected-note', 'expected-remark', 'expected-warning', 147 'expected-error', 'expected-no-diagnostics'] 148 use_verify = self.use_verify_for_fail and \ 149 any([tag in contents for tag in verify_tags]) 150 extra_flags = [] 151 if use_verify: 152 extra_flags += ['-Xclang', '-verify', 153 '-Xclang', '-verify-ignore-unexpected=note'] 154 cmd, out, err, rc = self.cxx.compile(source_path, out=os.devnull, 155 flags=extra_flags) 156 expected_rc = 0 if use_verify else 1 157 if rc == expected_rc: 158 return lit.Test.PASS, '' 159 else: 160 report = libcxx.util.makeReport(cmd, out, err, rc) 161 report_msg = ('Expected compilation to fail!' if not use_verify else 162 'Expected compilation using verify to pass!') 163 return lit.Test.FAIL, report + report_msg + '\n' 164