1#===----------------------------------------------------------------------===## 2# 3# The LLVM Compiler Infrastructure 4# 5# This file is dual licensed under the MIT and the University of Illinois Open 6# Source Licenses. See LICENSE.TXT for details. 7# 8#===----------------------------------------------------------------------===## 9 10import platform 11import os 12 13from libcxx.test import tracing 14from libcxx.util import executeCommand 15 16 17class Executor(object): 18 def run(self, exe_path, cmd, local_cwd, file_deps=None, env=None): 19 """Execute a command. 20 Be very careful not to change shared state in this function. 21 Executor objects are shared between python processes in `lit -jN`. 22 Args: 23 exe_path: str: Local path to the executable to be run 24 cmd: [str]: subprocess.call style command 25 local_cwd: str: Local path to the working directory 26 file_deps: [str]: Files required by the test 27 env: {str: str}: Environment variables to execute under 28 Returns: 29 cmd, out, err, exitCode 30 """ 31 raise NotImplementedError 32 33 34class LocalExecutor(Executor): 35 def __init__(self): 36 super(LocalExecutor, self).__init__() 37 self.is_windows = platform.system() == 'Windows' 38 39 def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): 40 cmd = cmd or [exe_path] 41 env_cmd = [] 42 if env: 43 env_cmd += ['env'] 44 env_cmd += ['%s=%s' % (k, v) for k, v in env.items()] 45 if work_dir == '.': 46 work_dir = os.getcwd() 47 if not self.is_windows: 48 out, err, rc = executeCommand(env_cmd + cmd, cwd=work_dir) 49 else: 50 out, err, rc = executeCommand(cmd, cwd=work_dir, 51 env=self._build_windows_env(env)) 52 return (env_cmd + cmd, out, err, rc) 53 54 def _build_windows_env(self, exec_env): 55 # FIXME: Finding Windows DLL's at runtime requires modifying the 56 # PATH environment variables. However we don't want to print out 57 # the entire PATH as part of the diagnostic for every failing test. 58 # Therefore this hack builds a new executable environment that 59 # merges the current environment and the supplied environment while 60 # still only printing the supplied environment in diagnostics. 61 if not self.is_windows or exec_env is None: 62 return None 63 new_env = dict(os.environ) 64 for key, value in exec_env.items(): 65 if key == 'PATH': 66 assert value.strip() != '' and "expected non-empty path" 67 new_env['PATH'] = "%s;%s" % (value, os.environ['PATH']) 68 else: 69 new_env[key] = value 70 return new_env 71 72class PrefixExecutor(Executor): 73 """Prefix an executor with some other command wrapper. 74 75 Most useful for setting ulimits on commands, or running an emulator like 76 qemu and valgrind. 77 """ 78 def __init__(self, commandPrefix, chain): 79 super(PrefixExecutor, self).__init__() 80 81 self.commandPrefix = commandPrefix 82 self.chain = chain 83 84 def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): 85 cmd = cmd or [exe_path] 86 return self.chain.run(exe_path, self.commandPrefix + cmd, work_dir, 87 file_deps, env=env) 88 89 90class PostfixExecutor(Executor): 91 """Postfix an executor with some args.""" 92 def __init__(self, commandPostfix, chain): 93 super(PostfixExecutor, self).__init__() 94 95 self.commandPostfix = commandPostfix 96 self.chain = chain 97 98 def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): 99 cmd = cmd or [exe_path] 100 return self.chain.run(cmd + self.commandPostfix, work_dir, file_deps, 101 env=env) 102 103 104 105class TimeoutExecutor(PrefixExecutor): 106 """Execute another action under a timeout. 107 108 Deprecated. http://reviews.llvm.org/D6584 adds timeouts to LIT. 109 """ 110 def __init__(self, duration, chain): 111 super(TimeoutExecutor, self).__init__( 112 ['timeout', duration], chain) 113 114 115class RemoteExecutor(Executor): 116 def __init__(self): 117 self.local_run = executeCommand 118 119 def remote_temp_dir(self): 120 return self._remote_temp(True) 121 122 def remote_temp_file(self): 123 return self._remote_temp(False) 124 125 def _remote_temp(self, is_dir): 126 raise NotImplementedError() 127 128 def copy_in(self, local_srcs, remote_dsts): 129 # This could be wrapped up in a tar->scp->untar for performance 130 # if there are lots of files to be copied/moved 131 for src, dst in zip(local_srcs, remote_dsts): 132 self._copy_in_file(src, dst) 133 134 def _copy_in_file(self, src, dst): 135 raise NotImplementedError() 136 137 def delete_remote(self, remote): 138 try: 139 self._execute_command_remote(['rm', '-rf', remote]) 140 except OSError: 141 # TODO: Log failure to delete? 142 pass 143 144 def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): 145 target_exe_path = None 146 target_cwd = None 147 try: 148 target_cwd = self.remote_temp_dir() 149 target_exe_path = os.path.join(target_cwd, 'libcxx_test.exe') 150 if cmd: 151 # Replace exe_path with target_exe_path. 152 cmd = [c if c != exe_path else target_exe_path for c in cmd] 153 else: 154 cmd = [target_exe_path] 155 156 srcs = [exe_path] 157 dsts = [target_exe_path] 158 if file_deps is not None: 159 dev_paths = [os.path.join(target_cwd, os.path.basename(f)) 160 for f in file_deps] 161 srcs.extend(file_deps) 162 dsts.extend(dev_paths) 163 self.copy_in(srcs, dsts) 164 # TODO(jroelofs): capture the copy_in and delete_remote commands, 165 # and conjugate them with '&&'s around the first tuple element 166 # returned here: 167 return self._execute_command_remote(cmd, target_cwd, env) 168 finally: 169 if target_cwd: 170 self.delete_remote(target_cwd) 171 172 def _execute_command_remote(self, cmd, remote_work_dir='.', env=None): 173 raise NotImplementedError() 174 175 176class SSHExecutor(RemoteExecutor): 177 def __init__(self, host, username=None): 178 super(SSHExecutor, self).__init__() 179 180 self.user_prefix = username + '@' if username else '' 181 self.host = host 182 self.scp_command = 'scp' 183 self.ssh_command = 'ssh' 184 185 # TODO(jroelofs): switch this on some -super-verbose-debug config flag 186 if False: 187 self.local_run = tracing.trace_function( 188 self.local_run, log_calls=True, log_results=True, 189 label='ssh_local') 190 191 def _remote_temp(self, is_dir): 192 # TODO: detect what the target system is, and use the correct 193 # mktemp command for it. (linux and darwin differ here, and I'm 194 # sure windows has another way to do it) 195 196 # Not sure how to do suffix on osx yet 197 dir_arg = '-d' if is_dir else '' 198 cmd = 'mktemp -q {} /tmp/libcxx.XXXXXXXXXX'.format(dir_arg) 199 temp_path, err, exitCode = self._execute_command_remote([cmd]) 200 temp_path = temp_path.strip() 201 if exitCode != 0: 202 raise RuntimeError(err) 203 return temp_path 204 205 def _copy_in_file(self, src, dst): 206 scp = self.scp_command 207 remote = self.host 208 remote = self.user_prefix + remote 209 cmd = [scp, '-p', src, remote + ':' + dst] 210 self.local_run(cmd) 211 212 def _execute_command_remote(self, cmd, remote_work_dir='.', env=None): 213 remote = self.user_prefix + self.host 214 ssh_cmd = [self.ssh_command, '-oBatchMode=yes', remote] 215 if env: 216 env_cmd = ['env'] + ['%s=%s' % (k, v) for k, v in env.items()] 217 else: 218 env_cmd = [] 219 remote_cmd = ' '.join(env_cmd + cmd) 220 if remote_work_dir != '.': 221 remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd 222 return self.local_run(ssh_cmd + [remote_cmd]) 223