1import os, shutil, tempfile, logging 2 3import common 4from autotest_lib.client.common_lib import utils, error, profiler_manager 5from autotest_lib.server import profiler, autotest, standalone_profiler 6 7 8PROFILER_TMPDIR = '/tmp/profilers' 9 10 11def get_profiler_results_dir(autodir): 12 """ 13 Given the directory of the autotest client used to run a profiler, 14 return the remote path where profiler results will be stored. 15 """ 16 return os.path.join(autodir, 'results', 'default', 'profiler_sync', 17 'profiling') 18 19 20def get_profiler_log_path(autodir): 21 """ 22 Given the directory of a profiler client, find the client log path. 23 """ 24 return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG') 25 26 27class profilers(profiler_manager.profiler_manager): 28 def __init__(self, job): 29 super(profilers, self).__init__(job) 30 self.add_log = {} 31 self.start_delay = 0 32 # maps hostname to (host object, autotest.Autotest object, Autotest 33 # install dir), where the host object is the one created specifically 34 # for profiling 35 self.installed_hosts = {} 36 self.current_test = None 37 38 39 def set_start_delay(self, start_delay): 40 self.start_delay = start_delay 41 42 43 def load_profiler(self, profiler_name, args, dargs): 44 newprofiler = profiler.profiler_proxy(profiler_name) 45 newprofiler.initialize(*args, **dargs) 46 newprofiler.setup(*args, **dargs) # lazy setup is done client-side 47 return newprofiler 48 49 50 def add(self, profiler, *args, **dargs): 51 super(profilers, self).add(profiler, *args, **dargs) 52 self.add_log[profiler] = (args, dargs) 53 54 55 def delete(self, profiler): 56 super(profilers, self).delete(profiler) 57 if profiler in self.add_log: 58 del self.add_log[profiler] 59 60 61 def _install_clients(self): 62 """ 63 Install autotest on any current job hosts. 64 """ 65 in_use_hosts = dict() 66 # find hosts in use but not used by us 67 for host in self.job.hosts: 68 if host.hostname not in self.job.machines: 69 # job.hosts include all host instances created on the fly. 70 # We only care DUTs in job.machines which are 71 # piped in from autoserv -m option. 72 continue 73 autodir = host.get_autodir() 74 if not (autodir and autodir.startswith(PROFILER_TMPDIR)): 75 in_use_hosts[host.hostname] = host 76 logging.debug('Hosts currently in use: %s', set(in_use_hosts)) 77 78 # determine what valid host objects we already have installed 79 profiler_hosts = set() 80 for host, at, profiler_dir in self.installed_hosts.values(): 81 if host.path_exists(profiler_dir): 82 profiler_hosts.add(host.hostname) 83 else: 84 # the profiler was wiped out somehow, drop this install 85 logging.warning('The profiler client on %s at %s was deleted', 86 host.hostname, profiler_dir) 87 del self.installed_hosts[host.hostname] 88 logging.debug('Hosts with profiler clients already installed: %s', 89 profiler_hosts) 90 91 # install autotest on any new hosts in use 92 for hostname in set(in_use_hosts) - profiler_hosts: 93 host = in_use_hosts[hostname] 94 tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR) 95 at = autotest.Autotest(host) 96 at.install_no_autoserv(autodir=tmp_dir) 97 self.installed_hosts[host.hostname] = (host, at, tmp_dir) 98 99 # drop any installs from hosts no longer in job.hosts 100 for hostname in profiler_hosts - set(in_use_hosts): 101 del self.installed_hosts[hostname] 102 103 104 def _get_hosts(self, host=None): 105 """ 106 Returns a list of (Host, Autotest, install directory) tuples for hosts 107 currently supported by this profiler. The returned Host object is always 108 the one created by this profiler, regardless of what's passed in. If 109 'host' is not None, all entries not matching that host object are 110 filtered out of the list. 111 """ 112 if host is None: 113 return self.installed_hosts.values() 114 if host.hostname in self.installed_hosts: 115 return [self.installed_hosts[host.hostname]] 116 return [] 117 118 119 def _get_local_profilers_dir(self, test, hostname): 120 in_machine_dir = ( 121 os.path.basename(test.job.resultdir) in test.job.machines) 122 if len(test.job.machines) > 1 and not in_machine_dir: 123 local_dir = os.path.join(test.profdir, hostname) 124 if not os.path.exists(local_dir): 125 os.makedirs(local_dir) 126 else: 127 local_dir = test.profdir 128 129 return local_dir 130 131 132 def _get_failure_logs(self, autodir, test, host): 133 """ 134 Collect the client logs from a profiler run and put them in a 135 file named failure-*.log. 136 """ 137 try: 138 fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-', 139 dir=self._get_local_profilers_dir(test, host.hostname)) 140 os.close(fd) 141 host.get_file(get_profiler_log_path(autodir), path) 142 # try to collect any partial profiler logs 143 self._get_profiler_logs(autodir, test, host) 144 except (error.AutotestError, error.AutoservError): 145 logging.exception('Profiler failure log collection failed') 146 # swallow the exception so that we don't override an existing 147 # exception being thrown 148 149 150 def _get_all_failure_logs(self, test, hosts): 151 for host, at, autodir in hosts: 152 self._get_failure_logs(autodir, test, host) 153 154 155 def _get_profiler_logs(self, autodir, test, host): 156 results_dir = get_profiler_results_dir(autodir) 157 local_dir = self._get_local_profilers_dir(test, host.hostname) 158 159 self.job.remove_client_log(host.hostname, results_dir, local_dir) 160 161 tempdir = tempfile.mkdtemp(dir=self.job.tmpdir) 162 try: 163 host.get_file(results_dir + '/', tempdir) 164 except error.AutoservRunError: 165 pass # no files to pull back, nothing we can do 166 utils.merge_trees(tempdir, local_dir) 167 shutil.rmtree(tempdir, ignore_errors=True) 168 169 170 def _run_clients(self, test, hosts): 171 """ 172 We initialize the profilers just before start because only then we 173 know all the hosts involved. 174 """ 175 176 hostnames = [host_info[0].hostname for host_info in hosts] 177 profilers_args = [(p.name, p.args, p.dargs) 178 for p in self.list] 179 180 for host, at, autodir in hosts: 181 control_script = standalone_profiler.generate_test(hostnames, 182 host.hostname, 183 profilers_args, 184 180, None) 185 try: 186 at.run(control_script, background=True) 187 except Exception: 188 self._get_failure_logs(autodir, test, host) 189 raise 190 191 remote_results_dir = get_profiler_results_dir(autodir) 192 local_results_dir = self._get_local_profilers_dir(test, 193 host.hostname) 194 self.job.add_client_log(host.hostname, remote_results_dir, 195 local_results_dir) 196 197 try: 198 # wait for the profilers to be added 199 standalone_profiler.wait_for_profilers(hostnames) 200 except Exception: 201 self._get_all_failure_logs(test, hosts) 202 raise 203 204 205 def before_start(self, test, host=None): 206 # create host objects and install the needed clients 207 # so later in start() we don't spend too much time 208 self._install_clients() 209 self._run_clients(test, self._get_hosts(host)) 210 211 212 def start(self, test, host=None): 213 hosts = self._get_hosts(host) 214 215 # wait for the profilers to start 216 hostnames = [host_info[0].hostname for host_info in hosts] 217 try: 218 standalone_profiler.start_profilers(hostnames) 219 except Exception: 220 self._get_all_failure_logs(test, hosts) 221 raise 222 223 self.current_test = test 224 225 226 def stop(self, test): 227 assert self.current_test == test 228 229 hosts = self._get_hosts() 230 # wait for the profilers to stop 231 hostnames = [host_info[0].hostname for host_info in hosts] 232 try: 233 standalone_profiler.stop_profilers(hostnames) 234 except Exception: 235 self._get_all_failure_logs(test, hosts) 236 raise 237 238 239 def report(self, test, host=None): 240 assert self.current_test == test 241 242 hosts = self._get_hosts(host) 243 # when running on specific hosts we cannot wait for the other 244 # hosts to sync with us 245 if not host: 246 hostnames = [host_info[0].hostname for host_info in hosts] 247 try: 248 standalone_profiler.finish_profilers(hostnames) 249 except Exception: 250 self._get_all_failure_logs(test, hosts) 251 raise 252 253 # pull back all the results 254 for host, at, autodir in hosts: 255 self._get_profiler_logs(autodir, test, host) 256 257 258 def handle_reboot(self, host): 259 if self.current_test: 260 test = self.current_test 261 for profiler in self.list: 262 if not profiler.supports_reboot: 263 msg = 'profiler %s does not support rebooting during tests' 264 msg %= profiler.name 265 self.job.record('WARN', os.path.basename(test.outputdir), 266 None, msg) 267 268 self.report(test, host) 269 self.before_start(test, host) 270 self.start(test, host) 271