1# Lint as: python2, python3 2import os, shutil, tempfile, logging 3 4import common 5from autotest_lib.client.common_lib import utils, error, profiler_manager 6from autotest_lib.server import profiler, autotest, standalone_profiler 7 8 9PROFILER_TMPDIR = '/tmp/profilers' 10 11 12def get_profiler_results_dir(autodir): 13 """ 14 Given the directory of the autotest client used to run a profiler, 15 return the remote path where profiler results will be stored. 16 """ 17 return os.path.join(autodir, 'results', 'default', 'profiler_sync', 18 'profiling') 19 20 21def get_profiler_log_path(autodir): 22 """ 23 Given the directory of a profiler client, find the client log path. 24 """ 25 return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG') 26 27 28class profilers(profiler_manager.profiler_manager): 29 def __init__(self, job): 30 super(profilers, self).__init__(job) 31 self.add_log = {} 32 self.start_delay = 0 33 # maps hostname to (host object, autotest.Autotest object, Autotest 34 # install dir), where the host object is the one created specifically 35 # for profiling 36 self.installed_hosts = {} 37 self.current_test = None 38 39 40 def set_start_delay(self, start_delay): 41 self.start_delay = start_delay 42 43 44 def load_profiler(self, profiler_name, args, dargs): 45 newprofiler = profiler.profiler_proxy(profiler_name) 46 newprofiler.initialize(*args, **dargs) 47 newprofiler.setup(*args, **dargs) # lazy setup is done client-side 48 return newprofiler 49 50 51 def add(self, profiler, *args, **dargs): 52 super(profilers, self).add(profiler, *args, **dargs) 53 self.add_log[profiler] = (args, dargs) 54 55 56 def delete(self, profiler): 57 super(profilers, self).delete(profiler) 58 if profiler in self.add_log: 59 del self.add_log[profiler] 60 61 62 def _install_clients(self): 63 """ 64 Install autotest on any current job hosts. 65 """ 66 in_use_hosts = dict() 67 # find hosts in use but not used by us 68 for host in self.job.hosts: 69 if host.hostname not in self.job.machines: 70 # job.hosts include all host instances created on the fly. 71 # We only care DUTs in job.machines which are 72 # piped in from autoserv -m option. 73 continue 74 autodir = host.get_autodir() 75 if not (autodir and autodir.startswith(PROFILER_TMPDIR)): 76 in_use_hosts[host.hostname] = host 77 logging.debug('Hosts currently in use: %s', set(in_use_hosts)) 78 79 # determine what valid host objects we already have installed 80 profiler_hosts = set() 81 for host, at, profiler_dir in self.installed_hosts.values(): 82 if host.path_exists(profiler_dir): 83 profiler_hosts.add(host.hostname) 84 else: 85 # the profiler was wiped out somehow, drop this install 86 logging.warning('The profiler client on %s at %s was deleted', 87 host.hostname, profiler_dir) 88 del self.installed_hosts[host.hostname] 89 logging.debug('Hosts with profiler clients already installed: %s', 90 profiler_hosts) 91 92 # install autotest on any new hosts in use 93 for hostname in set(in_use_hosts) - profiler_hosts: 94 host = in_use_hosts[hostname] 95 tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR) 96 at = autotest.Autotest(host) 97 at.install_no_autoserv(autodir=tmp_dir) 98 self.installed_hosts[host.hostname] = (host, at, tmp_dir) 99 100 # drop any installs from hosts no longer in job.hosts 101 for hostname in profiler_hosts - set(in_use_hosts): 102 del self.installed_hosts[hostname] 103 104 105 def _get_hosts(self, host=None): 106 """ 107 Returns a list of (Host, Autotest, install directory) tuples for hosts 108 currently supported by this profiler. The returned Host object is always 109 the one created by this profiler, regardless of what's passed in. If 110 'host' is not None, all entries not matching that host object are 111 filtered out of the list. 112 """ 113 if host is None: 114 return list(self.installed_hosts.values()) 115 if host.hostname in self.installed_hosts: 116 return [self.installed_hosts[host.hostname]] 117 return [] 118 119 120 def _get_local_profilers_dir(self, test, hostname): 121 in_machine_dir = ( 122 os.path.basename(test.job.resultdir) in test.job.machines) 123 if len(test.job.machines) > 1 and not in_machine_dir: 124 local_dir = os.path.join(test.profdir, hostname) 125 if not os.path.exists(local_dir): 126 os.makedirs(local_dir) 127 else: 128 local_dir = test.profdir 129 130 return local_dir 131 132 133 def _get_failure_logs(self, autodir, test, host): 134 """ 135 Collect the client logs from a profiler run and put them in a 136 file named failure-*.log. 137 """ 138 try: 139 fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-', 140 dir=self._get_local_profilers_dir(test, host.hostname)) 141 os.close(fd) 142 host.get_file(get_profiler_log_path(autodir), path) 143 # try to collect any partial profiler logs 144 self._get_profiler_logs(autodir, test, host) 145 except (error.AutotestError, error.AutoservError): 146 logging.exception('Profiler failure log collection failed') 147 # swallow the exception so that we don't override an existing 148 # exception being thrown 149 150 151 def _get_all_failure_logs(self, test, hosts): 152 for host, at, autodir in hosts: 153 self._get_failure_logs(autodir, test, host) 154 155 156 def _get_profiler_logs(self, autodir, test, host): 157 results_dir = get_profiler_results_dir(autodir) 158 local_dir = self._get_local_profilers_dir(test, host.hostname) 159 160 self.job.remove_client_log(host.hostname, results_dir, local_dir) 161 162 tempdir = tempfile.mkdtemp(dir=self.job.tmpdir) 163 try: 164 host.get_file(results_dir + '/', tempdir) 165 except error.AutoservRunError: 166 pass # no files to pull back, nothing we can do 167 utils.merge_trees(tempdir, local_dir) 168 shutil.rmtree(tempdir, ignore_errors=True) 169 170 171 def _run_clients(self, test, hosts): 172 """ 173 We initialize the profilers just before start because only then we 174 know all the hosts involved. 175 """ 176 177 hostnames = [host_info[0].hostname for host_info in hosts] 178 profilers_args = [(p.name, p.args, p.dargs) 179 for p in self.list] 180 181 for host, at, autodir in hosts: 182 control_script = standalone_profiler.generate_test(hostnames, 183 host.hostname, 184 profilers_args, 185 180, None) 186 try: 187 at.run(control_script, background=True) 188 except Exception: 189 self._get_failure_logs(autodir, test, host) 190 raise 191 192 remote_results_dir = get_profiler_results_dir(autodir) 193 local_results_dir = self._get_local_profilers_dir(test, 194 host.hostname) 195 self.job.add_client_log(host.hostname, remote_results_dir, 196 local_results_dir) 197 198 try: 199 # wait for the profilers to be added 200 standalone_profiler.wait_for_profilers(hostnames) 201 except Exception: 202 self._get_all_failure_logs(test, hosts) 203 raise 204 205 206 def before_start(self, test, host=None): 207 # create host objects and install the needed clients 208 # so later in start() we don't spend too much time 209 self._install_clients() 210 self._run_clients(test, self._get_hosts(host)) 211 212 213 def start(self, test, host=None): 214 hosts = self._get_hosts(host) 215 216 # wait for the profilers to start 217 hostnames = [host_info[0].hostname for host_info in hosts] 218 try: 219 standalone_profiler.start_profilers(hostnames) 220 except Exception: 221 self._get_all_failure_logs(test, hosts) 222 raise 223 224 self.current_test = test 225 226 227 def stop(self, test): 228 assert self.current_test == test 229 230 hosts = self._get_hosts() 231 # wait for the profilers to stop 232 hostnames = [host_info[0].hostname for host_info in hosts] 233 try: 234 standalone_profiler.stop_profilers(hostnames) 235 except Exception: 236 self._get_all_failure_logs(test, hosts) 237 raise 238 239 240 def report(self, test, host=None): 241 assert self.current_test == test 242 243 hosts = self._get_hosts(host) 244 # when running on specific hosts we cannot wait for the other 245 # hosts to sync with us 246 if not host: 247 hostnames = [host_info[0].hostname for host_info in hosts] 248 try: 249 standalone_profiler.finish_profilers(hostnames) 250 except Exception: 251 self._get_all_failure_logs(test, hosts) 252 raise 253 254 # pull back all the results 255 for host, at, autodir in hosts: 256 self._get_profiler_logs(autodir, test, host) 257 258 259 def handle_reboot(self, host): 260 if self.current_test: 261 test = self.current_test 262 for profiler in self.list: 263 if not profiler.supports_reboot: 264 msg = 'profiler %s does not support rebooting during tests' 265 msg %= profiler.name 266 self.job.record('WARN', os.path.basename(test.outputdir), 267 None, msg) 268 269 self.report(test, host) 270 self.before_start(test, host) 271 self.start(test, host) 272