# Lint as: python2, python3 import os, shutil, tempfile, logging import common from autotest_lib.client.common_lib import utils, error, profiler_manager from autotest_lib.server import profiler, autotest, standalone_profiler PROFILER_TMPDIR = '/tmp/profilers' def get_profiler_results_dir(autodir): """ Given the directory of the autotest client used to run a profiler, return the remote path where profiler results will be stored. """ return os.path.join(autodir, 'results', 'default', 'profiler_sync', 'profiling') def get_profiler_log_path(autodir): """ Given the directory of a profiler client, find the client log path. """ return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG') class profilers(profiler_manager.profiler_manager): def __init__(self, job): super(profilers, self).__init__(job) self.add_log = {} self.start_delay = 0 # maps hostname to (host object, autotest.Autotest object, Autotest # install dir), where the host object is the one created specifically # for profiling self.installed_hosts = {} self.current_test = None def set_start_delay(self, start_delay): self.start_delay = start_delay def load_profiler(self, profiler_name, args, dargs): newprofiler = profiler.profiler_proxy(profiler_name) newprofiler.initialize(*args, **dargs) newprofiler.setup(*args, **dargs) # lazy setup is done client-side return newprofiler def add(self, profiler, *args, **dargs): super(profilers, self).add(profiler, *args, **dargs) self.add_log[profiler] = (args, dargs) def delete(self, profiler): super(profilers, self).delete(profiler) if profiler in self.add_log: del self.add_log[profiler] def _install_clients(self): """ Install autotest on any current job hosts. """ in_use_hosts = dict() # find hosts in use but not used by us for host in self.job.hosts: if host.hostname not in self.job.machines: # job.hosts include all host instances created on the fly. # We only care DUTs in job.machines which are # piped in from autoserv -m option. continue autodir = host.get_autodir() if not (autodir and autodir.startswith(PROFILER_TMPDIR)): in_use_hosts[host.hostname] = host logging.debug('Hosts currently in use: %s', set(in_use_hosts)) # determine what valid host objects we already have installed profiler_hosts = set() for host, at, profiler_dir in self.installed_hosts.values(): if host.path_exists(profiler_dir): profiler_hosts.add(host.hostname) else: # the profiler was wiped out somehow, drop this install logging.warning('The profiler client on %s at %s was deleted', host.hostname, profiler_dir) del self.installed_hosts[host.hostname] logging.debug('Hosts with profiler clients already installed: %s', profiler_hosts) # install autotest on any new hosts in use for hostname in set(in_use_hosts) - profiler_hosts: host = in_use_hosts[hostname] tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR) at = autotest.Autotest(host) at.install_no_autoserv(autodir=tmp_dir) self.installed_hosts[host.hostname] = (host, at, tmp_dir) # drop any installs from hosts no longer in job.hosts for hostname in profiler_hosts - set(in_use_hosts): del self.installed_hosts[hostname] def _get_hosts(self, host=None): """ Returns a list of (Host, Autotest, install directory) tuples for hosts currently supported by this profiler. The returned Host object is always the one created by this profiler, regardless of what's passed in. If 'host' is not None, all entries not matching that host object are filtered out of the list. """ if host is None: return list(self.installed_hosts.values()) if host.hostname in self.installed_hosts: return [self.installed_hosts[host.hostname]] return [] def _get_local_profilers_dir(self, test, hostname): in_machine_dir = ( os.path.basename(test.job.resultdir) in test.job.machines) if len(test.job.machines) > 1 and not in_machine_dir: local_dir = os.path.join(test.profdir, hostname) if not os.path.exists(local_dir): os.makedirs(local_dir) else: local_dir = test.profdir return local_dir def _get_failure_logs(self, autodir, test, host): """ Collect the client logs from a profiler run and put them in a file named failure-*.log. """ try: fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-', dir=self._get_local_profilers_dir(test, host.hostname)) os.close(fd) host.get_file(get_profiler_log_path(autodir), path) # try to collect any partial profiler logs self._get_profiler_logs(autodir, test, host) except (error.AutotestError, error.AutoservError): logging.exception('Profiler failure log collection failed') # swallow the exception so that we don't override an existing # exception being thrown def _get_all_failure_logs(self, test, hosts): for host, at, autodir in hosts: self._get_failure_logs(autodir, test, host) def _get_profiler_logs(self, autodir, test, host): results_dir = get_profiler_results_dir(autodir) local_dir = self._get_local_profilers_dir(test, host.hostname) self.job.remove_client_log(host.hostname, results_dir, local_dir) tempdir = tempfile.mkdtemp(dir=self.job.tmpdir) try: host.get_file(results_dir + '/', tempdir) except error.AutoservRunError: pass # no files to pull back, nothing we can do utils.merge_trees(tempdir, local_dir) shutil.rmtree(tempdir, ignore_errors=True) def _run_clients(self, test, hosts): """ We initialize the profilers just before start because only then we know all the hosts involved. """ hostnames = [host_info[0].hostname for host_info in hosts] profilers_args = [(p.name, p.args, p.dargs) for p in self.list] for host, at, autodir in hosts: control_script = standalone_profiler.generate_test(hostnames, host.hostname, profilers_args, 180, None) try: at.run(control_script, background=True) except Exception: self._get_failure_logs(autodir, test, host) raise remote_results_dir = get_profiler_results_dir(autodir) local_results_dir = self._get_local_profilers_dir(test, host.hostname) self.job.add_client_log(host.hostname, remote_results_dir, local_results_dir) try: # wait for the profilers to be added standalone_profiler.wait_for_profilers(hostnames) except Exception: self._get_all_failure_logs(test, hosts) raise def before_start(self, test, host=None): # create host objects and install the needed clients # so later in start() we don't spend too much time self._install_clients() self._run_clients(test, self._get_hosts(host)) def start(self, test, host=None): hosts = self._get_hosts(host) # wait for the profilers to start hostnames = [host_info[0].hostname for host_info in hosts] try: standalone_profiler.start_profilers(hostnames) except Exception: self._get_all_failure_logs(test, hosts) raise self.current_test = test def stop(self, test): assert self.current_test == test hosts = self._get_hosts() # wait for the profilers to stop hostnames = [host_info[0].hostname for host_info in hosts] try: standalone_profiler.stop_profilers(hostnames) except Exception: self._get_all_failure_logs(test, hosts) raise def report(self, test, host=None): assert self.current_test == test hosts = self._get_hosts(host) # when running on specific hosts we cannot wait for the other # hosts to sync with us if not host: hostnames = [host_info[0].hostname for host_info in hosts] try: standalone_profiler.finish_profilers(hostnames) except Exception: self._get_all_failure_logs(test, hosts) raise # pull back all the results for host, at, autodir in hosts: self._get_profiler_logs(autodir, test, host) def handle_reboot(self, host): if self.current_test: test = self.current_test for profiler in self.list: if not profiler.supports_reboot: msg = 'profiler %s does not support rebooting during tests' msg %= profiler.name self.job.record('WARN', os.path.basename(test.outputdir), None, msg) self.report(test, host) self.before_start(test, host) self.start(test, host)