#!/usr/bin/python import cgi, os, sys, urllib2 import common from multiprocessing import pool from autotest_lib.frontend import setup_django_environment from autotest_lib.client.common_lib import global_config from autotest_lib.client.bin import utils from autotest_lib.frontend.afe.json_rpc import serviceHandler from autotest_lib.server import system_utils from autotest_lib.server import utils as server_utils _PAGE = """\ Status: 302 Found Content-Type: text/plain Location: %s\r\n\r """ GOOGLE_STORAGE_PATTERN = 'pantheon.corp.google.com/storage/browser/' # Define function for retrieving logs def _retrieve_logs_dummy(job_path): pass site_retrieve_logs = utils.import_site_function(__file__, "autotest_lib.tko.site_retrieve_logs", "site_retrieve_logs", _retrieve_logs_dummy) site_find_repository_host = utils.import_site_function(__file__, "autotest_lib.tko.site_retrieve_logs", "site_find_repository_host", _retrieve_logs_dummy) form = cgi.FieldStorage(keep_blank_values=True) # determine if this is a JSON-RPC request. we support both so that the new TKO # client can use its RPC client code, but the old TKO can still use simple GET # params. _is_json_request = form.has_key('callback') # if this key exists, we check if requested log exists in local machine, # and do not return Google Storage URL when the log doesn't exist. _local_only = form.has_key('localonly') def _get_requested_path(): if _is_json_request: request_data = form['request'].value request = serviceHandler.ServiceHandler.translateRequest(request_data) parameters = request['params'][0] return parameters['path'] return form['job'].value def _check_result(args): host = args['host'] job_path = args['job_path'] shard = args['shard'] if shard: http_path = 'http://%s/tko/retrieve_logs.cgi?localonly&job=%s' % ( host, job_path) else: http_path = 'http://%s%s' % (host, job_path) try: utils.urlopen(http_path) # On Vms the shard name is set to the default gateway but the # browser used to navigate frontends (that runs on the host of # the vms) is immune to the same NAT routing the vms have, so we # need to replace the gateway with 'localhost'. if utils.DEFAULT_VM_GATEWAY in host: normalized_host = host.replace(utils.DEFAULT_VM_GATEWAY, 'localhost') else: normalized_host = utils.normalize_hostname(host) return 'http', normalized_host, job_path except urllib2.URLError: return None def _get_tpool_args(hosts, job_path, is_shard, host_set): """Get a list of arguments to be passed to multiprocessing.pool.ThreadPool. @param hosts: a list of host names. @param job_path: a requested job path. @param is_shard: True if hosts are shards, False otherwise. @param host_set: a Set to filter out duplicated hosts. @return: a list of dictionaries to be used as input of _check_result(). """ args = [] for host in hosts: host = host.strip() if host and host != 'localhost' and host not in host_set: host_set.add(host) arg = {'host': host, 'job_path': job_path, 'shard': is_shard} args.append(arg) return args def find_repository_host(job_path): """Find the machine holding the given logs and return a URL to the logs""" site_repo_info = site_find_repository_host(job_path) if site_repo_info is not None: return site_repo_info # This cgi script is run only in master (cautotest) and shards. # Drones do not run this script when receiving '/results/...' request. # Only master should check drones and shards for the requested log. # Also restricted users do not have access to drones or shards, # always point them to localhost or google storage. if (not server_utils.is_shard() and not server_utils.is_restricted_user(os.environ.get('REMOTE_USER'))): drones = system_utils.get_drones() shards = system_utils.get_shards() host_set = set() tpool_args = _get_tpool_args(drones, job_path, False, host_set) tpool_args += _get_tpool_args(shards, job_path, True, host_set) tpool = pool.ThreadPool() for result_path in tpool.imap_unordered(_check_result, tpool_args): if result_path: return result_path # If the URL requested is a test result, it is now either on the local # host or in Google Storage. if job_path.startswith('/results/'): # We only care about the path after '/results/'. job_relative_path = job_path[9:] if not _local_only and not os.path.exists( os.path.join('/usr/local/autotest/results', job_relative_path)): gsuri = utils.get_offload_gsuri().split('gs://')[1] return ['https', GOOGLE_STORAGE_PATTERN, gsuri + job_relative_path] def get_full_url(info, log_path): if info is not None: protocol, host, path = info prefix = '%s://%s' % (protocol, host) else: prefix = '' path = log_path if _is_json_request: return '%s/tko/jsonp_fetcher.cgi?%s' % (prefix, os.environ['QUERY_STRING']) else: return prefix + path log_path = _get_requested_path() info = find_repository_host(log_path) site_retrieve_logs(log_path) print _PAGE % get_full_url(info, log_path)