1#!/usr/bin/python 2 3import cgi, os, socket, sys, urllib2 4import common 5from multiprocessing import pool 6from autotest_lib.frontend import setup_django_environment 7 8from autotest_lib.client.common_lib import global_config 9from autotest_lib.client.bin import utils 10from autotest_lib.frontend.afe.json_rpc import serviceHandler 11from autotest_lib.server import system_utils 12from autotest_lib.server import utils as server_utils 13 14 15_PAGE = """\ 16Status: 302 Found 17Content-Type: text/plain 18Location: %s\r\n\r 19""" 20 21GOOGLE_STORAGE_PATTERN = 'pantheon.corp.google.com/storage/browser/' 22 23# Define function for retrieving logs 24def _retrieve_logs_dummy(job_path): 25 pass 26 27site_retrieve_logs = utils.import_site_function(__file__, 28 "autotest_lib.tko.site_retrieve_logs", "site_retrieve_logs", 29 _retrieve_logs_dummy) 30 31site_find_repository_host = utils.import_site_function(__file__, 32 "autotest_lib.tko.site_retrieve_logs", "site_find_repository_host", 33 _retrieve_logs_dummy) 34 35form = cgi.FieldStorage(keep_blank_values=True) 36# determine if this is a JSON-RPC request. we support both so that the new TKO 37# client can use its RPC client code, but the old TKO can still use simple GET 38# params. 39_is_json_request = form.has_key('callback') 40 41# if this key exists, we check if requested log exists in local machine, 42# and do not return Google Storage URL when the log doesn't exist. 43_local_only = form.has_key('localonly') 44 45 46def _get_requested_path(): 47 if _is_json_request: 48 request_data = form['request'].value 49 request = serviceHandler.ServiceHandler.translateRequest(request_data) 50 parameters = request['params'][0] 51 return parameters['path'] 52 53 return form['job'].value 54 55 56def _check_result(args): 57 host = args['host'] 58 job_path = args['job_path'] 59 shard = args['shard'] 60 if shard: 61 http_path = 'http://%s/tko/retrieve_logs.cgi?localonly&job=%s' % ( 62 host, job_path) 63 else: 64 http_path = 'http://%s%s' % (host, job_path) 65 66 try: 67 utils.urlopen(http_path) 68 69 # On Vms the shard name is set to the default gateway but the 70 # browser used to navigate frontends (that runs on the host of 71 # the vms) is immune to the same NAT routing the vms have, so we 72 # need to replace the gateway with 'localhost'. 73 if utils.DEFAULT_VM_GATEWAY in host: 74 normalized_host = host.replace(utils.DEFAULT_VM_GATEWAY, 'localhost') 75 else: 76 try: 77 normalized_host = utils.normalize_hostname(host) 78 except socket.herror: 79 # Ignore error: 'socket.herror: [Errno 1] Unknown host' 80 # This can happen when reverse name lookup is not stable. 81 normalized_host = host 82 return 'http', normalized_host, job_path 83 except urllib2.URLError: 84 return None 85 86 87def _get_tpool_args(hosts, job_path, is_shard, host_set): 88 """Get a list of arguments to be passed to multiprocessing.pool.ThreadPool. 89 90 @param hosts: a list of host names. 91 @param job_path: a requested job path. 92 @param is_shard: True if hosts are shards, False otherwise. 93 @param host_set: a Set to filter out duplicated hosts. 94 95 @return: a list of dictionaries to be used as input of _check_result(). 96 """ 97 args = [] 98 for host in hosts: 99 host = host.strip() 100 if host and host != 'localhost' and host not in host_set: 101 host_set.add(host) 102 arg = {'host': host, 'job_path': job_path, 'shard': is_shard} 103 args.append(arg) 104 return args 105 106 107def find_repository_host(job_path): 108 """Find the machine holding the given logs and return a URL to the logs""" 109 site_repo_info = site_find_repository_host(job_path) 110 if site_repo_info is not None: 111 return site_repo_info 112 113 # This cgi script is run only in master (cautotest) and shards. 114 # Drones do not run this script when receiving '/results/...' request. 115 # Only master should check drones and shards for the requested log. 116 # Also restricted users do not have access to drones or shards, 117 # always point them to localhost or google storage. 118 if (not server_utils.is_shard() and 119 not server_utils.is_restricted_user(os.environ.get('REMOTE_USER'))): 120 drones = system_utils.get_drones() 121 shards = system_utils.get_shards() 122 123 host_set = set() 124 tpool_args = _get_tpool_args(drones, job_path, False, host_set) 125 tpool_args += _get_tpool_args(shards, job_path, True, host_set) 126 127 tpool = pool.ThreadPool() 128 for result_path in tpool.imap_unordered(_check_result, tpool_args): 129 if result_path: 130 return result_path 131 132 # If the URL requested is a test result, it is now either on the local 133 # host or in Google Storage. 134 if job_path.startswith('/results/'): 135 # We only care about the path after '/results/'. 136 job_relative_path = job_path[9:] 137 if not _local_only and not os.path.exists( 138 os.path.join('/usr/local/autotest/results', 139 job_relative_path)): 140 gsuri = utils.get_offload_gsuri().split('gs://')[1] 141 return ['https', GOOGLE_STORAGE_PATTERN, gsuri + job_relative_path] 142 143 144def get_full_url(info, log_path): 145 if info is not None: 146 protocol, host, path = info 147 prefix = '%s://%s' % (protocol, host) 148 else: 149 prefix = '' 150 path = log_path 151 152 if _is_json_request: 153 return '%s/tko/jsonp_fetcher.cgi?%s' % (prefix, 154 os.environ['QUERY_STRING']) 155 else: 156 return prefix + path 157 158 159log_path = _get_requested_path() 160info = find_repository_host(log_path) 161site_retrieve_logs(log_path) 162print _PAGE % get_full_url(info, log_path) 163