1import copy 2import re 3 4import common 5 6from autotest_lib.client.common_lib import global_config 7from autotest_lib.frontend.afe import rpc_client_lib 8 9 10# Number of times to retry if a gs command fails. Defaults to 10, 11# which is far too long given that we already wait on these files 12# before starting HWTests. 13_GS_RETRIES = 1 14 15 16_HTTP_ERROR_THRESHOLD = 400 17BUG_CONFIG_SECTION = 'BUG_REPORTING' 18 19# global configurations needed for build artifacts 20_gs_domain = global_config.global_config.get_config_value( 21 BUG_CONFIG_SECTION, 'gs_domain', default='') 22_chromeos_image_archive = global_config.global_config.get_config_value( 23 BUG_CONFIG_SECTION, 'chromeos_image_archive', default='') 24_arg_prefix = global_config.global_config.get_config_value( 25 BUG_CONFIG_SECTION, 'arg_prefix', default='') 26 27 28# global configurations needed for results log 29_retrieve_logs_cgi = global_config.global_config.get_config_value( 30 BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='') 31_generic_results_bin = global_config.global_config.get_config_value( 32 BUG_CONFIG_SECTION, 'generic_results_bin', default='') 33_debug_dir = global_config.global_config.get_config_value( 34 BUG_CONFIG_SECTION, 'debug_dir', default='') 35 36 37# Template for the url used to generate the link to the job 38_job_view = global_config.global_config.get_config_value( 39 BUG_CONFIG_SECTION, 'job_view', default='') 40 41 42# gs prefix to perform file like operations (gs://) 43_gs_file_prefix = global_config.global_config.get_config_value( 44 BUG_CONFIG_SECTION, 'gs_file_prefix', default='') 45 46 47_CRBUG_URL = global_config.global_config.get_config_value( 48 BUG_CONFIG_SECTION, 'crbug_url') 49 50 51WMATRIX_RETRY_URL = global_config.global_config.get_config_value( 52 BUG_CONFIG_SECTION, 'wmatrix_retry_url', default='') 53WMATRIX_TEST_HISTORY_URL = global_config.global_config.get_config_value( 54 BUG_CONFIG_SECTION, 'wmatrix_test_history_url', default='') 55STAINLESS_RETRY_URL = global_config.global_config.get_config_value( 56 BUG_CONFIG_SECTION, 'stainless_retry_url', default='') 57STAINLESS_TEST_HISTORY_URL = global_config.global_config.get_config_value( 58 BUG_CONFIG_SECTION, 'stainless_test_history_url', default='') 59 60 61class InvalidBugTemplateException(Exception): 62 """Exception raised when a bug template is not valid, e.g., missing value 63 for essential attributes. 64 """ 65 pass 66 67 68class BugTemplate(object): 69 """Wrapper class to merge a suite and test bug templates, and do validation. 70 """ 71 72 # Names of expected attributes. 73 EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title', 74 'cc', 'summary', 'components'] 75 LIST_ATTRIBUTES = ['cc', 'labels'] 76 EMAIL_ATTRIBUTES = ['owner', 'cc'] 77 78 EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+') 79 80 81 def __init__(self, bug_template): 82 """Initialize a BugTemplate object. 83 84 @param bug_template: initial bug template, e.g., bug template from suite 85 control file. 86 """ 87 self.bug_template = self.cleanup_bug_template(bug_template) 88 89 90 @classmethod 91 def validate_bug_template(cls, bug_template): 92 """Verify if a bug template has value for all essential attributes. 93 94 @param bug_template: bug template to be verified. 95 @raise InvalidBugTemplateException: raised when a bug template 96 is invalid, e.g., has missing essential attribute, or any given 97 template is not a dictionary. 98 """ 99 if not type(bug_template) is dict: 100 raise InvalidBugTemplateException('Bug template must be a ' 101 'dictionary.') 102 103 unexpected_keys = [] 104 for key, value in bug_template.iteritems(): 105 if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES: 106 raise InvalidBugTemplateException('Key %s is not expected in ' 107 'bug template.' % key) 108 if (key in cls.LIST_ATTRIBUTES and 109 not isinstance(value, list)): 110 raise InvalidBugTemplateException('Value for %s must be a list.' 111 % key) 112 if key in cls.EMAIL_ATTRIBUTES: 113 emails = value if isinstance(value, list) else [value] 114 for email in emails: 115 if not email or not cls.EMAIL_REGEX.match(email): 116 raise InvalidBugTemplateException( 117 'Invalid email address: %s.' % email) 118 119 120 @classmethod 121 def cleanup_bug_template(cls, bug_template): 122 """Remove empty entries in given bug template. 123 124 @param bug_template: bug template to be verified. 125 126 @return: A cleaned up bug template. 127 @raise InvalidBugTemplateException: raised when a bug template 128 is not a dictionary. 129 """ 130 if not type(bug_template) is dict: 131 raise InvalidBugTemplateException('Bug template must be a ' 132 'dictionary.') 133 template = copy.deepcopy(bug_template) 134 # If owner or cc is set but the value is empty or None, remove it from 135 # the template. 136 for email_attribute in cls.EMAIL_ATTRIBUTES: 137 if email_attribute in template: 138 value = template[email_attribute] 139 if isinstance(value, list): 140 template[email_attribute] = [email for email in value 141 if email] 142 if not template[email_attribute]: 143 del(template[email_attribute]) 144 return template 145 146 147 def finalize_bug_template(self, test_template): 148 """Merge test and suite bug templates. 149 150 @param test_template: Bug template from test control file. 151 @return: Merged bug template. 152 153 @raise InvalidBugTemplateException: raised when the merged template is 154 invalid, e.g., has missing essential attribute, or any given 155 template is not a dictionary. 156 """ 157 test_template = self.cleanup_bug_template(test_template) 158 self.validate_bug_template(self.bug_template) 159 self.validate_bug_template(test_template) 160 161 merged_template = test_template 162 merged_template.update((k, v) for k, v in self.bug_template.iteritems() 163 if k not in merged_template) 164 165 # test_template wins for common keys, unless values are list that can be 166 # merged. 167 for key in set(merged_template.keys()).intersection( 168 self.bug_template.keys()): 169 if (type(merged_template[key]) is list and 170 type(self.bug_template[key]) is list): 171 merged_template[key] = (merged_template[key] + 172 self.bug_template[key]) 173 elif not merged_template[key]: 174 merged_template[key] = self.bug_template[key] 175 self.validate_bug_template(merged_template) 176 return merged_template 177 178 179def link_build_artifacts(build): 180 """Returns a url to build artifacts on google storage. 181 182 @param build: A string, e.g. stout32-release/R30-4433.0.0 183 184 @returns: A url to build artifacts on google storage. 185 186 """ 187 return (_gs_domain + _arg_prefix + 188 _chromeos_image_archive + build) 189 190 191def link_job(job_id, instance_server=None): 192 """Returns an url to the job on cautotest. 193 194 @param job_id: A string, representing the job id. 195 @param instance_server: The instance server. 196 Eg: cautotest, cautotest-cq, localhost. 197 198 @returns: An url to the job on cautotest. 199 200 """ 201 if not job_id: 202 return 'Job did not run, or was aborted prematurely' 203 if not instance_server: 204 instance_server = global_config.global_config.get_config_value( 205 'SERVER', 'hostname', default='localhost') 206 207 instance_server = rpc_client_lib.add_protocol(instance_server) 208 return _job_view % (instance_server, job_id) 209 210 211def _base_results_log(job_id, result_owner, hostname): 212 """Returns the base url of the job's results. 213 214 @param job_id: A string, representing the job id. 215 @param result_owner: A string, representing the onwer of the job. 216 @param hostname: A string, representing the host on which 217 the job has run. 218 219 @returns: The base url of the job's results. 220 221 """ 222 if job_id and result_owner and hostname: 223 path_to_object = '%s-%s/%s' % (job_id, result_owner, 224 hostname) 225 return (_retrieve_logs_cgi + _generic_results_bin + 226 path_to_object) 227 228 229def link_result_logs(job_id, result_owner, hostname): 230 """Returns a url to test logs on google storage. 231 232 @param job_id: A string, representing the job id. 233 @param result_owner: A string, representing the owner of the job. 234 @param hostname: A string, representing the host on which the 235 jot has run. 236 237 @returns: A url to test logs on google storage. 238 239 """ 240 base_results = _base_results_log(job_id, result_owner, hostname) 241 if base_results: 242 return '%s/%s' % (base_results, _debug_dir) 243 return ('Could not generate results log: the job with id %s, ' 244 'scheduled by: %s on host: %s did not run' % 245 (job_id, result_owner, hostname)) 246 247 248def link_status_log(job_id, result_owner, hostname): 249 """Returns an url to status log of the job. 250 251 @param job_id: A string, representing the job id. 252 @param result_owner: A string, representing the owner of the job. 253 @param hostname: A string, representing the host on which the 254 jot has run. 255 256 @returns: A url to status log of the job. 257 258 """ 259 base_results = _base_results_log(job_id, result_owner, hostname) 260 if base_results: 261 return '%s/%s' % (base_results, 'status.log') 262 return 'NA' 263 264 265def link_wmatrix_retry_url(test_name): 266 """Link to the wmatrix retry stats page for this test. 267 268 @param test_name: Test we want to search the retry stats page for. 269 270 @return: A link to the wmatrix retry stats dashboard for this test. 271 """ 272 return WMATRIX_RETRY_URL % test_name 273 274 275def link_retry_url(test_name): 276 """Link to the retry stats page for this test. 277 278 @param test_name: Test we want to search the retry stats page for. 279 280 @return: A link to the retry stats dashboard for this test. 281 """ 282 if STAINLESS_RETRY_URL: 283 args_dict = { 284 'test_name_re': '^%s$' % re.escape(test_name), 285 } 286 return STAINLESS_RETRY_URL % args_dict 287 return WMATRIX_RETRY_URL % test_name 288 289 290def link_test_history(test_name): 291 """Link to the test history page for this test. 292 293 @param test_name: Test we want to search the test history for. 294 295 @return: A link to the test history page for this test. 296 """ 297 if STAINLESS_TEST_HISTORY_URL: 298 args_dict = { 299 'test_name_re': '^%s$' % re.escape(test_name), 300 } 301 return STAINLESS_TEST_HISTORY_URL % args_dict 302 return WMATRIX_TEST_HISTORY_URL % test_name 303 304 305def link_crbug(bug_id): 306 """Generate a bug link for the given bug_id. 307 308 @param bug_id: The id of the bug. 309 @return: A link, eg: https://crbug.com/<bug_id>. 310 """ 311 return _CRBUG_URL % (bug_id,) 312