1import copy 2import datetime 3import re 4 5import common 6 7from autotest_lib.client.common_lib import global_config 8from autotest_lib.frontend.afe import rpc_client_lib 9 10 11# Number of times to retry if a gs command fails. Defaults to 10, 12# which is far too long given that we already wait on these files 13# before starting HWTests. 14_GS_RETRIES = 1 15 16 17_HTTP_ERROR_THRESHOLD = 400 18BUG_CONFIG_SECTION = 'BUG_REPORTING' 19 20# global configurations needed for build artifacts 21_gs_domain = global_config.global_config.get_config_value( 22 BUG_CONFIG_SECTION, 'gs_domain', default='') 23_chromeos_image_archive = global_config.global_config.get_config_value( 24 BUG_CONFIG_SECTION, 'chromeos_image_archive', default='') 25_arg_prefix = global_config.global_config.get_config_value( 26 BUG_CONFIG_SECTION, 'arg_prefix', default='') 27 28 29# global configurations needed for results log 30_retrieve_logs_cgi = global_config.global_config.get_config_value( 31 BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='') 32_generic_results_bin = global_config.global_config.get_config_value( 33 BUG_CONFIG_SECTION, 'generic_results_bin', default='') 34_debug_dir = global_config.global_config.get_config_value( 35 BUG_CONFIG_SECTION, 'debug_dir', default='') 36 37 38# Template for the url used to generate the link to the job 39_job_view = global_config.global_config.get_config_value( 40 BUG_CONFIG_SECTION, 'job_view', default='') 41 42 43# gs prefix to perform file like operations (gs://) 44_gs_file_prefix = global_config.global_config.get_config_value( 45 BUG_CONFIG_SECTION, 'gs_file_prefix', default='') 46 47 48_CRBUG_URL = global_config.global_config.get_config_value( 49 BUG_CONFIG_SECTION, 'crbug_url') 50 51 52WMATRIX_RETRY_URL = global_config.global_config.get_config_value( 53 BUG_CONFIG_SECTION, 'wmatrix_retry_url', default='') 54WMATRIX_TEST_HISTORY_URL = global_config.global_config.get_config_value( 55 BUG_CONFIG_SECTION, 'wmatrix_test_history_url', default='') 56STAINLESS_RETRY_URL = global_config.global_config.get_config_value( 57 BUG_CONFIG_SECTION, 'stainless_retry_url', default='') 58STAINLESS_TEST_HISTORY_URL = global_config.global_config.get_config_value( 59 BUG_CONFIG_SECTION, 'stainless_test_history_url', default='') 60 61 62class InvalidBugTemplateException(Exception): 63 """Exception raised when a bug template is not valid, e.g., missing value 64 for essential attributes. 65 """ 66 pass 67 68 69class BugTemplate(object): 70 """Wrapper class to merge a suite and test bug templates, and do validation. 71 """ 72 73 # Names of expected attributes. 74 EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title', 75 'cc', 'summary', 'components'] 76 LIST_ATTRIBUTES = ['cc', 'labels'] 77 EMAIL_ATTRIBUTES = ['owner', 'cc'] 78 79 EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+') 80 81 82 def __init__(self, bug_template): 83 """Initialize a BugTemplate object. 84 85 @param bug_template: initial bug template, e.g., bug template from suite 86 control file. 87 """ 88 self.bug_template = self.cleanup_bug_template(bug_template) 89 90 91 @classmethod 92 def validate_bug_template(cls, bug_template): 93 """Verify if a bug template has value for all essential attributes. 94 95 @param bug_template: bug template to be verified. 96 @raise InvalidBugTemplateException: raised when a bug template 97 is invalid, e.g., has missing essential attribute, or any given 98 template is not a dictionary. 99 """ 100 if not type(bug_template) is dict: 101 raise InvalidBugTemplateException('Bug template must be a ' 102 'dictionary.') 103 104 unexpected_keys = [] 105 for key, value in bug_template.iteritems(): 106 if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES: 107 raise InvalidBugTemplateException('Key %s is not expected in ' 108 'bug template.' % key) 109 if (key in cls.LIST_ATTRIBUTES and 110 not isinstance(value, list)): 111 raise InvalidBugTemplateException('Value for %s must be a list.' 112 % key) 113 if key in cls.EMAIL_ATTRIBUTES: 114 emails = value if isinstance(value, list) else [value] 115 for email in emails: 116 if not email or not cls.EMAIL_REGEX.match(email): 117 raise InvalidBugTemplateException( 118 'Invalid email address: %s.' % email) 119 120 121 @classmethod 122 def cleanup_bug_template(cls, bug_template): 123 """Remove empty entries in given bug template. 124 125 @param bug_template: bug template to be verified. 126 127 @return: A cleaned up bug template. 128 @raise InvalidBugTemplateException: raised when a bug template 129 is not a dictionary. 130 """ 131 if not type(bug_template) is dict: 132 raise InvalidBugTemplateException('Bug template must be a ' 133 'dictionary.') 134 template = copy.deepcopy(bug_template) 135 # If owner or cc is set but the value is empty or None, remove it from 136 # the template. 137 for email_attribute in cls.EMAIL_ATTRIBUTES: 138 if email_attribute in template: 139 value = template[email_attribute] 140 if isinstance(value, list): 141 template[email_attribute] = [email for email in value 142 if email] 143 if not template[email_attribute]: 144 del(template[email_attribute]) 145 return template 146 147 148 def finalize_bug_template(self, test_template): 149 """Merge test and suite bug templates. 150 151 @param test_template: Bug template from test control file. 152 @return: Merged bug template. 153 154 @raise InvalidBugTemplateException: raised when the merged template is 155 invalid, e.g., has missing essential attribute, or any given 156 template is not a dictionary. 157 """ 158 test_template = self.cleanup_bug_template(test_template) 159 self.validate_bug_template(self.bug_template) 160 self.validate_bug_template(test_template) 161 162 merged_template = test_template 163 merged_template.update((k, v) for k, v in self.bug_template.iteritems() 164 if k not in merged_template) 165 166 # test_template wins for common keys, unless values are list that can be 167 # merged. 168 for key in set(merged_template.keys()).intersection( 169 self.bug_template.keys()): 170 if (type(merged_template[key]) is list and 171 type(self.bug_template[key]) is list): 172 merged_template[key] = (merged_template[key] + 173 self.bug_template[key]) 174 elif not merged_template[key]: 175 merged_template[key] = self.bug_template[key] 176 self.validate_bug_template(merged_template) 177 return merged_template 178 179 180def link_build_artifacts(build): 181 """Returns a url to build artifacts on google storage. 182 183 @param build: A string, e.g. stout32-release/R30-4433.0.0 184 185 @returns: A url to build artifacts on google storage. 186 187 """ 188 return (_gs_domain + _arg_prefix + 189 _chromeos_image_archive + build) 190 191 192def link_job(job_id, instance_server=None): 193 """Returns an url to the job on cautotest. 194 195 @param job_id: A string, representing the job id. 196 @param instance_server: The instance server. 197 Eg: cautotest, cautotest-cq, localhost. 198 199 @returns: An url to the job on cautotest. 200 201 """ 202 if not job_id: 203 return 'Job did not run, or was aborted prematurely' 204 if not instance_server: 205 instance_server = global_config.global_config.get_config_value( 206 'SERVER', 'hostname', default='localhost') 207 208 instance_server = rpc_client_lib.add_protocol(instance_server) 209 return _job_view % (instance_server, job_id) 210 211 212def _base_results_log(job_id, result_owner, hostname): 213 """Returns the base url of the job's results. 214 215 @param job_id: A string, representing the job id. 216 @param result_owner: A string, representing the onwer of the job. 217 @param hostname: A string, representing the host on which 218 the job has run. 219 220 @returns: The base url of the job's results. 221 222 """ 223 if job_id and result_owner and hostname: 224 path_to_object = '%s-%s/%s' % (job_id, result_owner, 225 hostname) 226 return (_retrieve_logs_cgi + _generic_results_bin + 227 path_to_object) 228 229 230def link_result_logs(job_id, result_owner, hostname): 231 """Returns a url to test logs on google storage. 232 233 @param job_id: A string, representing the job id. 234 @param result_owner: A string, representing the owner of the job. 235 @param hostname: A string, representing the host on which the 236 jot has run. 237 238 @returns: A url to test logs on google storage. 239 240 """ 241 base_results = _base_results_log(job_id, result_owner, hostname) 242 if base_results: 243 return '%s/%s' % (base_results, _debug_dir) 244 return ('Could not generate results log: the job with id %s, ' 245 'scheduled by: %s on host: %s did not run' % 246 (job_id, result_owner, hostname)) 247 248 249def link_status_log(job_id, result_owner, hostname): 250 """Returns an url to status log of the job. 251 252 @param job_id: A string, representing the job id. 253 @param result_owner: A string, representing the owner of the job. 254 @param hostname: A string, representing the host on which the 255 jot has run. 256 257 @returns: A url to status log of the job. 258 259 """ 260 base_results = _base_results_log(job_id, result_owner, hostname) 261 if base_results: 262 return '%s/%s' % (base_results, 'status.log') 263 return 'NA' 264 265 266def link_wmatrix_retry_url(test_name): 267 """Link to the wmatrix retry stats page for this test. 268 269 @param test_name: Test we want to search the retry stats page for. 270 271 @return: A link to the wmatrix retry stats dashboard for this test. 272 """ 273 return WMATRIX_RETRY_URL % test_name 274 275 276def link_retry_url(test_name): 277 """Link to the retry stats page for this test. 278 279 @param test_name: Test we want to search the retry stats page for. 280 281 @return: A link to the retry stats dashboard for this test. 282 """ 283 if STAINLESS_RETRY_URL: 284 args_dict = { 285 'test_name_re': '^%s$' % re.escape(test_name), 286 } 287 return STAINLESS_RETRY_URL % args_dict 288 return WMATRIX_RETRY_URL % test_name 289 290 291def link_test_history(test_name): 292 """Link to the test history page for this test. 293 294 @param test_name: Test we want to search the test history for. 295 296 @return: A link to the test history page for this test. 297 """ 298 date_format = '%Y-%m-%d' 299 now = datetime.datetime.utcnow() 300 last_date = now.strftime(date_format) 301 first_date = (now - datetime.timedelta(days=28)).strftime(date_format) 302 # Please note that stainless url doesn't work for tests whose test name is 303 # different from its job name. E.g. for moblab_RunSuite/control.dummyServer 304 # Its job name (NAME in control file) is moblab_DummyServerSuite. 305 # Its test name is moblab_RunSuite. 306 # Stainless use 'moblab_DummyServerSuite' as the test name, however, 307 # TKO uses 'moblab_RunSuite' as the test name. 308 return STAINLESS_TEST_HISTORY_URL % ( 309 '^%s$' % re.escape(test_name), first_date, last_date) 310 311 312def link_crbug(bug_id): 313 """Generate a bug link for the given bug_id. 314 315 @param bug_id: The id of the bug. 316 @return: A link, eg: https://crbug.com/<bug_id>. 317 """ 318 return _CRBUG_URL % (bug_id,) 319