1# Lint as: python2, python3 2from __future__ import absolute_import 3from __future__ import division 4from __future__ import print_function 5 6import copy 7import datetime 8import re 9import six 10 11import common 12 13from autotest_lib.client.common_lib import global_config 14from autotest_lib.frontend.afe import rpc_client_lib 15 16 17# Number of times to retry if a gs command fails. Defaults to 10, 18# which is far too long given that we already wait on these files 19# before starting HWTests. 20_GS_RETRIES = 1 21 22 23_HTTP_ERROR_THRESHOLD = 400 24BUG_CONFIG_SECTION = 'BUG_REPORTING' 25 26# global configurations needed for build artifacts 27_gs_domain = global_config.global_config.get_config_value( 28 BUG_CONFIG_SECTION, 'gs_domain', default='') 29_chromeos_image_archive = global_config.global_config.get_config_value( 30 BUG_CONFIG_SECTION, 'chromeos_image_archive', default='') 31_arg_prefix = global_config.global_config.get_config_value( 32 BUG_CONFIG_SECTION, 'arg_prefix', default='') 33 34 35# global configurations needed for results log 36_retrieve_logs_cgi = global_config.global_config.get_config_value( 37 BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='') 38_generic_results_bin = global_config.global_config.get_config_value( 39 BUG_CONFIG_SECTION, 'generic_results_bin', default='') 40_debug_dir = global_config.global_config.get_config_value( 41 BUG_CONFIG_SECTION, 'debug_dir', default='') 42 43 44# Template for the url used to generate the link to the job 45_job_view = global_config.global_config.get_config_value( 46 BUG_CONFIG_SECTION, 'job_view', default='') 47 48 49# gs prefix to perform file like operations (gs://) 50_gs_file_prefix = global_config.global_config.get_config_value( 51 BUG_CONFIG_SECTION, 'gs_file_prefix', default='') 52 53 54_CRBUG_URL = global_config.global_config.get_config_value( 55 BUG_CONFIG_SECTION, 'crbug_url') 56 57 58WMATRIX_RETRY_URL = global_config.global_config.get_config_value( 59 BUG_CONFIG_SECTION, 'wmatrix_retry_url', default='') 60WMATRIX_TEST_HISTORY_URL = global_config.global_config.get_config_value( 61 BUG_CONFIG_SECTION, 'wmatrix_test_history_url', default='') 62STAINLESS_RETRY_URL = global_config.global_config.get_config_value( 63 BUG_CONFIG_SECTION, 'stainless_retry_url', default='') 64STAINLESS_TEST_HISTORY_URL = global_config.global_config.get_config_value( 65 BUG_CONFIG_SECTION, 'stainless_test_history_url', default='') 66 67 68class InvalidBugTemplateException(Exception): 69 """Exception raised when a bug template is not valid, e.g., missing value 70 for essential attributes. 71 """ 72 pass 73 74 75class BugTemplate(object): 76 """Wrapper class to merge a suite and test bug templates, and do validation. 77 """ 78 79 # Names of expected attributes. 80 EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title', 81 'cc', 'summary', 'components'] 82 LIST_ATTRIBUTES = ['cc', 'labels'] 83 EMAIL_ATTRIBUTES = ['owner', 'cc'] 84 85 EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+') 86 87 88 def __init__(self, bug_template): 89 """Initialize a BugTemplate object. 90 91 @param bug_template: initial bug template, e.g., bug template from suite 92 control file. 93 """ 94 self.bug_template = self.cleanup_bug_template(bug_template) 95 96 97 @classmethod 98 def validate_bug_template(cls, bug_template): 99 """Verify if a bug template has value for all essential attributes. 100 101 @param bug_template: bug template to be verified. 102 @raise InvalidBugTemplateException: raised when a bug template 103 is invalid, e.g., has missing essential attribute, or any given 104 template is not a dictionary. 105 """ 106 if not type(bug_template) is dict: 107 raise InvalidBugTemplateException('Bug template must be a ' 108 'dictionary.') 109 110 unexpected_keys = [] 111 for key, value in six.iteritems(bug_template): 112 if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES: 113 raise InvalidBugTemplateException('Key %s is not expected in ' 114 'bug template.' % key) 115 if (key in cls.LIST_ATTRIBUTES and 116 not isinstance(value, list)): 117 raise InvalidBugTemplateException('Value for %s must be a list.' 118 % key) 119 if key in cls.EMAIL_ATTRIBUTES: 120 emails = value if isinstance(value, list) else [value] 121 for email in emails: 122 if not email or not cls.EMAIL_REGEX.match(email): 123 raise InvalidBugTemplateException( 124 'Invalid email address: %s.' % email) 125 126 127 @classmethod 128 def cleanup_bug_template(cls, bug_template): 129 """Remove empty entries in given bug template. 130 131 @param bug_template: bug template to be verified. 132 133 @return: A cleaned up bug template. 134 @raise InvalidBugTemplateException: raised when a bug template 135 is not a dictionary. 136 """ 137 if not type(bug_template) is dict: 138 raise InvalidBugTemplateException('Bug template must be a ' 139 'dictionary.') 140 template = copy.deepcopy(bug_template) 141 # If owner or cc is set but the value is empty or None, remove it from 142 # the template. 143 for email_attribute in cls.EMAIL_ATTRIBUTES: 144 if email_attribute in template: 145 value = template[email_attribute] 146 if isinstance(value, list): 147 template[email_attribute] = [email for email in value 148 if email] 149 if not template[email_attribute]: 150 del(template[email_attribute]) 151 return template 152 153 154 def finalize_bug_template(self, test_template): 155 """Merge test and suite bug templates. 156 157 @param test_template: Bug template from test control file. 158 @return: Merged bug template. 159 160 @raise InvalidBugTemplateException: raised when the merged template is 161 invalid, e.g., has missing essential attribute, or any given 162 template is not a dictionary. 163 """ 164 test_template = self.cleanup_bug_template(test_template) 165 self.validate_bug_template(self.bug_template) 166 self.validate_bug_template(test_template) 167 168 merged_template = test_template 169 merged_template.update((k, v) for k, v in six.iteritems(self.bug_template) 170 if k not in merged_template) 171 172 # test_template wins for common keys, unless values are list that can be 173 # merged. 174 for key in set(merged_template.keys()).intersection( 175 list(self.bug_template.keys())): 176 if (type(merged_template[key]) is list and 177 type(self.bug_template[key]) is list): 178 merged_template[key] = (merged_template[key] + 179 self.bug_template[key]) 180 elif not merged_template[key]: 181 merged_template[key] = self.bug_template[key] 182 self.validate_bug_template(merged_template) 183 return merged_template 184 185 186def link_build_artifacts(build): 187 """Returns a url to build artifacts on google storage. 188 189 @param build: A string, e.g. stout32-release/R30-4433.0.0 190 191 @returns: A url to build artifacts on google storage. 192 193 """ 194 return (_gs_domain + _arg_prefix + 195 _chromeos_image_archive + build) 196 197 198def link_job(job_id, instance_server=None): 199 """Returns an url to the job on cautotest. 200 201 @param job_id: A string, representing the job id. 202 @param instance_server: The instance server. 203 Eg: cautotest, cautotest-cq, localhost. 204 205 @returns: An url to the job on cautotest. 206 207 """ 208 if not job_id: 209 return 'Job did not run, or was aborted prematurely' 210 if not instance_server: 211 instance_server = global_config.global_config.get_config_value( 212 'SERVER', 'hostname', default='localhost') 213 214 instance_server = rpc_client_lib.add_protocol(instance_server) 215 return _job_view % (instance_server, job_id) 216 217 218def _base_results_log(job_id, result_owner, hostname): 219 """Returns the base url of the job's results. 220 221 @param job_id: A string, representing the job id. 222 @param result_owner: A string, representing the onwer of the job. 223 @param hostname: A string, representing the host on which 224 the job has run. 225 226 @returns: The base url of the job's results. 227 228 """ 229 if job_id and result_owner and hostname: 230 path_to_object = '%s-%s/%s' % (job_id, result_owner, 231 hostname) 232 return (_retrieve_logs_cgi + _generic_results_bin + 233 path_to_object) 234 235 236def link_result_logs(job_id, result_owner, hostname): 237 """Returns a url to test logs on google storage. 238 239 @param job_id: A string, representing the job id. 240 @param result_owner: A string, representing the owner of the job. 241 @param hostname: A string, representing the host on which the 242 jot has run. 243 244 @returns: A url to test logs on google storage. 245 246 """ 247 base_results = _base_results_log(job_id, result_owner, hostname) 248 if base_results: 249 return '%s/%s' % (base_results, _debug_dir) 250 return ('Could not generate results log: the job with id %s, ' 251 'scheduled by: %s on host: %s did not run' % 252 (job_id, result_owner, hostname)) 253 254 255def link_status_log(job_id, result_owner, hostname): 256 """Returns an url to status log of the job. 257 258 @param job_id: A string, representing the job id. 259 @param result_owner: A string, representing the owner of the job. 260 @param hostname: A string, representing the host on which the 261 jot has run. 262 263 @returns: A url to status log of the job. 264 265 """ 266 base_results = _base_results_log(job_id, result_owner, hostname) 267 if base_results: 268 return '%s/%s' % (base_results, 'status.log') 269 return 'NA' 270 271 272def link_wmatrix_retry_url(test_name): 273 """Link to the wmatrix retry stats page for this test. 274 275 @param test_name: Test we want to search the retry stats page for. 276 277 @return: A link to the wmatrix retry stats dashboard for this test. 278 """ 279 return WMATRIX_RETRY_URL % test_name 280 281 282def link_retry_url(test_name): 283 """Link to the retry stats page for this test. 284 285 @param test_name: Test we want to search the retry stats page for. 286 287 @return: A link to the retry stats dashboard for this test. 288 """ 289 if STAINLESS_RETRY_URL: 290 args_dict = { 291 'test_name_re': '^%s$' % re.escape(test_name), 292 } 293 return STAINLESS_RETRY_URL % args_dict 294 return WMATRIX_RETRY_URL % test_name 295 296 297def link_test_history(test_name): 298 """Link to the test history page for this test. 299 300 @param test_name: Test we want to search the test history for. 301 302 @return: A link to the test history page for this test. 303 """ 304 date_format = '%Y-%m-%d' 305 now = datetime.datetime.utcnow() 306 last_date = now.strftime(date_format) 307 first_date = (now - datetime.timedelta(days=28)).strftime(date_format) 308 # Please note that stainless url doesn't work for tests whose test name is 309 # different from its job name. E.g. for moblab_RunSuite/control.dummyServer 310 # Its job name (NAME in control file) is moblab_DummyServerSuite. 311 # Its test name is moblab_RunSuite. 312 # Stainless use 'moblab_DummyServerSuite' as the test name, however, 313 # TKO uses 'moblab_RunSuite' as the test name. 314 return STAINLESS_TEST_HISTORY_URL % ( 315 '^%s$' % re.escape(test_name), first_date, last_date) 316 317 318def link_crbug(bug_id): 319 """Generate a bug link for the given bug_id. 320 321 @param bug_id: The id of the bug. 322 @return: A link, eg: https://crbug.com/<bug_id>. 323 """ 324 return _CRBUG_URL % (bug_id,) 325