1# 2# Copyright (C) 2017 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import base64 18import getpass 19import logging 20import os 21import socket 22import time 23 24from vts.proto import VtsReportMessage_pb2 as ReportMsg 25from vts.runners.host import keys 26from vts.utils.python.web import dashboard_rest_client 27from vts.utils.python.web import feature_utils 28 29_PROFILING_POINTS = "profiling_points" 30 31 32class WebFeature(feature_utils.Feature): 33 """Feature object for web functionality. 34 35 Attributes: 36 enabled: boolean, True if web feature is enabled, False otherwise 37 report_msg: TestReportMessage, Proto summarizing the test run 38 current_test_report_msg: TestCaseReportMessage, Proto summarizing the current test case 39 rest_client: DashboardRestClient, client to which data will be posted 40 """ 41 42 _TOGGLE_PARAM = keys.ConfigKeys.IKEY_ENABLE_WEB 43 _REQUIRED_PARAMS = [ 44 keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND, 45 keys.ConfigKeys.IKEY_SERVICE_JSON_PATH, 46 keys.ConfigKeys.KEY_TESTBED_NAME, keys.ConfigKeys.IKEY_BUILD, 47 keys.ConfigKeys.IKEY_ANDROID_DEVICE, keys.ConfigKeys.IKEY_ABI_NAME, 48 keys.ConfigKeys.IKEY_ABI_BITNESS 49 ] 50 _OPTIONAL_PARAMS = [ 51 keys.ConfigKeys.RUN_AS_VTS_SELFTEST, 52 keys.ConfigKeys.IKEY_ENABLE_PROFILING, 53 ] 54 55 def __init__(self, user_params): 56 """Initializes the web feature. 57 58 Parses the arguments and initializes the web functionality. 59 60 Args: 61 user_params: A dictionary from parameter name (String) to parameter value. 62 """ 63 self.ParseParameters( 64 toggle_param_name=self._TOGGLE_PARAM, 65 required_param_names=self._REQUIRED_PARAMS, 66 optional_param_names=self._OPTIONAL_PARAMS, 67 user_params=user_params) 68 if not self.enabled: 69 return 70 71 # Initialize the dashboard client 72 post_cmd = getattr(self, keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND) 73 service_json_path = str( 74 getattr(self, keys.ConfigKeys.IKEY_SERVICE_JSON_PATH)) 75 try: 76 self.rest_client = dashboard_rest_client.DashboardRestClient( 77 post_cmd, service_json_path) 78 if not self.rest_client.Initialize(): 79 self.enabled = False 80 except Exception as e: 81 logging.exception("Failed to create REST client.") 82 self.enabled = False 83 84 self.report_msg = ReportMsg.TestReportMessage() 85 self.report_msg.test = str( 86 getattr(self, keys.ConfigKeys.KEY_TESTBED_NAME)) 87 88 if getattr(self, keys.ConfigKeys.IKEY_ENABLE_PROFILING, False): 89 self.report_msg.test += "Profiling" 90 91 self.report_msg.test_type = ReportMsg.VTS_HOST_DRIVEN_STRUCTURAL 92 self.report_msg.start_timestamp = feature_utils.GetTimestamp() 93 self.report_msg.host_info.hostname = socket.gethostname() 94 95 android_devices = getattr(self, keys.ConfigKeys.IKEY_ANDROID_DEVICE, 96 None) 97 if not android_devices or not isinstance(android_devices, list): 98 logging.warn("android device information not available") 99 return 100 101 for device_spec in android_devices: 102 dev_info = self.report_msg.device_info.add() 103 for elem in [ 104 keys.ConfigKeys.IKEY_PRODUCT_TYPE, 105 keys.ConfigKeys.IKEY_PRODUCT_VARIANT, 106 keys.ConfigKeys.IKEY_BUILD_FLAVOR, 107 keys.ConfigKeys.IKEY_BUILD_ID, keys.ConfigKeys.IKEY_BRANCH, 108 keys.ConfigKeys.IKEY_BUILD_ALIAS, 109 keys.ConfigKeys.IKEY_API_LEVEL, keys.ConfigKeys.IKEY_SERIAL 110 ]: 111 if elem in device_spec: 112 setattr(dev_info, elem, str(device_spec[elem])) 113 # TODO: get abi information differently for multi-device support. 114 setattr(dev_info, keys.ConfigKeys.IKEY_ABI_NAME, 115 str(getattr(self, keys.ConfigKeys.IKEY_ABI_NAME))) 116 setattr(dev_info, keys.ConfigKeys.IKEY_ABI_BITNESS, 117 str(getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS))) 118 119 def SetTestResult(self, result=None): 120 """Set the current test case result to the provided result. 121 122 If None is provided as a result, the current test report will be cleared, which results 123 in a silent skip. 124 125 Requires the feature to be enabled; no-op otherwise. 126 127 Args: 128 result: ReportMsg.TestCaseResult, the result of the current test or None. 129 """ 130 if not self.enabled: 131 return 132 133 if not result: 134 self.report_msg.test_case.remove(self.current_test_report_msg) 135 self.current_test_report_msg = None 136 else: 137 self.current_test_report_msg.test_result = result 138 139 def AddTestReport(self, test_name): 140 """Creates a report for the specified test. 141 142 Requires the feature to be enabled; no-op otherwise. 143 144 Args: 145 test_name: String, the name of the test 146 """ 147 if not self.enabled: 148 return 149 self.current_test_report_msg = self.report_msg.test_case.add() 150 self.current_test_report_msg.name = test_name 151 self.current_test_report_msg.start_timestamp = feature_utils.GetTimestamp( 152 ) 153 154 def AddApiCoverageReport(self, api_coverage_data_vec, isGlobal=True): 155 """Adds an API coverage report to the VtsReportMessage. 156 157 Translate each element in the give coverage data vector into a 158 ApiCoverageReportMessage within the report message. 159 160 Args: 161 api_coverage_data_vec: list of VTSApiCoverageData which contains 162 the metadata (e.g. package_name, version) 163 and the total/covered api names. 164 isGlobal: boolean, True if the coverage data is for the entire test, 165 False if only for the current test case. 166 """ 167 168 if not self.enabled: 169 return 170 171 if isGlobal: 172 report = self.report_msg 173 else: 174 report = self.current_test_report_msg 175 176 for api_coverage_data in api_coverage_data_vec: 177 api_coverage = report.api_coverage.add() 178 api_coverage.hal_interface.hal_package_name = api_coverage_data.package_name 179 api_coverage.hal_interface.hal_version_major = int( 180 api_coverage_data.version_major) 181 api_coverage.hal_interface.hal_version_minor = int( 182 api_coverage_data.version_minor) 183 api_coverage.hal_interface.hal_interface_name = api_coverage_data.interface_name 184 api_coverage.hal_api.extend(api_coverage_data.total_apis) 185 api_coverage.covered_hal_api.extend(api_coverage_data.covered_apis) 186 187 def AddCoverageReport(self, 188 coverage_vec, 189 src_file_path, 190 git_project_name, 191 git_project_path, 192 revision, 193 covered_count, 194 line_count, 195 isGlobal=True): 196 """Adds a coverage report to the VtsReportMessage. 197 198 Processes the source information, git project information, and processed 199 coverage information and stores it into a CoverageReportMessage within the 200 report message. 201 202 Args: 203 coverage_vec: list, list of coverage counts (int) for each line 204 src_file_path: the path to the original source file 205 git_project_name: the name of the git project containing the source 206 git_project_path: the path from the root to the git project 207 revision: the commit hash identifying the source code that was used to 208 build a device image 209 covered_count: int, number of lines covered 210 line_count: int, total number of lines 211 isGlobal: boolean, True if the coverage data is for the entire test, False if only for 212 the current test case. 213 """ 214 if not self.enabled: 215 return 216 217 if isGlobal: 218 report = self.report_msg 219 else: 220 report = self.current_test_report_msg 221 222 coverage = report.coverage.add() 223 coverage.total_line_count = line_count 224 coverage.covered_line_count = covered_count 225 coverage.line_coverage_vector.extend(coverage_vec) 226 227 src_file_path = os.path.relpath(src_file_path, git_project_path) 228 coverage.file_path = src_file_path 229 coverage.revision = revision 230 coverage.project_name = git_project_name 231 232 def AddProfilingDataTimestamp( 233 self, 234 name, 235 start_timestamp, 236 end_timestamp, 237 x_axis_label="Latency (nano secs)", 238 y_axis_label="Frequency", 239 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 240 """Adds the timestamp profiling data to the web DB. 241 242 Requires the feature to be enabled; no-op otherwise. 243 244 Args: 245 name: string, profiling point name. 246 start_timestamp: long, nanoseconds start time. 247 end_timestamp: long, nanoseconds end time. 248 x-axis_label: string, the x-axis label title for a graph plot. 249 y-axis_label: string, the y-axis label title for a graph plot. 250 regression_mode: specifies the direction of change which indicates 251 performance regression. 252 """ 253 if not self.enabled: 254 return 255 256 if not hasattr(self, _PROFILING_POINTS): 257 setattr(self, _PROFILING_POINTS, set()) 258 259 if name in getattr(self, _PROFILING_POINTS): 260 logging.error("profiling point %s is already active.", name) 261 return 262 263 getattr(self, _PROFILING_POINTS).add(name) 264 profiling_msg = self.report_msg.profiling.add() 265 profiling_msg.name = name 266 profiling_msg.type = ReportMsg.VTS_PROFILING_TYPE_TIMESTAMP 267 profiling_msg.regression_mode = regression_mode 268 profiling_msg.start_timestamp = start_timestamp 269 profiling_msg.end_timestamp = end_timestamp 270 profiling_msg.x_axis_label = x_axis_label 271 profiling_msg.y_axis_label = y_axis_label 272 273 def AddProfilingDataVector( 274 self, 275 name, 276 labels, 277 values, 278 data_type, 279 options=[], 280 x_axis_label="x-axis", 281 y_axis_label="y-axis", 282 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 283 """Adds the vector profiling data in order to upload to the web DB. 284 285 Requires the feature to be enabled; no-op otherwise. 286 287 Args: 288 name: string, profiling point name. 289 labels: a list or set of labels. 290 values: a list or set of values where each value is an integer. 291 data_type: profiling data type. 292 options: a set of options. 293 x-axis_label: string, the x-axis label title for a graph plot. 294 y-axis_label: string, the y-axis label title for a graph plot. 295 regression_mode: specifies the direction of change which indicates 296 performance regression. 297 """ 298 if not self.enabled: 299 return 300 301 if not hasattr(self, _PROFILING_POINTS): 302 setattr(self, _PROFILING_POINTS, set()) 303 304 if name in getattr(self, _PROFILING_POINTS): 305 logging.error("profiling point %s is already active.", name) 306 return 307 308 getattr(self, _PROFILING_POINTS).add(name) 309 profiling_msg = self.report_msg.profiling.add() 310 profiling_msg.name = name 311 profiling_msg.type = data_type 312 profiling_msg.regression_mode = regression_mode 313 if labels: 314 profiling_msg.label.extend(labels) 315 profiling_msg.value.extend(values) 316 profiling_msg.x_axis_label = x_axis_label 317 profiling_msg.y_axis_label = y_axis_label 318 profiling_msg.options.extend(options) 319 320 def AddProfilingDataLabeledVector( 321 self, 322 name, 323 labels, 324 values, 325 options=[], 326 x_axis_label="x-axis", 327 y_axis_label="y-axis", 328 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 329 """Adds the labeled vector profiling data in order to upload to the web DB. 330 331 Requires the feature to be enabled; no-op otherwise. 332 333 Args: 334 name: string, profiling point name. 335 labels: a list or set of labels. 336 values: a list or set of values where each value is an integer. 337 options: a set of options. 338 x-axis_label: string, the x-axis label title for a graph plot. 339 y-axis_label: string, the y-axis label title for a graph plot. 340 regression_mode: specifies the direction of change which indicates 341 performance regression. 342 """ 343 self.AddProfilingDataVector( 344 name, labels, values, ReportMsg.VTS_PROFILING_TYPE_LABELED_VECTOR, 345 options, x_axis_label, y_axis_label, regression_mode) 346 347 def AddProfilingDataUnlabeledVector( 348 self, 349 name, 350 values, 351 options=[], 352 x_axis_label="x-axis", 353 y_axis_label="y-axis", 354 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 355 """Adds the unlabeled vector profiling data in order to upload to the web DB. 356 357 Requires the feature to be enabled; no-op otherwise. 358 359 Args: 360 name: string, profiling point name. 361 values: a list or set of values where each value is an integer. 362 options: a set of options. 363 x-axis_label: string, the x-axis label title for a graph plot. 364 y-axis_label: string, the y-axis label title for a graph plot. 365 regression_mode: specifies the direction of change which indicates 366 performance regression. 367 """ 368 self.AddProfilingDataVector( 369 name, None, values, ReportMsg.VTS_PROFILING_TYPE_UNLABELED_VECTOR, 370 options, x_axis_label, y_axis_label, regression_mode) 371 372 def AddSystraceUrl(self, url): 373 """Creates a systrace report message with a systrace URL. 374 375 Adds a systrace report to the current test case report and supplies the 376 url to the systrace report. 377 378 Requires the feature to be enabled; no-op otherwise. 379 380 Args: 381 url: String, the url of the systrace report. 382 """ 383 if not self.enabled: 384 return 385 systrace_msg = self.current_test_report_msg.systrace.add() 386 systrace_msg.url.append(url) 387 388 def AddLogUrls(self, urls): 389 """Creates a log message with log file URLs. 390 391 Adds a log message to the current test module report and supplies the 392 url to the log files. 393 394 Requires the feature to be enabled; no-op otherwise. 395 396 Args: 397 urls: list of string, the URLs of the logs. 398 """ 399 if not self.enabled or urls is None: 400 return 401 402 for url in urls: 403 for log_msg in self.report_msg.log: 404 if log_msg.url == url: 405 continue 406 407 log_msg = self.report_msg.log.add() 408 log_msg.url = url 409 log_msg.name = os.path.basename(url) 410 411 def GetTestModuleKeys(self): 412 """Returns the test module name and start timestamp. 413 414 Those two values can be used to find the corresponding entry 415 in a used nosql database without having to lock all the data 416 (which is infesiable) thus are essential for strong consistency. 417 """ 418 return self.report_msg.test, self.report_msg.start_timestamp 419 420 def GenerateReportMessage(self, requested, executed): 421 """Uploads the result to the web service. 422 423 Requires the feature to be enabled; no-op otherwise. 424 425 Args: 426 requested: list, A list of test case records requested to run 427 executed: list, A list of test case records that were executed 428 429 Returns: 430 binary string, serialized report message. 431 None if web is not enabled. 432 """ 433 if not self.enabled: 434 return None 435 436 for test in requested[len(executed):]: 437 msg = self.report_msg.test_case.add() 438 msg.name = test.test_name 439 msg.start_timestamp = feature_utils.GetTimestamp() 440 msg.end_timestamp = msg.start_timestamp 441 msg.test_result = ReportMsg.TEST_CASE_RESULT_FAIL 442 443 self.report_msg.end_timestamp = feature_utils.GetTimestamp() 444 445 build = getattr(self, keys.ConfigKeys.IKEY_BUILD) 446 if keys.ConfigKeys.IKEY_BUILD_ID in build: 447 build_id = str(build[keys.ConfigKeys.IKEY_BUILD_ID]) 448 self.report_msg.build_info.id = build_id 449 450 logging.debug("_tearDownClass hook: start (username: %s)", 451 getpass.getuser()) 452 453 if len(self.report_msg.test_case) == 0: 454 logging.warn("_tearDownClass hook: skip uploading (no test case)") 455 return '' 456 457 post_msg = ReportMsg.DashboardPostMessage() 458 post_msg.test_report.extend([self.report_msg]) 459 460 self.rest_client.AddAuthToken(post_msg) 461 462 message_b = base64.b64encode(post_msg.SerializeToString()) 463 464 logging.debug('Result proto message generated. size: %s', 465 len(message_b)) 466 467 logging.debug("_tearDownClass hook: status upload time stamp %s", 468 str(self.report_msg.start_timestamp)) 469 470 return message_b