1# 2# Copyright (C) 2017 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import base64 18import getpass 19import logging 20import os 21import socket 22import time 23 24from vts.proto import VtsReportMessage_pb2 as ReportMsg 25from vts.runners.host import keys 26from vts.utils.python.web import dashboard_rest_client 27from vts.utils.python.web import feature_utils 28 29_PROFILING_POINTS = "profiling_points" 30 31 32class WebFeature(feature_utils.Feature): 33 """Feature object for web functionality. 34 35 Attributes: 36 enabled: boolean, True if web feature is enabled, False otherwise 37 report_msg: TestReportMessage, Proto summarizing the test run 38 current_test_report_msg: TestCaseReportMessage, Proto summarizing the current test case 39 rest_client: DashboardRestClient, client to which data will be posted 40 """ 41 42 _TOGGLE_PARAM = keys.ConfigKeys.IKEY_ENABLE_WEB 43 _REQUIRED_PARAMS = [ 44 keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND, 45 keys.ConfigKeys.IKEY_SERVICE_JSON_PATH, 46 keys.ConfigKeys.KEY_TESTBED_NAME, keys.ConfigKeys.IKEY_BUILD, 47 keys.ConfigKeys.IKEY_ANDROID_DEVICE, keys.ConfigKeys.IKEY_ABI_NAME, 48 keys.ConfigKeys.IKEY_ABI_BITNESS 49 ] 50 _OPTIONAL_PARAMS = [ 51 keys.ConfigKeys.RUN_AS_VTS_SELFTEST, 52 keys.ConfigKeys.IKEY_ENABLE_PROFILING, 53 ] 54 55 def __init__(self, user_params): 56 """Initializes the web feature. 57 58 Parses the arguments and initializes the web functionality. 59 60 Args: 61 user_params: A dictionary from parameter name (String) to parameter value. 62 """ 63 self.ParseParameters( 64 toggle_param_name=self._TOGGLE_PARAM, 65 required_param_names=self._REQUIRED_PARAMS, 66 optional_param_names=self._OPTIONAL_PARAMS, 67 user_params=user_params) 68 if not self.enabled: 69 return 70 71 # Initialize the dashboard client 72 post_cmd = getattr(self, keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND) 73 service_json_path = str( 74 getattr(self, keys.ConfigKeys.IKEY_SERVICE_JSON_PATH)) 75 self.rest_client = dashboard_rest_client.DashboardRestClient( 76 post_cmd, service_json_path) 77 if not self.rest_client.Initialize(): 78 self.enabled = False 79 80 self.report_msg = ReportMsg.TestReportMessage() 81 self.report_msg.test = str( 82 getattr(self, keys.ConfigKeys.KEY_TESTBED_NAME)) 83 84 if getattr(self, keys.ConfigKeys.IKEY_ENABLE_PROFILING, False): 85 self.report_msg.test += "Profiling" 86 87 self.report_msg.test_type = ReportMsg.VTS_HOST_DRIVEN_STRUCTURAL 88 self.report_msg.start_timestamp = feature_utils.GetTimestamp() 89 self.report_msg.host_info.hostname = socket.gethostname() 90 91 android_devices = getattr(self, keys.ConfigKeys.IKEY_ANDROID_DEVICE, 92 None) 93 if not android_devices or not isinstance(android_devices, list): 94 logging.warn("android device information not available") 95 return 96 97 for device_spec in android_devices: 98 dev_info = self.report_msg.device_info.add() 99 for elem in [ 100 keys.ConfigKeys.IKEY_PRODUCT_TYPE, 101 keys.ConfigKeys.IKEY_PRODUCT_VARIANT, 102 keys.ConfigKeys.IKEY_BUILD_FLAVOR, 103 keys.ConfigKeys.IKEY_BUILD_ID, keys.ConfigKeys.IKEY_BRANCH, 104 keys.ConfigKeys.IKEY_BUILD_ALIAS, 105 keys.ConfigKeys.IKEY_API_LEVEL, keys.ConfigKeys.IKEY_SERIAL 106 ]: 107 if elem in device_spec: 108 setattr(dev_info, elem, str(device_spec[elem])) 109 # TODO: get abi information differently for multi-device support. 110 setattr(dev_info, keys.ConfigKeys.IKEY_ABI_NAME, 111 str(getattr(self, keys.ConfigKeys.IKEY_ABI_NAME))) 112 setattr(dev_info, keys.ConfigKeys.IKEY_ABI_BITNESS, 113 str(getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS))) 114 115 def SetTestResult(self, result=None): 116 """Set the current test case result to the provided result. 117 118 If None is provided as a result, the current test report will be cleared, which results 119 in a silent skip. 120 121 Requires the feature to be enabled; no-op otherwise. 122 123 Args: 124 result: ReportMsg.TestCaseResult, the result of the current test or None. 125 """ 126 if not self.enabled: 127 return 128 129 if not result: 130 self.report_msg.test_case.remove(self.current_test_report_msg) 131 self.current_test_report_msg = None 132 else: 133 self.current_test_report_msg.test_result = result 134 135 def AddTestReport(self, test_name): 136 """Creates a report for the specified test. 137 138 Requires the feature to be enabled; no-op otherwise. 139 140 Args: 141 test_name: String, the name of the test 142 """ 143 if not self.enabled: 144 return 145 self.current_test_report_msg = self.report_msg.test_case.add() 146 self.current_test_report_msg.name = test_name 147 self.current_test_report_msg.start_timestamp = feature_utils.GetTimestamp( 148 ) 149 150 def AddApiCoverageReport(self, api_coverage_data_vec, isGlobal=True): 151 """Adds an API coverage report to the VtsReportMessage. 152 153 Translate each element in the give coverage data vector into a 154 ApiCoverageReportMessage within the report message. 155 156 Args: 157 api_coverage_data_vec: list of VTSApiCoverageData which contains 158 the metadata (e.g. package_name, version) 159 and the total/covered api names. 160 isGlobal: boolean, True if the coverage data is for the entire test, 161 False if only for the current test case. 162 """ 163 164 if not self.enabled: 165 return 166 167 if isGlobal: 168 report = self.report_msg 169 else: 170 report = self.current_test_report_msg 171 172 for api_coverage_data in api_coverage_data_vec: 173 api_coverage = report.api_coverage.add() 174 api_coverage.hal_interface.hal_package_name = api_coverage_data.package_name 175 api_coverage.hal_interface.hal_version_major = int( 176 api_coverage_data.version_major) 177 api_coverage.hal_interface.hal_version_minor = int( 178 api_coverage_data.version_minor) 179 api_coverage.hal_interface.hal_interface_name = api_coverage_data.interface_name 180 api_coverage.hal_api.extend(api_coverage_data.total_apis) 181 api_coverage.covered_hal_api.extend(api_coverage_data.covered_apis) 182 183 def AddCoverageReport(self, 184 coverage_vec, 185 src_file_path, 186 git_project_name, 187 git_project_path, 188 revision, 189 covered_count, 190 line_count, 191 isGlobal=True): 192 """Adds a coverage report to the VtsReportMessage. 193 194 Processes the source information, git project information, and processed 195 coverage information and stores it into a CoverageReportMessage within the 196 report message. 197 198 Args: 199 coverage_vec: list, list of coverage counts (int) for each line 200 src_file_path: the path to the original source file 201 git_project_name: the name of the git project containing the source 202 git_project_path: the path from the root to the git project 203 revision: the commit hash identifying the source code that was used to 204 build a device image 205 covered_count: int, number of lines covered 206 line_count: int, total number of lines 207 isGlobal: boolean, True if the coverage data is for the entire test, False if only for 208 the current test case. 209 """ 210 if not self.enabled: 211 return 212 213 if isGlobal: 214 report = self.report_msg 215 else: 216 report = self.current_test_report_msg 217 218 coverage = report.coverage.add() 219 coverage.total_line_count = line_count 220 coverage.covered_line_count = covered_count 221 coverage.line_coverage_vector.extend(coverage_vec) 222 223 src_file_path = os.path.relpath(src_file_path, git_project_path) 224 coverage.file_path = src_file_path 225 coverage.revision = revision 226 coverage.project_name = git_project_name 227 228 def AddProfilingDataTimestamp( 229 self, 230 name, 231 start_timestamp, 232 end_timestamp, 233 x_axis_label="Latency (nano secs)", 234 y_axis_label="Frequency", 235 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 236 """Adds the timestamp profiling data to the web DB. 237 238 Requires the feature to be enabled; no-op otherwise. 239 240 Args: 241 name: string, profiling point name. 242 start_timestamp: long, nanoseconds start time. 243 end_timestamp: long, nanoseconds end time. 244 x-axis_label: string, the x-axis label title for a graph plot. 245 y-axis_label: string, the y-axis label title for a graph plot. 246 regression_mode: specifies the direction of change which indicates 247 performance regression. 248 """ 249 if not self.enabled: 250 return 251 252 if not hasattr(self, _PROFILING_POINTS): 253 setattr(self, _PROFILING_POINTS, set()) 254 255 if name in getattr(self, _PROFILING_POINTS): 256 logging.error("profiling point %s is already active.", name) 257 return 258 259 getattr(self, _PROFILING_POINTS).add(name) 260 profiling_msg = self.report_msg.profiling.add() 261 profiling_msg.name = name 262 profiling_msg.type = ReportMsg.VTS_PROFILING_TYPE_TIMESTAMP 263 profiling_msg.regression_mode = regression_mode 264 profiling_msg.start_timestamp = start_timestamp 265 profiling_msg.end_timestamp = end_timestamp 266 profiling_msg.x_axis_label = x_axis_label 267 profiling_msg.y_axis_label = y_axis_label 268 269 def AddProfilingDataVector( 270 self, 271 name, 272 labels, 273 values, 274 data_type, 275 options=[], 276 x_axis_label="x-axis", 277 y_axis_label="y-axis", 278 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 279 """Adds the vector profiling data in order to upload to the web DB. 280 281 Requires the feature to be enabled; no-op otherwise. 282 283 Args: 284 name: string, profiling point name. 285 labels: a list or set of labels. 286 values: a list or set of values where each value is an integer. 287 data_type: profiling data type. 288 options: a set of options. 289 x-axis_label: string, the x-axis label title for a graph plot. 290 y-axis_label: string, the y-axis label title for a graph plot. 291 regression_mode: specifies the direction of change which indicates 292 performance regression. 293 """ 294 if not self.enabled: 295 return 296 297 if not hasattr(self, _PROFILING_POINTS): 298 setattr(self, _PROFILING_POINTS, set()) 299 300 if name in getattr(self, _PROFILING_POINTS): 301 logging.error("profiling point %s is already active.", name) 302 return 303 304 getattr(self, _PROFILING_POINTS).add(name) 305 profiling_msg = self.report_msg.profiling.add() 306 profiling_msg.name = name 307 profiling_msg.type = data_type 308 profiling_msg.regression_mode = regression_mode 309 if labels: 310 profiling_msg.label.extend(labels) 311 profiling_msg.value.extend(values) 312 profiling_msg.x_axis_label = x_axis_label 313 profiling_msg.y_axis_label = y_axis_label 314 profiling_msg.options.extend(options) 315 316 def AddProfilingDataLabeledVector( 317 self, 318 name, 319 labels, 320 values, 321 options=[], 322 x_axis_label="x-axis", 323 y_axis_label="y-axis", 324 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 325 """Adds the labeled vector profiling data in order to upload to the web DB. 326 327 Requires the feature to be enabled; no-op otherwise. 328 329 Args: 330 name: string, profiling point name. 331 labels: a list or set of labels. 332 values: a list or set of values where each value is an integer. 333 options: a set of options. 334 x-axis_label: string, the x-axis label title for a graph plot. 335 y-axis_label: string, the y-axis label title for a graph plot. 336 regression_mode: specifies the direction of change which indicates 337 performance regression. 338 """ 339 self.AddProfilingDataVector( 340 name, labels, values, ReportMsg.VTS_PROFILING_TYPE_LABELED_VECTOR, 341 options, x_axis_label, y_axis_label, regression_mode) 342 343 def AddProfilingDataUnlabeledVector( 344 self, 345 name, 346 values, 347 options=[], 348 x_axis_label="x-axis", 349 y_axis_label="y-axis", 350 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 351 """Adds the unlabeled vector profiling data in order to upload to the web DB. 352 353 Requires the feature to be enabled; no-op otherwise. 354 355 Args: 356 name: string, profiling point name. 357 values: a list or set of values where each value is an integer. 358 options: a set of options. 359 x-axis_label: string, the x-axis label title for a graph plot. 360 y-axis_label: string, the y-axis label title for a graph plot. 361 regression_mode: specifies the direction of change which indicates 362 performance regression. 363 """ 364 self.AddProfilingDataVector( 365 name, None, values, ReportMsg.VTS_PROFILING_TYPE_UNLABELED_VECTOR, 366 options, x_axis_label, y_axis_label, regression_mode) 367 368 def AddSystraceUrl(self, url): 369 """Creates a systrace report message with a systrace URL. 370 371 Adds a systrace report to the current test case report and supplies the 372 url to the systrace report. 373 374 Requires the feature to be enabled; no-op otherwise. 375 376 Args: 377 url: String, the url of the systrace report. 378 """ 379 if not self.enabled: 380 return 381 systrace_msg = self.current_test_report_msg.systrace.add() 382 systrace_msg.url.append(url) 383 384 def AddLogUrls(self, urls): 385 """Creates a log message with log file URLs. 386 387 Adds a log message to the current test module report and supplies the 388 url to the log files. 389 390 Requires the feature to be enabled; no-op otherwise. 391 392 Args: 393 urls: list of string, the URLs of the logs. 394 """ 395 if not self.enabled or urls is None: 396 return 397 398 for url in urls: 399 for log_msg in self.report_msg.log: 400 if log_msg.url == url: 401 continue 402 403 log_msg = self.report_msg.log.add() 404 log_msg.url = url 405 log_msg.name = os.path.basename(url) 406 407 def GetTestModuleKeys(self): 408 """Returns the test module name and start timestamp. 409 410 Those two values can be used to find the corresponding entry 411 in a used nosql database without having to lock all the data 412 (which is infesiable) thus are essential for strong consistency. 413 """ 414 return self.report_msg.test, self.report_msg.start_timestamp 415 416 def GenerateReportMessage(self, requested, executed): 417 """Uploads the result to the web service. 418 419 Requires the feature to be enabled; no-op otherwise. 420 421 Args: 422 requested: list, A list of test case records requested to run 423 executed: list, A list of test case records that were executed 424 425 Returns: 426 binary string, serialized report message. 427 None if web is not enabled. 428 """ 429 if not self.enabled: 430 return None 431 432 for test in requested[len(executed):]: 433 msg = self.report_msg.test_case.add() 434 msg.name = test.test_name 435 msg.start_timestamp = feature_utils.GetTimestamp() 436 msg.end_timestamp = msg.start_timestamp 437 msg.test_result = ReportMsg.TEST_CASE_RESULT_FAIL 438 439 self.report_msg.end_timestamp = feature_utils.GetTimestamp() 440 441 build = getattr(self, keys.ConfigKeys.IKEY_BUILD) 442 if keys.ConfigKeys.IKEY_BUILD_ID in build: 443 build_id = str(build[keys.ConfigKeys.IKEY_BUILD_ID]) 444 self.report_msg.build_info.id = build_id 445 446 logging.debug("_tearDownClass hook: start (username: %s)", 447 getpass.getuser()) 448 449 if len(self.report_msg.test_case) == 0: 450 logging.warn("_tearDownClass hook: skip uploading (no test case)") 451 return '' 452 453 post_msg = ReportMsg.DashboardPostMessage() 454 post_msg.test_report.extend([self.report_msg]) 455 456 self.rest_client.AddAuthToken(post_msg) 457 458 message_b = base64.b64encode(post_msg.SerializeToString()) 459 460 logging.debug('Result proto message generated. size: %s', 461 len(message_b)) 462 463 logging.debug("_tearDownClass hook: status upload time stamp %s", 464 str(self.report_msg.start_timestamp)) 465 466 return message_b