1# 2# Copyright (C) 2018 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the 'License'); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an 'AS IS' BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import logging 18import os 19import shutil 20import socket 21import time 22 23from host_controller import common 24from host_controller.command_processor import base_command_processor 25from host_controller.utils.gcp import gcs_utils 26from host_controller.utils.parser import xml_utils 27 28from vts.utils.python.common import cmd_utils 29 30from vti.dashboard.proto import TestSuiteResultMessage_pb2 as SuiteResMsg 31from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg 32 33 34class CommandUpload(base_command_processor.BaseCommandProcessor): 35 """Command processor for upload command. 36 37 Attributes: 38 arg_parser: ConsoleArgumentParser object, argument parser. 39 console: cmd.Cmd console object. 40 command: string, command name which this processor will handle. 41 command_detail: string, detailed explanation for the command. 42 """ 43 44 command = "upload" 45 command_detail = "Upload <src> file to <dest> Google Cloud Storage. In <src> and <dest>, variables enclosed in {} are replaced with the values stored in the console." 46 47 # @Override 48 def SetUp(self): 49 """Initializes the parser for upload command.""" 50 self.arg_parser.add_argument( 51 "--src", 52 required=True, 53 default="latest-system.img", 54 help="Path to a source file to upload. Only single file can be " 55 "uploaded per once. Use 'latest- prefix to upload the latest " 56 "fetch images. e.g. --src=latest-system.img If argument " 57 "value is not given, the recently fetched system.img will be " 58 "uploaded.") 59 self.arg_parser.add_argument( 60 "--dest", 61 required=True, 62 help="Google Cloud Storage URL to which the file is uploaded.") 63 self.arg_parser.add_argument( 64 "--report_path", 65 help="Google Cloud Storage URL, the dest path of a report file") 66 self.arg_parser.add_argument( 67 "--clear_dest", 68 action="store_true", 69 help="Delete dest recursively before the upload.") 70 self.arg_parser.add_argument( 71 "--clear_results", 72 default=False, 73 help="True to clear all the results after the upload.") 74 self.arg_parser.add_argument( 75 "--result_from_suite", 76 default="", 77 choices=("", "vts", "cts", "gts", "sts"), 78 help="To specify the type of a test suite report, since there can " 79 "be multiple numbers of result sets from different test " 80 "suites. If not specified, the HC will upload the report " 81 "from last run suite and plan.") 82 self.arg_parser.add_argument( 83 "--result_from_plan", 84 default="", 85 help="To specify the type of the plan name from which " 86 "the report is generated.") 87 88 # @Override 89 def Run(self, arg_line): 90 """Upload args.src file to args.dest Google Cloud Storage.""" 91 args = self.arg_parser.ParseLine(arg_line) 92 93 gsutil_path = gcs_utils.GetGsutilPath() 94 if not gsutil_path: 95 logging.error("Please check gsutil is installed and on your PATH") 96 return False 97 98 if args.src.startswith("latest-"): 99 src_name = args.src[7:] 100 if src_name in self.console.device_image_info: 101 src_paths = self.console.device_image_info[src_name] 102 else: 103 logging.error( 104 "Unable to find {} in device_image_info".format(src_name)) 105 return False 106 else: 107 try: 108 src_paths = self.console.FormatString(args.src) 109 except KeyError as e: 110 logging.error("Unknown or uninitialized variable in src: %s", 111 e) 112 return False 113 114 src_path_list_tmp = src_paths.split(" ") 115 src_path_list = [] 116 if src_path_list_tmp: 117 for src_path in src_path_list_tmp: 118 file_path = src_path.strip() 119 if os.path.isfile(file_path): 120 src_path_list.append(file_path) 121 else: 122 logging.error("Cannot find a file: {}".format(file_path)) 123 src_paths = " ".join(src_path_list) 124 125 try: 126 dest_path = self.console.FormatString(args.dest) 127 except KeyError as e: 128 logging.error("Unknown or uninitialized variable in dest: %s", e) 129 return False 130 131 if not dest_path.startswith("gs://"): 132 logging.error("{} is not correct GCS url.".format(dest_path)) 133 return False 134 """ TODO(jongmok) : Before upload, login status, authorization, 135 and dest check are required. """ 136 if args.clear_dest: 137 if not gcs_utils.Remove(gsutil_path, dest_path, recursive=True): 138 logging.error("Fail to remove %s", dest_path) 139 140 if not gcs_utils.Copy(gsutil_path, src_paths, dest_path): 141 logging.error("Fail to copy %s to %s", src_paths, dest_path) 142 143 if args.report_path or args.clear_results: 144 tools_path = "" 145 if args.result_from_suite: 146 tools_path = os.path.dirname( 147 self.console.test_suite_info[args.result_from_suite]) 148 else: 149 try: 150 tools_path = os.path.dirname(self.console.test_suite_info[ 151 self.console.FormatString("{suite_name}")]) 152 except KeyError: 153 if self.console.vti_endpoint_client.CheckBootUpStatus(): 154 logging.error( 155 "No test results found from any fetched test suite." 156 " Please fetch a test suite and run 'test' command," 157 " then try running 'upload' command again.") 158 return False 159 results_base_path = os.path.join(tools_path, 160 common._RESULTS_BASE_PATH) 161 162 if args.report_path: 163 report_path = self.console.FormatString(args.report_path) 164 if not report_path.startswith("gs://"): 165 logging.error( 166 "{} is not correct GCS url.".format(report_path)) 167 else: 168 self.UploadReport( 169 gsutil_path, report_path, dest_path, results_base_path, 170 args.result_from_suite, args.result_from_plan) 171 172 if args.clear_results: 173 shutil.rmtree(results_base_path, ignore_errors=True) 174 175 def UploadReport(self, gsutil_path, report_path, log_path, results_path, 176 suite_name, plan_name): 177 """Uploads report summary file to the given path. 178 179 Args: 180 gsutil_path: string, the path of a gsutil binary. 181 report_path: string, the dest GCS URL to which the summarized report 182 file will be uploaded. 183 log_path: string, GCS URL where the log files from the test run 184 have been uploaded. 185 results_path: string, the base path for the results. 186 """ 187 suite_res_msg = SuiteResMsg.TestSuiteResultMessage() 188 suite_res_msg.result_path = log_path 189 suite_res_msg.branch = self.console.FormatString("{branch}") 190 suite_res_msg.target = self.console.FormatString("{target}") 191 vti = self.console.vti_endpoint_client 192 suite_res_msg.boot_success = vti.CheckBootUpStatus() 193 suite_res_msg.test_type = vti.GetJobTestType() 194 195 device_fetch_info = self.console.detailed_fetch_info[ 196 common._ARTIFACT_TYPE_DEVICE] 197 gsi_fetch_info = None 198 if common._ARTIFACT_TYPE_GSI in self.console.detailed_fetch_info: 199 gsi_fetch_info = self.console.detailed_fetch_info[ 200 common._ARTIFACT_TYPE_GSI] 201 202 if vti.CheckBootUpStatus(): 203 former_results = [ 204 result for result in os.listdir(results_path) 205 if os.path.isdir(os.path.join(results_path, result)) 206 and not os.path.islink(os.path.join(results_path, result)) 207 ] 208 209 if not former_results: 210 logging.error("No test result found.") 211 return False 212 213 former_results.sort() 214 latest_result = former_results[-1] 215 latest_result_xml_path = os.path.join(results_path, latest_result, 216 common._TEST_RESULT_XML) 217 218 result_attrs = xml_utils.GetAttributes( 219 latest_result_xml_path, common._RESULT_TAG, [ 220 common._SUITE_NAME_ATTR_KEY, common._SUITE_PLAN_ATTR_KEY, 221 common._SUITE_VERSION_ATTR_KEY, 222 common._SUITE_BUILD_NUM_ATTR_KEY, 223 common._START_TIME_ATTR_KEY, common._END_TIME_ATTR_KEY, 224 common._HOST_NAME_ATTR_KEY 225 ]) 226 build_attrs = xml_utils.GetAttributes( 227 latest_result_xml_path, common._BUILD_TAG, [ 228 common._FINGERPRINT_ATTR_KEY, 229 common._SYSTEM_FINGERPRINT_ATTR_KEY, 230 common._VENDOR_FINGERPRINT_ATTR_KEY 231 ]) 232 summary_attrs = xml_utils.GetAttributes( 233 latest_result_xml_path, common._SUMMARY_TAG, [ 234 common._PASSED_ATTR_KEY, common._FAILED_ATTR_KEY, 235 common._MODULES_TOTAL_ATTR_KEY, 236 common._MODULES_DONE_ATTR_KEY 237 ]) 238 239 suite_res_msg.build_id = result_attrs[ 240 common._SUITE_BUILD_NUM_ATTR_KEY] 241 suite_res_msg.suite_name = result_attrs[ 242 common._SUITE_NAME_ATTR_KEY] 243 suite_res_msg.suite_plan = result_attrs[ 244 common._SUITE_PLAN_ATTR_KEY] 245 suite_res_msg.suite_version = result_attrs[ 246 common._SUITE_VERSION_ATTR_KEY] 247 suite_res_msg.suite_build_number = result_attrs[ 248 common._SUITE_BUILD_NUM_ATTR_KEY] 249 suite_res_msg.start_time = long( 250 result_attrs[common._START_TIME_ATTR_KEY]) 251 suite_res_msg.end_time = long( 252 result_attrs[common._END_TIME_ATTR_KEY]) 253 suite_res_msg.host_name = result_attrs[common._HOST_NAME_ATTR_KEY] 254 if common._SYSTEM_FINGERPRINT_ATTR_KEY in build_attrs: 255 suite_res_msg.build_system_fingerprint = build_attrs[ 256 common._SYSTEM_FINGERPRINT_ATTR_KEY] 257 else: 258 suite_res_msg.build_system_fingerprint = build_attrs[ 259 common._FINGERPRINT_ATTR_KEY] 260 if common._VENDOR_FINGERPRINT_ATTR_KEY in build_attrs: 261 suite_res_msg.build_vendor_fingerprint = build_attrs[ 262 common._VENDOR_FINGERPRINT_ATTR_KEY] 263 else: 264 suite_res_msg.build_vendor_fingerprint = build_attrs[ 265 common._FINGERPRINT_ATTR_KEY] 266 suite_res_msg.passed_test_case_count = int( 267 summary_attrs[common._PASSED_ATTR_KEY]) 268 suite_res_msg.failed_test_case_count = int( 269 summary_attrs[common._FAILED_ATTR_KEY]) 270 suite_res_msg.modules_done = int( 271 summary_attrs[common._MODULES_DONE_ATTR_KEY]) 272 suite_res_msg.modules_total = int( 273 summary_attrs[common._MODULES_TOTAL_ATTR_KEY]) 274 else: 275 suite_res_msg.build_id = self.console.fetch_info["build_id"] 276 suite_res_msg.suite_name = suite_name 277 suite_res_msg.suite_plan = plan_name 278 suite_res_msg.suite_version = "" 279 suite_res_msg.suite_build_number = suite_res_msg.build_id 280 suite_res_msg.start_time = long(time.time() * 1000) 281 suite_res_msg.end_time = suite_res_msg.start_time 282 suite_res_msg.host_name = socket.gethostname() 283 suite_res_msg.build_vendor_fingerprint = "%s/%s/%s" % ( 284 device_fetch_info["branch"], device_fetch_info["target"], 285 device_fetch_info["build_id"]) 286 if gsi_fetch_info: 287 suite_res_msg.build_system_fingerprint = "%s/%s/%s" % ( 288 gsi_fetch_info["branch"], gsi_fetch_info["target"], 289 gsi_fetch_info["build_id"]) 290 else: 291 suite_res_msg.build_system_fingerprint = suite_res_msg.build_vendor_fingerprint 292 suite_res_msg.passed_test_case_count = 0 293 suite_res_msg.failed_test_case_count = 0 294 suite_res_msg.modules_done = 0 295 suite_res_msg.modules_total = 0 296 297 suite_res_msg.infra_log_path = self.console.FormatString( 298 "{hc_log_upload_path}") 299 repack_path_list = [] 300 repack_path_list.append(self.console.FormatString("{repack_path}")) 301 suite_res_msg.repacked_image_path.extend(repack_path_list) 302 303 suite_res_msg.schedule_config.build_target.extend( 304 [SchedCfgMsg.BuildScheduleConfigMessage()]) 305 build_target_msg = suite_res_msg.schedule_config.build_target[0] 306 build_target_msg.test_schedule.extend( 307 [SchedCfgMsg.TestScheduleConfigMessage()]) 308 test_schedule_msg = build_target_msg.test_schedule[0] 309 310 suite_res_msg.vendor_build_id = device_fetch_info["build_id"] 311 suite_res_msg.schedule_config.manifest_branch = str( 312 device_fetch_info["branch"]) 313 build_target_msg.name = str(device_fetch_info["target"]) 314 if device_fetch_info["account_id"]: 315 suite_res_msg.schedule_config.pab_account_id = str( 316 device_fetch_info["account_id"]) 317 if device_fetch_info["fetch_signed_build"]: 318 build_target_msg.require_signed_device_build = device_fetch_info[ 319 "fetch_signed_build"] 320 if gsi_fetch_info: 321 test_schedule_msg.gsi_branch = str(gsi_fetch_info["branch"]) 322 test_schedule_msg.gsi_build_target = str(gsi_fetch_info["target"]) 323 suite_res_msg.gsi_build_id = str(gsi_fetch_info["build_id"]) 324 if gsi_fetch_info["account_id"]: 325 test_schedule_msg.gsi_pab_account_id = str( 326 gsi_fetch_info["account_id"]) 327 test_schedule_msg.gsi_vendor_version = str( 328 self.console.FormatString("{gsispl.vendor_version}")) 329 test_schedule_msg.test_pab_account_id = str( 330 self.console.FormatString("{account_id}")) 331 build_target_msg.has_bootloader_img = "bootloader.img" in self.console.device_image_info 332 build_target_msg.has_radio_img = "radio.img" in self.console.device_image_info 333 334 report_file_path = os.path.join( 335 self.console.tmp_logdir, 336 self.console.FormatString("{timestamp_time}.bin")) 337 with open(report_file_path, "w") as fd: 338 fd.write(suite_res_msg.SerializeToString()) 339 fd.close() 340 341 copy_command = "{} cp {} {}".format( 342 gsutil_path, report_file_path, 343 os.path.join(report_path, os.path.basename(report_file_path))) 344 _, stderr, err_code = cmd_utils.ExecuteOneShellCommand(copy_command) 345 if err_code: 346 logging.error(stderr) 347