1#!/usr/bin/env python3 2# coding=utf-8 3 4# 5# Copyright (c) 2020-2021 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18 19import copy 20import os 21import socket 22import time 23import platform 24import argparse 25import subprocess 26import signal 27import uuid 28import json 29import stat 30from tempfile import NamedTemporaryFile 31 32from _core.executor.listener import SuiteResult 33from _core.driver.parser_lite import ShellHandler 34from _core.exception import ParamError 35from _core.exception import ExecuteTerminate 36from _core.logger import platform_logger 37from _core.report.suite_reporter import SuiteReporter 38from _core.plugin import get_plugin 39from _core.plugin import Plugin 40from _core.constants import ModeType 41from _core.constants import ConfigConst 42 43LOG = platform_logger("Utils") 44 45 46def get_filename_extension(file_path): 47 _, fullname = os.path.split(file_path) 48 filename, ext = os.path.splitext(fullname) 49 return filename, ext 50 51 52def unique_id(type_name, value): 53 return "{}_{}_{:0>8}".format(type_name, value, 54 str(uuid.uuid1()).split("-")[0]) 55 56 57def start_standing_subprocess(cmd, pipe=subprocess.PIPE, return_result=False): 58 """Starts a non-blocking subprocess that is going to continue running after 59 this function returns. 60 61 A subprocess group is actually started by setting sid, so we can kill all 62 the processes spun out from the subprocess when stopping it. This is 63 necessary in case users pass in pipe commands. 64 65 Args: 66 cmd: Command to start the subprocess with. 67 pipe: pipe to get execution result 68 return_result: return execution result or not 69 70 Returns: 71 The subprocess that got started. 72 """ 73 sys_type = platform.system() 74 process = subprocess.Popen(cmd, stdout=pipe, shell=False, 75 preexec_fn=None if sys_type == "Windows" 76 else os.setsid) 77 if not return_result: 78 return process 79 else: 80 rev = process.stdout.read() 81 return rev.decode("utf-8").strip() 82 83 84def stop_standing_subprocess(process): 85 """Stops a subprocess started by start_standing_subprocess. 86 87 Catches and ignores the PermissionError which only happens on Macs. 88 89 Args: 90 process: Subprocess to terminate. 91 """ 92 try: 93 sys_type = platform.system() 94 signal_value = signal.SIGINT if sys_type == "Windows" \ 95 else signal.SIGTERM 96 os.kill(process.pid, signal_value) 97 except (PermissionError, AttributeError, FileNotFoundError, 98 SystemError) as error: 99 LOG.error("stop standing subprocess error '%s'" % error) 100 101 102def get_decode(stream): 103 if isinstance(stream, str): 104 return stream 105 106 if not isinstance(stream, bytes): 107 return str(stream) 108 109 try: 110 ret = stream.decode("utf-8", errors="ignore") 111 except (ValueError, AttributeError, TypeError): 112 ret = str(stream) 113 return ret 114 115 116def is_proc_running(pid, name=None): 117 if platform.system() == "Windows": 118 list_command = ["C:\\Windows\\System32\\tasklist"] 119 find_command = ["C:\\Windows\\System32\\findstr", "%s" % pid] 120 else: 121 list_command = ["/bin/ps", "-ef"] 122 find_command = ["/bin/grep", "%s" % pid] 123 proc = _get_find_proc(find_command, list_command) 124 (out, _) = proc.communicate() 125 out = get_decode(out).strip() 126 if out == "": 127 return False 128 else: 129 return True if name is None else out.find(name) != -1 130 131 132def _get_find_proc(find_command, list_command): 133 proc_sub = subprocess.Popen(list_command, stdout=subprocess.PIPE, 134 shell=False) 135 proc = subprocess.Popen(find_command, stdin=proc_sub.stdout, 136 stdout=subprocess.PIPE, shell=False) 137 return proc 138 139 140def exec_cmd(cmd, timeout=1 * 60, error_print=True, join_result=False): 141 """ 142 Executes commands in a new shell. Directing stderr to PIPE. 143 144 This is fastboot's own exe_cmd because of its peculiar way of writing 145 non-error info to stderr. 146 147 Args: 148 cmd: A sequence of commands and arguments. 149 timeout: timeout for exe cmd. 150 error_print: print error output or not. 151 join_result: join error and out 152 Returns: 153 The output of the command run. 154 """ 155 156 sys_type = platform.system() 157 if isinstance(cmd, list): 158 LOG.info("The running command is: {}".format(" ".join(cmd))) 159 if isinstance(cmd, str): 160 LOG.info("The running command is: {}".format(cmd)) 161 if sys_type == "Linux" or sys_type == "Darwin": 162 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, 163 stderr=subprocess.PIPE, shell=False, 164 preexec_fn=os.setsid) 165 else: 166 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, 167 stderr=subprocess.PIPE, shell=False) 168 try: 169 (out, err) = proc.communicate(timeout=timeout) 170 err = get_decode(err).strip() 171 out = get_decode(out).strip() 172 if err and error_print: 173 LOG.exception(err, exc_info=False) 174 if join_result: 175 return "%s\n %s" % (out, err) if err else out 176 else: 177 return err if err else out 178 179 except (TimeoutError, KeyboardInterrupt, AttributeError, ValueError, 180 EOFError, IOError, subprocess.TimeoutExpired): 181 sys_type = platform.system() 182 if sys_type == "Linux" or sys_type == "Darwin": 183 os.killpg(proc.pid, signal.SIGTERM) 184 else: 185 os.kill(proc.pid, signal.SIGINT) 186 raise 187 188 189def create_dir(path): 190 """Creates a directory if it does not exist already. 191 192 Args: 193 path: The path of the directory to create. 194 """ 195 full_path = os.path.abspath(os.path.expanduser(path)) 196 if not os.path.exists(full_path): 197 os.makedirs(full_path, exist_ok=True) 198 199 200def get_config_value(key, config_dict, is_list=True, default=None): 201 """get corresponding values for key in config_dict 202 203 Args: 204 key: target key in config_dict 205 config_dict: dictionary that store values 206 is_list: decide return values is list type or not 207 default: if key not in config_dict, default value will be returned 208 209 Returns: 210 corresponding values for key 211 """ 212 if not isinstance(config_dict, dict): 213 return default 214 215 value = config_dict.get(key, None) 216 if isinstance(value, bool): 217 return value 218 219 if value is None: 220 if default is not None: 221 return default 222 return [] if is_list else "" 223 224 if isinstance(value, list): 225 return value if is_list else value[0] 226 return [value] if is_list else value 227 228 229def get_file_absolute_path(input_name, paths=None, alt_dir=None): 230 """find absolute path for input_name 231 232 Args: 233 input_name: the target file to search 234 paths: path list for searching input_name 235 alt_dir: extra dir that appended to paths 236 237 Returns: 238 absolute path for input_name 239 """ 240 input_name = str(input_name) 241 abs_paths = set(paths) if paths else set() 242 _update_paths(abs_paths) 243 244 _inputs = [input_name] 245 if input_name.startswith("resource/"): 246 _inputs.append(input_name.replace("resource/", "", 1)) 247 elif input_name.startswith("testcases/"): 248 _inputs.append(input_name.replace("testcases/", "", 1)) 249 elif input_name.startswith("resource\\"): 250 _inputs.append(input_name.replace("resource\\", "", 1)) 251 elif input_name.startswith("testcases\\"): 252 _inputs.append(input_name.replace("testcases\\", "", 1)) 253 254 for _input in _inputs: 255 for path in abs_paths: 256 if alt_dir: 257 file_path = os.path.join(path, alt_dir, _input) 258 if os.path.exists(file_path): 259 return os.path.abspath(file_path) 260 261 file_path = os.path.join(path, _input) 262 if os.path.exists(file_path): 263 return os.path.abspath(file_path) 264 265 err_msg = "The file {} does not exist".format(input_name) 266 if check_mode(ModeType.decc): 267 LOG.error(err_msg, error_no="00109") 268 err_msg = "Load Error[00109]" 269 270 if alt_dir: 271 LOG.debug("alt_dir is %s" % alt_dir) 272 LOG.debug("paths is:") 273 for path in abs_paths: 274 LOG.debug(path) 275 raise ParamError(err_msg, error_no="00109") 276 277 278def _update_paths(paths): 279 from xdevice import Variables 280 resource_dir = "resource" 281 testcases_dir = "testcases" 282 283 need_add_path = set() 284 for path in paths: 285 if not os.path.exists(path): 286 continue 287 head, tail = os.path.split(path) 288 if not tail: 289 head, tail = os.path.split(head) 290 if tail in [resource_dir, testcases_dir]: 291 need_add_path.add(head) 292 paths.update(need_add_path) 293 294 inner_dir = os.path.abspath(os.path.join(Variables.exec_dir, 295 testcases_dir)) 296 top_inner_dir = os.path.abspath(os.path.join(Variables.top_dir, 297 testcases_dir)) 298 res_dir = os.path.abspath(os.path.join(Variables.exec_dir, resource_dir)) 299 top_res_dir = os.path.abspath(os.path.join(Variables.top_dir, 300 resource_dir)) 301 paths.update([inner_dir, res_dir, top_inner_dir, top_res_dir, 302 Variables.exec_dir, Variables.top_dir]) 303 304 305def modify_props(device, local_prop_file, target_prop_file, new_props): 306 """To change the props if need 307 Args: 308 device: the device to modify props 309 local_prop_file : the local file to save the old props 310 target_prop_file : the target prop file to change 311 new_props : the new props 312 Returns: 313 True : prop file changed 314 False : prop file no need to change 315 """ 316 is_changed = False 317 device.pull_file(target_prop_file, local_prop_file) 318 old_props = {} 319 changed_prop_key = [] 320 lines = [] 321 flags = os.O_RDONLY 322 modes = stat.S_IWUSR | stat.S_IRUSR 323 with os.fdopen(os.open(local_prop_file, flags, modes), "r") as old_file: 324 lines = old_file.readlines() 325 if lines: 326 lines[-1] = lines[-1] + '\n' 327 for line in lines: 328 line = line.strip() 329 if not line.startswith("#") and line.find("=") > 0: 330 key_value = line.split("=") 331 if len(key_value) == 2: 332 old_props[line.split("=")[0]] = line.split("=")[1] 333 334 for key, value in new_props.items(): 335 if key not in old_props.keys(): 336 lines.append("".join([key, "=", value, '\n'])) 337 is_changed = True 338 elif old_props.get(key) != value: 339 changed_prop_key.append(key) 340 is_changed = True 341 342 if is_changed: 343 local_temp_prop_file = NamedTemporaryFile(mode='w', prefix='build', 344 suffix='.tmp', delete=False) 345 for index, line in enumerate(lines): 346 if not line.startswith("#") and line.find("=") > 0: 347 key = line.split("=")[0] 348 if key in changed_prop_key: 349 lines[index] = "".join([key, "=", new_props[key], '\n']) 350 local_temp_prop_file.writelines(lines) 351 local_temp_prop_file.close() 352 device.push_file(local_temp_prop_file.name, target_prop_file) 353 device.execute_shell_command(" ".join(["chmod 644", target_prop_file])) 354 LOG.info("Changed the system property as required successfully") 355 os.remove(local_temp_prop_file.name) 356 357 return is_changed 358 359 360def get_device_log_file(report_path, serial=None, log_name="device_log", 361 device_name=""): 362 from xdevice import Variables 363 log_path = os.path.join(report_path, Variables.report_vars.log_dir) 364 os.makedirs(log_path, exist_ok=True) 365 366 serial = serial or time.time_ns() 367 if device_name: 368 serial = "%s_%s" % (device_name, serial) 369 device_file_name = "{}_{}.log".format(log_name, str(serial).replace( 370 ":", "_")) 371 device_log_file = os.path.join(log_path, device_file_name) 372 LOG.info("generate device log file: %s", device_log_file) 373 return device_log_file 374 375 376def check_result_report(report_root_dir, report_file, error_message="", 377 report_name="", module_name=""): 378 """ 379 check whether report_file exits or not. if report_file is not exist, 380 create empty report with error_message under report_root_dir 381 """ 382 383 if os.path.exists(report_file): 384 return report_file 385 report_dir = os.path.dirname(report_file) 386 if os.path.isabs(report_dir): 387 result_dir = report_dir 388 else: 389 result_dir = os.path.join(report_root_dir, "result", report_dir) 390 os.makedirs(result_dir, exist_ok=True) 391 if check_mode(ModeType.decc): 392 LOG.error("report not exist, create empty report") 393 else: 394 LOG.error("report %s not exist, create empty report under %s" % ( 395 report_file, result_dir)) 396 397 suite_name = report_name 398 if not suite_name: 399 suite_name, _ = get_filename_extension(report_file) 400 suite_result = SuiteResult() 401 suite_result.suite_name = suite_name 402 suite_result.stacktrace = error_message 403 if module_name: 404 suite_name = module_name 405 suite_reporter = SuiteReporter([(suite_result, [])], suite_name, 406 result_dir, modulename=module_name) 407 suite_reporter.create_empty_report() 408 return "%s.xml" % os.path.join(result_dir, suite_name) 409 410 411def get_sub_path(test_suite_path): 412 pattern = "%stests%s" % (os.sep, os.sep) 413 file_dir = os.path.dirname(test_suite_path) 414 pos = file_dir.find(pattern) 415 if -1 == pos: 416 return "" 417 418 sub_path = file_dir[pos + len(pattern):] 419 pos = sub_path.find(os.sep) 420 if -1 == pos: 421 return "" 422 return sub_path[pos + len(os.sep):] 423 424 425def is_config_str(content): 426 return True if "{" in content and "}" in content else False 427 428 429def get_version(): 430 from xdevice import Variables 431 ver = '' 432 ver_file_path = os.path.join(Variables.res_dir, 'version.txt') 433 if not os.path.isfile(ver_file_path): 434 return ver 435 flags = os.O_RDONLY 436 modes = stat.S_IWUSR | stat.S_IRUSR 437 with os.fdopen(os.open(ver_file_path, flags, modes), "r") as ver_file: 438 line = ver_file.readline() 439 if '-v' in line: 440 ver = line.strip().split('-')[1] 441 ver = ver.split(':')[0] 442 443 return ver 444 445 446def get_instance_name(instance): 447 return instance.__class__.__name__ 448 449 450def convert_ip(origin_ip): 451 addr = origin_ip.strip().split(".") 452 if len(addr) == 4: 453 return "{}.{}.{}.{}".format( 454 addr[0], '*'*len(addr[1]), '*'*len(addr[2]), addr[-1]) 455 else: 456 return origin_ip 457 458 459def convert_port(port): 460 _port = str(port) 461 if len(_port) >= 2: 462 return "{}{}{}".format(_port[0], "*" * (len(_port) - 2), _port[-1]) 463 else: 464 return "*{}".format(_port[-1]) 465 466 467def convert_serial(serial): 468 if serial.startswith("local_"): 469 return serial 470 elif serial.startswith("remote_"): 471 return "remote_{}_{}".format(convert_ip(serial.split("_")[1]), 472 convert_port(serial.split("_")[-1])) 473 else: 474 length = len(serial)//3 475 return "{}{}{}".format( 476 serial[0:length], "*"*(len(serial)-length*2), serial[-length:]) 477 478 479def get_shell_handler(request, parser_type): 480 suite_name = request.root.source.test_name 481 parsers = get_plugin(Plugin.PARSER, parser_type) 482 parser_instances = [] 483 for listener in request.listeners: 484 listener.device_sn = request.config.environment.devices[0].device_sn 485 for parser in parsers: 486 parser_instance = parser.__class__() 487 parser_instance.suite_name = suite_name 488 parser_instance.listeners = request.listeners 489 parser_instances.append(parser_instance) 490 handler = ShellHandler(parser_instances) 491 return handler 492 493 494def get_kit_instances(json_config, resource_path="", testcases_path=""): 495 from _core.testkit.json_parser import JsonParser 496 kit_instances = [] 497 498 # check input param 499 if not isinstance(json_config, JsonParser): 500 return kit_instances 501 502 # get kit instances 503 for kit in json_config.config.kits: 504 kit["paths"] = [resource_path, testcases_path] 505 kit_type = kit.get("type", "") 506 device_name = kit.get("device_name", None) 507 if get_plugin(plugin_type=Plugin.TEST_KIT, plugin_id=kit_type): 508 test_kit = \ 509 get_plugin(plugin_type=Plugin.TEST_KIT, plugin_id=kit_type)[0] 510 test_kit_instance = test_kit.__class__() 511 test_kit_instance.__check_config__(kit) 512 setattr(test_kit_instance, "device_name", device_name) 513 kit_instances.append(test_kit_instance) 514 else: 515 raise ParamError("kit %s not exists" % kit_type, error_no="00107") 516 return kit_instances 517 518 519def check_device_name(device, kit, step="setup"): 520 kit_device_name = getattr(kit, "device_name", None) 521 device_name = device.get("name") 522 if kit_device_name and device_name and \ 523 kit_device_name != device_name: 524 return False 525 if kit_device_name and device_name: 526 LOG.debug("do kit:%s %s for device:%s", 527 kit.__class__.__name__, step, device_name) 528 else: 529 LOG.debug("do kit:%s %s", kit.__class__.__name__, step) 530 return True 531 532 533def check_path_legal(path): 534 if path and " " in path: 535 return "\"%s\"" % path 536 return path 537 538 539def get_local_ip(): 540 sys_type = platform.system() 541 if sys_type == "Windows": 542 _list = socket.gethostbyname_ex(socket.gethostname()) 543 _list = _list[2] 544 for ip_add in _list: 545 if ip_add.startswith("10."): 546 return ip_add 547 548 return socket.gethostbyname(socket.getfqdn(socket.gethostname())) 549 elif sys_type == "Darwin": 550 hostname = socket.getfqdn(socket.gethostname()) 551 return socket.gethostbyname(hostname) 552 elif sys_type == "Linux": 553 real_ip = "/%s/%s" % ("hostip", "realip") 554 if os.path.exists(real_ip): 555 srw = None 556 try: 557 import codecs 558 srw = codecs.open(real_ip, "r", "utf-8") 559 lines = srw.readlines() 560 local_ip = str(lines[0]).strip() 561 except (IOError, ValueError) as error_message: 562 LOG.error(error_message) 563 local_ip = "127.0.0.1" 564 finally: 565 if srw is not None: 566 srw.close() 567 else: 568 local_ip = "127.0.0.1" 569 return local_ip 570 else: 571 return "127.0.0.1" 572 573 574class SplicingAction(argparse.Action): 575 def __call__(self, parser, namespace, values, option_string=None): 576 setattr(namespace, self.dest, " ".join(values)) 577 578 579def get_test_component_version(config): 580 if check_mode(ModeType.decc): 581 return "" 582 583 try: 584 paths = [config.resource_path, config.testcases_path] 585 test_file = get_file_absolute_path("test_component.json", paths) 586 flags = os.O_RDONLY 587 modes = stat.S_IWUSR | stat.S_IRUSR 588 with os.fdopen(os.open(test_file, flags, modes), "r") as file_content: 589 json_content = json.load(file_content) 590 version = json_content.get("version", "") 591 return version 592 except (ParamError, ValueError) as error: 593 LOG.error("The exception {} happened when get version".format(error)) 594 return "" 595 596 597def check_mode(mode): 598 from xdevice import Scheduler 599 return Scheduler.mode == mode 600 601 602def do_module_kit_setup(request, kits): 603 for device in request.get_devices(): 604 setattr(device, ConfigConst.module_kits, []) 605 606 from xdevice import Scheduler 607 for kit in kits: 608 run_flag = False 609 for device in request.get_devices(): 610 if not Scheduler.is_execute: 611 raise ExecuteTerminate() 612 if check_device_name(device, kit): 613 run_flag = True 614 kit_copy = copy.deepcopy(kit) 615 module_kits = getattr(device, ConfigConst.module_kits) 616 module_kits.append(kit_copy) 617 kit_copy.__setup__(device, request=request) 618 if not run_flag: 619 kit_device_name = getattr(kit, "device_name", None) 620 error_msg = "device name '%s' of '%s' not exist" % ( 621 kit_device_name, kit.__class__.__name__) 622 LOG.error(error_msg, error_no="00108") 623 raise ParamError(error_msg, error_no="00108") 624 625 626def do_module_kit_teardown(request): 627 for device in request.get_devices(): 628 for kit in getattr(device, ConfigConst.module_kits, []): 629 if check_device_name(device, kit, step="teardown"): 630 kit.__teardown__(device) 631 setattr(device, ConfigConst.module_kits, []) 632