1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2023 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19import time 20import os 21import sys 22import platform 23import pty 24import threading 25import re 26import traceback 27import select 28import subprocess 29import queue 30import shutil 31 32from collections import defaultdict 33 34sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "mylogger.py")) 35from mylogger import get_logger, parse_json 36 37Log = get_logger("performance") 38 39log_info = Log.info 40log_error = Log.error 41script_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 42 43config = parse_json() 44if not config: 45 log_error("config file: build_example.json not exist") 46 raise FileNotFoundError("config file: build_example.json not exist") 47 48 49class PerformanceAnalyse: 50 try: 51 TIMEOUT = int(config.get("performance").get("performance_exec_timeout")) 52 select_timeout = float(config.get("performance").get("performance_select_timeout")) 53 top_count = int(config.get("performance").get("performance_top_count")) 54 overflow = float(config.get("performance").get("performance_overflow")) 55 exclude = config.get("performance").get("exclude") 56 log_info("TIMEOUT:{}".format(TIMEOUT)) 57 log_info("select_timeout:{}".format(select_timeout)) 58 log_info("top_count:{}".format(top_count)) 59 log_info("overflow:{} sec".format(overflow)) 60 except Exception as e: 61 log_error("config file:build_example.json has error:{}".format(e)) 62 raise FileNotFoundError("config file:build_example.json has error:{}".format(e)) 63 64 def __init__(self, performance_cmd, output_path, report_title, ptyflag=False): 65 self.performance_cmd = script_path + performance_cmd 66 self.output_path = script_path + output_path 67 self.report_title = report_title 68 self.ptyflag = ptyflag 69 self.out_queue = queue.Queue() 70 self.system_info = list() 71 self.ninjia_trace_list = list() 72 self.gn_exec_li = list() 73 self.gn_script_li = list() 74 self.gn_end_li = list() 75 self.ccache_li = list() 76 self.c_targets_li = list() 77 self.root_dir = None 78 self.gn_dir = None 79 self.gn_script_res = None 80 self.gn_exec_res = None 81 self.cost_time_res = list() 82 self.gn_exec_flag = re.compile(r"File execute times") 83 self.gn_script_flag = re.compile(r"Script execute times") 84 self.gn_end_flag = re.compile(r"Done\. Made \d+ targets from \d+ files in (\d+)ms") 85 self.root_dir_flag = re.compile(r"""loader args.*source_root_dir="([a-zA-Z\d/\\_]+)""""") 86 self.gn_dir_flag = re.compile(r"""loader args.*gn_root_out_dir="([a-zA-Z\d/\\_]+)""""") 87 self.ccache_start_flag = re.compile(r"ccache_dir =") 88 self.ccache_end_flag = re.compile(r"c targets overlap rate statistics") 89 self.c_targets_flag = re.compile(r"c overall build overlap rate") 90 91 self.build_error = re.compile(r"=====build\s\serror=====") 92 self.ohos_error = re.compile(r"OHOS ERROR") 93 self.total_flag = re.compile(r"Cost time:.*(\d+:\d+:\d+)") 94 self.total_cost_time = None 95 self.error_message = list() 96 97 self.during_time_dic = { 98 "Preloader": {"start_pattern": re.compile(r"Set cache size"), 99 "end_pattern": re.compile(r"generated compile_standard_whitelist"), 100 "start_time": 0, 101 "end_time": 0 102 }, 103 "Loader": {"start_pattern": re.compile(r"Checking all build args"), 104 "end_pattern": re.compile(r"generate target syscap"), 105 "start_time": 0, 106 "end_time": 0 107 }, 108 "Ninjia": {"start_pattern": re.compile(r"Done\. Made \d+ targets from \d+ files in (\d+)ms"), 109 "end_pattern": re.compile(r"ccache_dir ="), 110 "start_time": 0, 111 "end_time": 0 112 }} 113 114 self.table_html = "" 115 116 self.base_html = """ 117 <!DOCTYPE html> 118 <html lang="en"> 119 <head> 120 <style type="text/css" media="screen"> 121 table {{ 122 border-collapse: collapse; 123 width: 80%; 124 max-width: 1200px; 125 margin-bottom: 30px; 126 margin-left: auto; 127 margin-right: auto; 128 table-layout: fixed; 129 }} 130 th, td {{ 131 padding: 10px; 132 text-align: center; 133 font-size: 12px; 134 border: 1px solid #ddd; 135 word-wrap: break-word; 136 }} 137 th {{ 138 background-color: #f2f2f2; 139 font-weight: bold; 140 text-transform: capitalize; 141 }} 142 tr:nth-child(even) {{ 143 background-color: #f9f9f9; 144 }} 145 caption {{ 146 font-size: 24px; 147 margin-bottom: 16px; 148 color: #333; 149 text-transform: uppercase; 150 letter-spacing: 2px; 151 font-family: Arial, sans-serif; 152 text-align: center; 153 text-transform: capitalize; 154 }} 155 .container {{ 156 width: 80%; 157 margin: 0 auto; 158 }} 159 body {{ font-family: Microsoft YaHei,Tahoma,arial,helvetica,sans-serif;padding: 20px;}} 160 h1 {{ text-align: center; }} 161 </style> 162 </head> 163 <body> 164 <div class="container"> 165 <h1>{}</h1> 166 """.format(self.report_title) 167 self.remove_out() 168 169 def remove_out(self): 170 """ 171 Description: remove out dir 172 """ 173 out_dir = os.path.join(script_path, "out") 174 try: 175 if not os.path.exists(out_dir): 176 return 177 for tmp_dir in os.listdir(out_dir): 178 if tmp_dir in self.exclude: 179 continue 180 if os.path.isdir(os.path.join(out_dir, tmp_dir)): 181 shutil.rmtree(os.path.join(out_dir, tmp_dir)) 182 else: 183 os.remove(os.path.join(out_dir, tmp_dir)) 184 except Exception as e: 185 log_error(e) 186 187 def write_html(self, content): 188 """ 189 Description: convert html str 190 @parameter content: html str 191 """ 192 if not os.path.exists(os.path.dirname(self.output_path)): 193 os.makedirs(os.path.dirname(self.output_path), exist_ok=True) 194 with open(self.output_path, "w", encoding="utf-8") as html_file: 195 html_file.write(content) 196 197 def generate_content(self, table_name, data_rows, switch=False): 198 """ 199 Description: generate html content 200 @parameter table_name: table name 201 @parameter data_rows: two-dimensional array data 202 @parameter switch: change overflow data color 203 """ 204 table_title = table_name.capitalize() 205 if not data_rows[1:]: 206 log_error("【{}】 is None") 207 return False 208 tb_html = """ 209 <table style="width: 100%; max-width: 1200px;"> 210 <caption>{0}</caption> 211 <colgroup> 212 <col style="width: {1}%"/> 213 </colgroup> 214 <thead> 215 <tr class="text-center success" style="font-weight: bold;font-size: 14px;"> 216 """.format(table_title, int(100 / len(data_rows[0]))) 217 218 self.table_html += tb_html 219 220 for header in data_rows[0]: 221 self.table_html += "<th>{}</th>".format(header.capitalize()) 222 self.table_html += "</tr></thead>" 223 224 self.table_html += "<tbody>" 225 if switch: 226 for index, row in enumerate(data_rows[1:]): 227 if float(row[-1]) > float(self.overflow): 228 self.table_html += "<tr style='background-color: #ff7f50;'>" 229 elif float(row[-1]) <= float(self.overflow) and index % 2 == 0: 230 self.table_html += "<tr style='background-color: #f5f5f5;'>" 231 else: 232 self.table_html += "<tr>" 233 for data in row: 234 self.table_html += "<td>{}</td>".format(data) 235 self.table_html += "</tr>" 236 else: 237 for index, row in enumerate(data_rows[1:]): 238 if index % 2 == 0: 239 self.table_html += "<tr style='background-color: #f5f5f5;'>" 240 else: 241 self.table_html += "<tr>" 242 for data in row: 243 self.table_html += "<td>{}</td>".format(data) 244 self.table_html += "</tr>" 245 self.table_html += "</tbody>" 246 247 self.table_html += "</table></div></body></html>" 248 249 @staticmethod 250 def generate_error_content(table_name, lines): 251 """ 252 Description: generate error html content 253 @parameter table_name: table name 254 @parameter lines: error message 255 """ 256 table_title = table_name.capitalize() 257 lines = ['<br>' + text for text in lines] 258 html_content = '<center><h1>{}</h1><div style="text-align:left;">{}<div></center>'.format(table_title, 259 '\n'.join(lines)) 260 error_html = '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>Ohos Error</title></head><body>{}</body></html>'.format( 261 html_content) 262 return error_html 263 264 def read_ninjia_trace_file(self): 265 """ 266 Description: read ninjia trace file 267 """ 268 try: 269 ninja_trace_path = os.path.join(self.root_dir, self.gn_dir, "sorted_action_duration.txt") 270 with open(ninja_trace_path, 'r') as f: 271 for line in f: 272 yield line.strip() 273 except Exception as e: 274 log_error("open ninjia trace file error is:{}".format(e)) 275 276 def process_ninja_trace(self): 277 """ 278 Description: generate ninja trace table data 279 """ 280 data = defaultdict(list) 281 result_list = list() 282 283 for line in self.read_ninjia_trace_file(): 284 line_list = line.split(":") 285 name = line_list[0].strip() 286 if name == "total time": 287 continue 288 duration = int(line_list[1].strip()) 289 data[name].append(duration) 290 291 for key, value in data.items(): 292 result = [key, len(value), max(value)] 293 result_list.append(result) 294 sort_result = sorted(result_list, key=lambda x: x[2], reverse=True) 295 for i in range(len(sort_result)): 296 sort_result[i][2] = round(float(sort_result[i][2]) / 1000, 4) 297 298 self.ninjia_trace_list = sort_result[:self.top_count] 299 300 self.ninjia_trace_list.insert(0, ["Ninjia Trace File", "Call Count", "Ninjia Trace Cost Time(s)"]) 301 302 def process_gn_trace(self): 303 """ 304 Description: generate gn trace table data 305 """ 306 self.gn_exec_res = [[item[2], item[1], round(float(item[0]) / 1000, 4)] for item in self.gn_exec_li if 307 item and re.match(r"[\d.]+", item[0])][ 308 :self.top_count] 309 self.gn_script_res = [[item[2], item[1], round(float(item[0]) / 1000, 4)] for item in self.gn_script_li if 310 item and re.match(r"[\d.]+", item[0])][:self.top_count] 311 312 self.gn_exec_res.insert(0, ["Gn Trace Exec File", "Call Count", "GN Trace Exec Cost Time(s)"]) 313 314 self.gn_script_res.insert(0, ["Gn Trace Script File", "Call Count", "GN Trace Script Cost Time(s)"]) 315 316 def process_ccache_ctargets(self): 317 """ 318 Description: generate gn trace table data 319 """ 320 ccache_res = [] 321 c_targets_res = [] 322 for tmp in self.ccache_li: 323 if ":" in tmp and len(tmp.split(":")) == 2: 324 ccache_res.append(tmp.split(":")) 325 ccache_res.insert(0, ["ccache item", "data"]) 326 327 for item in self.c_targets_li: 328 if len(item.split()) == 6: 329 c_targets_res.append(item.split()) 330 c_targets_res.insert(0, ["subsystem", "files NO.", " percentage", "builds NO.", "percentage", "verlap rate"]) 331 return ccache_res, c_targets_res 332 333 def process_system(self): 334 """ 335 Description: generate system data 336 """ 337 start_li = [ 338 ["System Information name", "System Value"], 339 ['Python Version', sys.version], 340 ['Cpu Count', os.cpu_count()], 341 ["System Info", platform.platform()] 342 ] 343 344 self.system_info.extend(start_li) 345 try: 346 disk_info = os.statvfs('/') 347 total_disk = round(float(disk_info.f_frsize * disk_info.f_blocks) / (1024 ** 3), 4) 348 349 self.system_info.append(["Disk Size", "{} GB".format(total_disk)]) 350 with open('/proc/meminfo', 'r') as f: 351 lines = f.readlines() 352 total_memory_line = [line for line in lines if line.startswith('MemTotal')] 353 total_memory = round(float(total_memory_line[0].split()[1]) / (1024 ** 2), 4) if total_memory_line else " " 354 355 self.system_info.append(["Total Memory", "{} GB".format(total_memory)]) 356 except Exception as e: 357 log_error(e) 358 359 def process_cost_time(self): 360 """ 361 Description: generate summary table data 362 """ 363 for i in self.during_time_dic.keys(): 364 cost_time = (self.during_time_dic.get(i).get("end_time") - self.during_time_dic.get(i).get( 365 "start_time")) / 10 ** 9 366 new_cost_time = round(float(cost_time), 4) 367 self.cost_time_res.append([i, new_cost_time]) 368 gn_res = re.search(self.gn_end_flag, self.gn_end_li[0]) 369 if gn_res: 370 gn_time = round(float(gn_res.group(1)) / 1000, 4) 371 self.cost_time_res.append(['GN', gn_time]) 372 self.cost_time_res.append(["Total", self.total_cost_time]) 373 self.cost_time_res.insert(0, ["Compile Process Phase", "Cost Time(s)"]) 374 375 def producer(self, execute_cmd, out_queue, timeout=TIMEOUT): 376 """ 377 Description: execute cmd and put cmd result data to queue 378 @parameter execute_cmd: execute cmd 379 @parameter out_queue: save out data 380 @parameter timeout: execute cmd time out 381 @return returncode: returncode 382 """ 383 log_info("exec cmd is :{}".format(" ".join(execute_cmd))) 384 log_info("ptyflag is :{}".format(self.ptyflag)) 385 386 if self.ptyflag: 387 try: 388 master, slave = pty.openpty() 389 proc = subprocess.Popen( 390 execute_cmd, 391 stdin=slave, 392 stdout=slave, 393 stderr=slave, 394 encoding="utf-8", 395 universal_newlines=True, 396 errors='ignore', 397 cwd=script_path 398 399 ) 400 start_time = time.time() 401 incomplete_line = "" 402 while True: 403 if timeout and time.time() - start_time > timeout: 404 raise Exception("exec cmd time out,select") 405 ready_to_read, _, _ = select.select([master, ], [], [], PerformanceAnalyse.select_timeout) 406 for stream in ready_to_read: 407 output_bytes = os.read(stream, 1024) 408 output = output_bytes.decode('utf-8') 409 lines = (incomplete_line + output).split("\n") 410 for line in lines[:-1]: 411 line = line.strip() 412 if line: 413 out_str = "{}".format(time.time_ns()) + "[timestamp]" + line 414 out_queue.put(out_str) 415 incomplete_line = lines[-1] 416 if proc.poll() is not None: 417 out_queue.put(None) 418 break 419 returncode = proc.wait() 420 return returncode 421 except Exception as e: 422 out_queue.put(None) 423 log_error("Producer An error occurred:{}".format(e)) 424 err_str = traceback.format_exc() 425 log_error(err_str) 426 raise e 427 else: 428 try: 429 start_time = time.time() 430 proc = subprocess.Popen( 431 execute_cmd, 432 stdout=subprocess.PIPE, 433 stderr=subprocess.PIPE, 434 encoding="utf-8", 435 universal_newlines=True, 436 errors='ignore', 437 cwd=script_path 438 ) 439 440 while True: 441 if timeout and time.time() - start_time > timeout: 442 raise TimeoutError("exec cmd timeout") 443 ready_to_read, _, _ = select.select([proc.stdout, proc.stderr], [], [], 444 PerformanceAnalyse.select_timeout) 445 for stream in ready_to_read: 446 output = stream.readline().strip() 447 if output: 448 out_str = "{}".format(time.time_ns()) + "[timestamp]" + output 449 out_queue.put(out_str) 450 if proc.poll() is not None: 451 out_queue.put(None) 452 break 453 returncode = proc.wait() 454 return returncode 455 except Exception as e: 456 out_queue.put(None) 457 log_error("Producer An error occurred:{}".format(e)) 458 err_str = traceback.format_exc() 459 log_error(err_str) 460 raise e 461 462 def consumer(self, out_queue, timeout=TIMEOUT): 463 """ 464 Description: get cmd result data from queue 465 @parameter out_queue: save out data 466 @parameter timeout: execute cmd time out 467 """ 468 start_time = time.time() 469 try: 470 line_count = 0 471 gn_exec_start, gn_script, gn_end, ccache_start, ccache_end, c_tagart_end = None, None, None, None, None, None 472 while True: 473 if timeout and time.time() - start_time > timeout: 474 raise TimeoutError("consumer timeout") 475 476 output = out_queue.get() 477 if output is None: 478 log_info(".....................exec end...........................") 479 break 480 line_count += 1 481 log_info(output.split("[timestamp]")[1]) 482 line_mes = " ".join(output.split("[timestamp]")[1].split()[2:]) 483 time_stamp = output.split("[timestamp]")[0] 484 485 if re.search(self.root_dir_flag, output): 486 self.root_dir = re.search(self.root_dir_flag, output).group(1) 487 488 if re.search(self.gn_dir_flag, output): 489 self.gn_dir = re.search(self.gn_dir_flag, output).group(1) 490 491 for key, value in self.during_time_dic.items(): 492 if re.search(value.get("start_pattern"), output): 493 self.during_time_dic.get(key)["start_time"] = int(time_stamp) 494 if re.search(value["end_pattern"], output): 495 self.during_time_dic.get(key)["end_time"] = int(time_stamp) 496 497 if re.search(self.gn_exec_flag, output): 498 gn_exec_start = line_count 499 elif re.search(self.gn_script_flag, output): 500 gn_script = line_count 501 elif re.search(self.gn_end_flag, output): 502 gn_end = line_count 503 self.gn_end_li.append(line_mes) 504 elif re.search(self.ccache_start_flag, output): 505 ccache_start = line_count 506 elif re.search(self.ccache_end_flag, output): 507 ccache_end = line_count 508 elif re.search(self.c_targets_flag, output): 509 c_tagart_end = line_count 510 511 if gn_exec_start and line_count > gn_exec_start and not gn_script: 512 self.gn_exec_li.append(line_mes.split()) 513 elif gn_script and line_count > gn_script and not gn_end: 514 self.gn_script_li.append(line_mes.split()) 515 elif ccache_start and line_count > ccache_start and not ccache_end: 516 self.ccache_li.append(line_mes) 517 elif ccache_end and line_count > ccache_end and not c_tagart_end: 518 self.c_targets_li.append(line_mes) 519 520 if re.search(self.ohos_error, output) or re.search(self.build_error, output): 521 self.error_message.append( 522 re.sub(r"\x1b\[[0-9;]*m", "", output.split("[timestamp]")[1].strip())) 523 524 if re.search(self.total_flag, output): 525 total_time_str = re.search(self.total_flag, output).group(1) 526 time_obj = time.strptime(total_time_str, "%H:%M:%S") 527 milliseconds = (time_obj.tm_hour * 3600 + time_obj.tm_min * 60 + time_obj.tm_sec) 528 self.total_cost_time = milliseconds 529 530 except Exception as e: 531 log_error("Consumer An error occurred:{}".format(e)) 532 err_str = traceback.format_exc() 533 log_error(err_str) 534 raise e 535 536 def exec_command_pipe(self, execute_cmd): 537 """ 538 Description: start producer and consumer 539 @parameter execute_cmd: execute cmd 540 """ 541 try: 542 producer_thread = threading.Thread(target=self.producer, args=(execute_cmd, self.out_queue)) 543 consumer_thread = threading.Thread(target=self.consumer, args=(self.out_queue,)) 544 producer_thread.daemon = True 545 consumer_thread.daemon = True 546 producer_thread.start() 547 consumer_thread.start() 548 producer_thread.join() 549 consumer_thread.join() 550 except Exception as e: 551 err_str = traceback.format_exc() 552 log_error(err_str) 553 raise Exception(e) 554 555 def process(self): 556 """ 557 Description: start performance test 558 """ 559 try: 560 cmd = self.performance_cmd.split(" ") 561 self.exec_command_pipe(cmd) 562 if self.error_message: 563 err_html = self.generate_error_content("Ohos Error", self.error_message) 564 self.write_html(err_html) 565 return 566 567 self.process_system() 568 self.process_cost_time() 569 self.process_gn_trace() 570 self.process_ninja_trace() 571 ccache_res, c_targets_res = self.process_ccache_ctargets() 572 573 log_info(self.cost_time_res) 574 log_info(self.gn_exec_res) 575 log_info(self.gn_script_res) 576 log_info(self.ninjia_trace_list) 577 log_info(ccache_res) 578 log_info(c_targets_res) 579 self.generate_content("System Information", self.system_info) 580 self.generate_content("Compile Process Summary", self.cost_time_res) 581 self.generate_content("Gn Trace Exec File", self.gn_exec_res, switch=True) 582 self.generate_content("Gn Trace Script File ", self.gn_script_res, switch=True) 583 self.generate_content("Ninjia Trace File", self.ninjia_trace_list, switch=True) 584 self.generate_content("Ccache Data Statistics", ccache_res) 585 self.generate_content("C Targets Overlap Rate Statistics", c_targets_res) 586 res_html = self.base_html + self.table_html 587 self.write_html(res_html) 588 except Exception as e: 589 err_str = traceback.format_exc() 590 log_error(err_str) 591 err_html = self.generate_error_content("Performance system Error", err_str.split("\n")) 592 self.write_html(err_html) 593 594 595if __name__ == '__main__': 596 performance_script_data = config.get("performance").get("performance_script_data") 597 for item in performance_script_data: 598 cmd = item.get("performance_cmd") 599 path = item.get("output_path") 600 report_title = item.get("report_title") 601 ptyflag = True if item.get("ptyflag").lower() == "true" else False 602 if all([cmd, path, report_title]): 603 performance = PerformanceAnalyse(cmd, path, report_title, ptyflag) 604 performance.process() 605 606