1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2025 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18from abc import ABCMeta, abstractmethod 19from enum import IntEnum, unique 20from typing import List, Any 21import optparse 22import os 23import re 24import stat 25import struct 26import sys 27 28import parse_functions 29 30TRACE_REGEX_ASYNC = "\s*(\d+)\s+(.*?)\|\d+\|[SFC]\s+:(.*?)\s+:(.*?)\s+(.*?)\s+\]\d+\[\s+\)(\d+)\s*\(\s+(\d+?)-(.*?)\s+" 31TRACE_REGEX_SYNC = "\s*\|\d+\|E\s+:(.*?)\s+:(.*?)\s+(.*?)\s+\]\d+\[\s+\)(\d+)\s*\(\s+(\d+?)-(.*?)\s+" 32text_file = "" 33binary_file = "" 34out_file = "" 35file_dir = "" 36 37TRACE_TXT_HEADER_FORMAT = """# tracer: nop 38# 39# entries-in-buffer/entries-written: %lu/%lu #P:%d 40# 41# _-----=> irqs-off 42# / _----=> need-resched 43# | / _---=> hardirq/softirq 44# || / _--=> preempt-depth 45# ||| / delay 46# TASK-PID TGID CPU# |||| TIMESTAMP FUNCTION 47# | | | | |||| | | 48""" 49 50 51def parse_options() -> None: 52 global text_file 53 global binary_file 54 global out_file 55 global file_dir 56 57 usage = "Usage: %prog -t text_file -o out_file or\n%prog -b binary_file -o out_file" 58 desc = "Example: %prog -t my_trace_file.htrace -o my_trace_file.systrace" 59 60 parser = optparse.OptionParser(usage=usage, description=desc) 61 parser.add_option('-t', '--text_file', dest='text_file', 62 help='Name of the text file to be parsed.', metavar='FILE') 63 parser.add_option('-b', '--binary_file', dest='binary_file', 64 help='Name of the binary file to be parsed.', metavar='FILE') 65 parser.add_option('-o', '--out_file', dest='out_file', 66 help='File name after successful parsing.', metavar='FILE') 67 parser.add_option('-d', '--file_dir', dest='file_dir', 68 help='Folder to be parsed.', metavar='FILE') 69 70 options, args = parser.parse_args() 71 72 if options.file_dir is None: 73 if options.out_file is not None: 74 out_file = options.out_file 75 else: 76 print("Error: out_file must be specified") 77 exit(-1) 78 if options.text_file is not None: 79 text_file = options.text_file 80 if options.binary_file is not None: 81 binary_file = options.binary_file 82 83 if text_file == '' and binary_file == '': 84 print("Error: You must specify a text or binary file") 85 exit(-1) 86 if text_file != '' and binary_file != '': 87 print("Error: Only one parsed file can be specified") 88 exit(-1) 89 else: 90 folder = os.path.exists(options.file_dir) 91 if not folder: 92 print("Error: file_dir does not exist") 93 exit(-1) 94 else: 95 file_dir = options.file_dir 96 97 98def parse_text_trace_file() -> None: 99 print("start processing text trace file") 100 pattern_async = re.compile(TRACE_REGEX_ASYNC) 101 pattern_sync = re.compile(TRACE_REGEX_SYNC) 102 match_num = 0 103 104 infile_flags = os.O_RDONLY 105 infile_mode = stat.S_IRUSR 106 infile = os.fdopen(os.open(text_file, infile_flags, infile_mode), "r", encoding="utf-8") 107 outfile_flags = os.O_RDWR | os.O_CREAT 108 outfile_mode = stat.S_IRUSR | stat.S_IWUSR 109 outfile = os.fdopen(os.open(out_file, outfile_flags, outfile_mode), "w+", encoding="utf-8") 110 111 for line in infile: 112 reverse_line = line[::-1] 113 trace_match_async = pattern_async.match(reverse_line) 114 trace_match_sync = pattern_sync.match(reverse_line) 115 if trace_match_async: 116 line = line.rstrip(' ') 117 pos = line.rfind(' ') 118 line = "%s%s%s" % (line[:pos], '|', line[pos + 1:]) 119 match_num += 1 120 elif trace_match_sync: 121 line = "%s\n" % (line.rstrip()[:-1]) 122 match_num += 1 123 outfile.write(line) 124 infile.close() 125 outfile.close() 126 print("total matched and modified lines: ", match_num) 127 128 129class DataType: 130 @staticmethod 131 def get_data_bytes(data_types: str) -> int: 132 return struct.calcsize(data_types) 133 134 135@unique 136class FieldType(IntEnum): 137 # 对HiTrace文件中逻辑划分的域 138 TRACE_FILE_HEADER = 1001 139 TRACE_EVENT_PAGE_HEADER = 1002 140 TRACE_EVENT_HEADER = 1003 141 TRACE_EVENT_CONTENT = 1004 142 143 # HiTrace文件中定义的段类型取值 144 SEGMENT_SEGMENTS = 0 145 SEGMENT_EVENTS_FORMAT = 1 146 SEGMENT_CMDLINES = 2 147 SEGMENT_TGIDS = 3 148 SEGMENT_RAW_TRACE = 4 149 SEGMENT_HEADER_PAGE = 30 150 SEGMENT_PRINTK_FORMATS = 31 151 SEGMENT_KALLSYMS = 32 152 SEGMENT_UNSUPPORT = -1 153 pass 154 155 156class Field: 157 """ 158 功能描述: field为一个复合的结构，该结构包含多少item，每个item都有自已的类型 159 """ 160 def __init__(self, vtype: int, size: int, vformat: str, item_types: List) -> None: 161 self.field_type = vtype 162 self.field_size = size 163 self.format = vformat 164 self.item_types = item_types 165 pass 166 167 168class TraceParseContext: 169 """ 170 功能描述: 解析过程的中间数据，需要缓存到上下文中，解决段与段间的数据依赖问题 171 """ 172 CONTEXT_CPU_NUM = 1 173 CONTEXT_CMD_LINES = 2 174 CONTEXT_EVENT_SIZE = 3 175 CONTEXT_CORE_ID = 5 176 CONTEXT_TIMESTAMP = 6 177 CONTEXT_TIMESTAMP_OFFSET = 7 178 CONTEXT_TID_GROUPS = 8 179 CONTEXT_EVENT_FORMAT = 9 180 181 def __init__(self) -> None: 182 self.values = {} 183 self.default_values = { 184 TraceParseContext.CONTEXT_CPU_NUM: 1, 185 TraceParseContext.CONTEXT_CMD_LINES: {}, 186 TraceParseContext.CONTEXT_EVENT_SIZE: 0, 187 TraceParseContext.CONTEXT_CORE_ID: 0, 188 TraceParseContext.CONTEXT_TIMESTAMP: 0, 189 TraceParseContext.CONTEXT_TIMESTAMP_OFFSET: 0, 190 TraceParseContext.CONTEXT_TID_GROUPS: {}, 191 TraceParseContext.CONTEXT_EVENT_FORMAT: {}, 192 } 193 pass 194 195 def set_param(self, key: int, value: Any) -> None: 196 self.values[key] = value 197 pass 198 199 def get_param(self, key: int) -> Any: 200 value = self.values.get(key) 201 if value is None: 202 return self.default_values.get(key) 203 return value 204 205 206class TraceViewerInterface(metaclass=ABCMeta): 207 """ 208 功能描述: 声明解析HiTrace结果的接口 209 """ 210 @abstractmethod 211 def append_trace_event(self, trace_event: List) -> None: 212 pass 213 214 @abstractmethod 215 def calculate(self, context: TraceParseContext) -> None: 216 pass 217 218 219class TraceFileParserInterface(metaclass=ABCMeta): 220 """ 221 功能描述: 声明解析HiTrace文件的接口 222 """ 223 @abstractmethod 224 def parse_simple_field(self, field: Field) -> tuple: 225 return () 226 227 @abstractmethod 228 def parse_event_format(self, data: List) -> dict: 229 return {} 230 231 @abstractmethod 232 def parse_cmd_lines(self, data: List) -> dict: 233 return {} 234 235 @abstractmethod 236 def parse_tid_groups(self, data: List) -> dict: 237 return {} 238 239 @abstractmethod 240 def get_context(self) -> TraceParseContext: 241 return None 242 243 @abstractmethod 244 def get_viewer(self) -> TraceViewerInterface: 245 return None 246 247 @abstractmethod 248 def get_segment_data(self, segment_size) -> List: 249 return None 250 251 252class OperatorInterface(metaclass=ABCMeta): 253 @abstractmethod 254 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 255 return True 256 257 258class FieldOperator(Field, OperatorInterface): 259 def __init__(self, vtype, size, vformat, item_types) -> None: 260 Field.__init__(self, vtype, size, vformat, item_types) 261 pass 262 263 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 264 return True 265 266 267class SegmentOperator(Field, OperatorInterface): 268 def __init__(self, vtype) -> None: 269 Field.__init__(self, vtype, -1, "", []) 270 pass 271 272 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 273 return True 274 275 276class FileHeader(FieldOperator): 277 """ 278 功能描述: 声明HiTrace文件的头部格式 279 """ 280 # 描述HiTrace文件头的pack格式 281 FORMAT = "HBHI" 282 283 # HiTrace文件头包含的字段 284 ITEM_MAGIC_NUMBER = 0 285 ITEM_FILE_TYPE = 1 286 ITEM_VERSION_NUMBER = 2 287 ITEM_RESERVED = 3 288 289 def __init__(self, item_types=None) -> None: 290 item_types = item_types or [] 291 super().__init__( 292 FieldType.TRACE_FILE_HEADER, 293 DataType.get_data_bytes(FileHeader.FORMAT), 294 FileHeader.FORMAT, 295 item_types 296 ) 297 298 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 299 data = parser.get_segment_data(self.field_size) 300 values = parser.parse_simple_field(self.format, data) 301 if len(values) == 0: 302 return False 303 cpu_num = (values[FileHeader.ITEM_RESERVED] >> 1) & 0b00011111 304 context = parser.get_context() 305 context.set_param(TraceParseContext.CONTEXT_CPU_NUM, cpu_num) 306 307 version = values[FileHeader.ITEM_VERSION_NUMBER] 308 file_type = values[FileHeader.ITEM_FILE_TYPE] 309 print("version is %d, file type is %d, cpu number %d" % (version, file_type, cpu_num)) 310 return True 311 312 313class PageHeader(FieldOperator): 314 """ 315 功能描述: 声明HiTrace文件的第个 raw event 页头部格式 316 """ 317 # 描述HiTrace文件的page header的pack格式 318 FORMAT = "QQB" 319 320 # HiTrace文件的page header包含的字段，每一页是同一个CPU核产生的数据，每一页都是固定大小4096字节 321 # page内trace event的时间基准点 322 ITEM_TIMESTAMP = 0 323 ITEM_LENGTH = 1 324 # page所属的CPU核 325 ITEM_CORE_ID = 2 326 327 def __init__(self, item_types: List) -> None: 328 super().__init__( 329 FieldType.TRACE_EVENT_PAGE_HEADER, 330 DataType.get_data_bytes(PageHeader.FORMAT), 331 PageHeader.FORMAT, 332 item_types 333 ) 334 335 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 336 segment = segment or [] 337 value = parser.parse_simple_field(self.format, segment) 338 context = parser.get_context() 339 context.set_param(TraceParseContext.CONTEXT_TIMESTAMP, value[PageHeader.ITEM_TIMESTAMP]) 340 context.set_param(TraceParseContext.CONTEXT_CORE_ID, value[PageHeader.ITEM_CORE_ID]) 341 return True 342 pass 343 344 345class TraceEventHeader(FieldOperator): 346 """ 347 功能描述: 声明HiTrace文件的trace event头部格式 348 """ 349 # 描述HiTrace文件trace event的pack格式 350 FORMAT = "IH" 351 RMQ_ENTRY_ALIGN_MASK = 3 352 353 # HiTrace文件的trace event header包含的字段，事件的打点时间是基于page header的时间做偏移可计算出来 354 ITEM_TIMESTAMP_OFFSET = 0 355 # trace event的大小，实际占用内存的大小还需要考虑内存对齐的因素 356 ITEM_EVENT_SIZE = 1 357 358 def __init__(self, item_types: List) -> None: 359 super().__init__( 360 FieldType.TRACE_EVENT_HEADER, 361 DataType.get_data_bytes(TraceEventHeader.FORMAT), 362 TraceEventHeader.FORMAT, 363 item_types 364 ) 365 366 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 367 segment = segment or [] 368 value = parser.parse_simple_field(self.format, segment) 369 if len(value) == 0: 370 return False 371 event_size = value[TraceEventHeader.ITEM_EVENT_SIZE] 372 align_event_size = ((event_size + TraceEventHeader.RMQ_ENTRY_ALIGN_MASK) & (~TraceEventHeader.RMQ_ENTRY_ALIGN_MASK)) 373 374 context = parser.get_context() 375 context.set_param(TraceParseContext.CONTEXT_EVENT_SIZE, (event_size, align_event_size)) 376 context.set_param(TraceParseContext.CONTEXT_TIMESTAMP_OFFSET, value[TraceEventHeader.ITEM_TIMESTAMP_OFFSET]) 377 return True 378 pass 379 380 381class TraceEventContent(SegmentOperator): 382 """ 383 功能描述: 声明HiTrace文件的trace event的内容格式 384 """ 385 def __init__(self) -> None: 386 super().__init__(FieldType.TRACE_EVENT_CONTENT) 387 pass 388 389 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 390 segment = segment or [] 391 context = parser.get_context() 392 timestamp = context.get_param(TraceParseContext.CONTEXT_TIMESTAMP) 393 timestamp = timestamp + context.get_param(TraceParseContext.CONTEXT_TIMESTAMP_OFFSET) 394 core_id = context.get_param(TraceParseContext.CONTEXT_CORE_ID) 395 396 vformat = 'H' 397 data_bytes = DataType.get_data_bytes(vformat) 398 event_id = parser.parse_simple_field(vformat, segment[0:data_bytes]) 399 if len(event_id) == 0: 400 return False 401 trace_event = [timestamp, core_id, event_id[0], segment] 402 viewer = parser.get_viewer() 403 viewer.append_trace_event(trace_event) 404 return True 405 pass 406 407 408class TraceEventWrapper(OperatorInterface): 409 """ 410 功能描述: 声明HiTrace文件的1条trace event的格式 411 """ 412 def __init__(self, event_header: TraceEventHeader, event_content: TraceEventContent) -> None: 413 self.event_header = event_header 414 self.event_content = event_content 415 pass 416 417 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 418 segment = segment or [] 419 cur_post = 0 420 while cur_post < len(segment): 421 (cur_post, data) = self.next_event_header(cur_post, segment) 422 if self.event_header.accept(parser, data) is False: 423 break 424 (cur_post, data) = self.next_event_context(cur_post, segment, parser) 425 if self.event_content.accept(parser, data) is False: 426 break 427 return True 428 429 def next_event_context(self, cur_post: int, segment: List, parser: TraceFileParserInterface) -> tuple: 430 context = parser.get_context() 431 (event_size, align_event_size) = context.get_param(TraceParseContext.CONTEXT_EVENT_SIZE) 432 data = segment[cur_post: cur_post + event_size] 433 next_post = cur_post + align_event_size 434 return (next_post, data) 435 436 def next_event_header(self, cur_post: int, segment: List) -> tuple: 437 data = segment[cur_post: cur_post + self.event_header.field_size] 438 next_post = cur_post + self.event_header.field_size 439 return (next_post, data) 440 pass 441 442 443class PageWrapper(OperatorInterface): 444 """ 445 功能描述: 声明HiTrace文件的1个page的格式 446 """ 447 # trace event的page是固定大小, trace event的内容大小，应该是4096的整数倍，trace event需要按页进行解析 448 TRACE_PAGE_SIZE = 4096 449 450 def __init__(self, page_header: PageHeader, trace_event: TraceEventWrapper) -> None: 451 self.page_header = page_header 452 self.trace_event = trace_event 453 pass 454 455 def next(self, cur_post: int, segment: List) -> tuple: 456 if segment is None: 457 return (-1, None) 458 459 if (cur_post + PageWrapper.TRACE_PAGE_SIZE) > len(segment): 460 return (-1, None) 461 462 data = segment[cur_post: cur_post + PageWrapper.TRACE_PAGE_SIZE] 463 next_post = cur_post + PageWrapper.TRACE_PAGE_SIZE 464 return (next_post, data) 465 466 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 467 segment = segment or [] 468 cur_post = 0 469 while True: 470 (cur_post, data) = self.next(cur_post, segment) 471 if data is None: 472 break 473 474 self.page_header.accept(parser, data[0: self.page_header.field_size]) 475 self.trace_event.accept(parser, data[self.page_header.field_size:]) 476 return True 477 pass 478 479 480class RawTraceSegment(SegmentOperator): 481 """ 482 功能描述: 声明HiTrace文件trace_pipe_raw内容的段格式 483 """ 484 def __init__(self) -> None: 485 super().__init__(FieldType.SEGMENT_RAW_TRACE) 486 self.field = PageWrapper( 487 PageHeader([ 488 PageHeader.ITEM_TIMESTAMP, 489 PageHeader.ITEM_LENGTH, 490 PageHeader.ITEM_CORE_ID, 491 ]), 492 TraceEventWrapper( 493 TraceEventHeader([ 494 TraceEventHeader.ITEM_TIMESTAMP_OFFSET, 495 TraceEventHeader.ITEM_EVENT_SIZE 496 ]), 497 TraceEventContent(), 498 ) 499 ) 500 pass 501 502 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 503 segment = segment or [] 504 self.field.accept(parser, segment) 505 return True 506 507 508class EventFormatSegment(SegmentOperator): 509 """ 510 功能描述: 声明HiTrace文件event/format内容的段格式 511 """ 512 def __init__(self) -> None: 513 super().__init__(FieldType.SEGMENT_EVENTS_FORMAT) 514 pass 515 516 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 517 segment = segment or [] 518 events_format = parser.parse_event_format(segment) 519 context = parser.get_context() 520 context.set_param(TraceParseContext.CONTEXT_EVENT_FORMAT, events_format) 521 return True 522 523 524class CmdLinesSegment(SegmentOperator): 525 """ 526 功能描述: 声明HiTrace文件/sys/kernel/tracing/saved_cmdlines内容的段格式 527 """ 528 def __init__(self) -> None: 529 super().__init__(FieldType.SEGMENT_CMDLINES) 530 pass 531 532 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 533 segment = segment or [] 534 cmd_lines = parser.parse_cmd_lines(segment) 535 context = parser.get_context() 536 context.set_param(TraceParseContext.CONTEXT_CMD_LINES, cmd_lines) 537 return True 538 539 540class TidGroupsSegment(SegmentOperator): 541 """ 542 功能描述: 声明HiTrace文件/sys/kernel/tracing/saved_tgids内容的段格式 543 """ 544 def __init__(self) -> None: 545 super().__init__(FieldType.SEGMENT_TGIDS) 546 pass 547 548 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 549 segment = segment or [] 550 tid_groups = parser.parse_tid_groups(segment) 551 context = parser.get_context() 552 context.set_param(TraceParseContext.CONTEXT_TID_GROUPS, tid_groups) 553 return True 554 555 556class PrintkFormatSegment(SegmentOperator): 557 """ 558 功能描述: 声明HiTrace文件/sys/kernel/tracing/printk_formats内容的段格式 559 """ 560 def __init__(self) -> None: 561 super().__init__(FieldType.SEGMENT_PRINTK_FORMATS) 562 pass 563 564 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 565 print("in parse_printk_formats") 566 return True 567 568 569class KallSymsSegment(SegmentOperator): 570 """ 571 功能描述: 声明HiTrace文件Kall Sysms内容的段格式 572 """ 573 def __init__(self) -> None: 574 super().__init__(FieldType.SEGMENT_KALLSYMS) 575 pass 576 577 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 578 print("in parse_kallsyms") 579 return True 580 581 582class HeaderPageSegment(SegmentOperator): 583 """ 584 功能描述: 声明HiTrace文件header page内容的段格式 585 """ 586 def __init__(self) -> None: 587 super().__init__(FieldType.SEGMENT_HEADER_PAGE) 588 pass 589 590 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 591 print("in parse_header_page") 592 return True 593 594 595class UnSupportSegment(FieldOperator): 596 """ 597 功能描述: 声明HiTrace文件还不支持解析的段 598 """ 599 def __init__(self) -> None: 600 super().__init__(FieldType.SEGMENT_UNSUPPORT, -1, "", []) 601 pass 602 603 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 604 print("unsupport segment") 605 return True 606 607 608class SegmentWrapper(FieldOperator): 609 """ 610 功能描述: 声明HiTrace文件包含所有段的格式 611 """ 612 # 描述段的pack的格式 613 FORMAT = "II" 614 615 ITEM_SEGMENT_TYPE = 0 616 ITEM_SEGMENT_SIZE = 1 617 618 def __init__(self, fields: List) -> None: 619 super().__init__( 620 FieldType.SEGMENT_SEGMENTS, 621 DataType.get_data_bytes(SegmentWrapper.FORMAT), 622 SegmentWrapper.FORMAT, 623 [ 624 SegmentWrapper.ITEM_SEGMENT_TYPE, 625 SegmentWrapper.ITEM_SEGMENT_SIZE, 626 ] 627 ) 628 self.fields = fields 629 pass 630 631 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 632 segment = segment or [] 633 while True: 634 data = parser.get_segment_data(self.field_size) 635 values = parser.parse_simple_field(self.format, data) 636 if len(values) == 0: 637 break 638 segment_type = values[SegmentWrapper.ITEM_SEGMENT_TYPE] 639 segment_size = values[SegmentWrapper.ITEM_SEGMENT_SIZE] 640 conext = parser.get_context() 641 642 segment = self.get_segment(segment_type, conext.get_param(TraceParseContext.CONTEXT_CPU_NUM)) 643 segment_data = parser.get_segment_data(segment_size) 644 segment.accept(parser, segment_data) 645 pass 646 return True 647 648 def get_segment(self, segment_type: int, cpu_num: int) -> FieldOperator: 649 for field in self.fields: 650 if field.field_type == segment_type: 651 return field 652 653 if field.field_type < FieldType.SEGMENT_RAW_TRACE: 654 continue 655 656 if field.field_type > FieldType.SEGMENT_RAW_TRACE + cpu_num: 657 continue 658 return field 659 return UnSupportSegment() 660 661 662class TraceFile: 663 """ 664 功能描述: 读取hitrace的二进制文件 665 """ 666 def __init__(self, name: str) -> None: 667 self.name = name 668 flags = os.O_RDONLY 669 if sys.platform == 'win32': 670 flags = os.O_RDONLY | os.O_BINARY 671 mode = stat.S_IRUSR 672 self.file = os.fdopen(os.open(name, flags, mode), 'rb') 673 self.size = os.path.getsize(name) 674 self.cur_post = 0 675 pass 676 677 def read_data(self, block_size: int) -> List: 678 """ 679 功能描述: 从文件当前位置读取blockSize字节的内容 680 参数: 读取字节数 681 返回值: 文件内容 682 """ 683 if (self.cur_post + block_size) > self.size: 684 return None 685 self.cur_post = self.cur_post + block_size 686 return self.file.read(block_size) 687 688 689class TraceFileFormat(OperatorInterface): 690 """ 691 功能描述: 声明整个HiTrace文件的二进制格式 692 """ 693 def __init__(self) -> None: 694 self.fields = [ 695 FileHeader([ 696 FileHeader.ITEM_MAGIC_NUMBER, 697 FileHeader.ITEM_FILE_TYPE, 698 FileHeader.ITEM_VERSION_NUMBER, 699 FileHeader.ITEM_RESERVED 700 ]), 701 SegmentWrapper([ 702 CmdLinesSegment(), 703 TidGroupsSegment(), 704 EventFormatSegment(), 705 RawTraceSegment(), 706 PrintkFormatSegment(), 707 KallSymsSegment(), 708 HeaderPageSegment(), 709 ]) 710 ] 711 pass 712 713 def accept(self, parser: TraceFileParserInterface, segment=None) -> bool: 714 for field in self.fields: 715 if field.accept(parser) is False: 716 return False 717 return True 718 719 720class Viewer: 721 @abstractmethod 722 def set_context(self, context: TraceParseContext) -> None: 723 pass 724 725 @abstractmethod 726 def calculate(self, timestamp: int, core_id: int, event_id: int, segment: List) -> None: 727 pass 728 729 @abstractmethod 730 def show(self) -> None: 731 pass 732 733 734class SysTraceViewer(Viewer): 735 COMM_STR_MAX = 16 736 PID_STR_MAX = 6 737 TGID_STR_MAX = 5 738 CPU_STR_MAX = 3 739 TS_SECS_MIN = 5 740 TS_MICRO_SECS = 6 741 742 TRACE_FLAG_IRQS_OFF = 0x01 743 TRACE_FLAG_IRQS_NOSUPPORT = 0x02 744 TRACE_FLAG_NEED_RESCHED = 0x04 745 TRACE_FLAG_HARDIRQ = 0x08 746 TRACE_FLAG_SOFTIRQ = 0x10 747 TRACE_FLAG_PREEMPT_RESCHED = 0x20 748 TRACE_FLAG_NMI = 0x40 749 750 def __init__(self, output_file: str): 751 self.out_file = output_file 752 self.format_miss_cnt = 0 753 self.format_miss_set = set() 754 self.trace_event_count_dict = {} # trace event count dict 755 self.trace_event_mem_dict = {} # trace event mem dict 756 self.get_not_found_format = set() 757 self.systrace = [] 758 pass 759 760 def set_context(self, context: TraceParseContext) -> None: 761 self.context = context 762 self.events_format = context.get_param(TraceParseContext.CONTEXT_EVENT_FORMAT) 763 self.cmd_lines = context.get_param(TraceParseContext.CONTEXT_CMD_LINES) 764 self.tgids = context.get_param(TraceParseContext.CONTEXT_TID_GROUPS) 765 pass 766 767 def calculate(self, timestamp: int, core_id: int, event_id: int, segment: List, context: TraceParseContext) -> None: 768 if event_id in self.trace_event_count_dict: 769 self.trace_event_count_dict[event_id] += 1 770 else: 771 self.trace_event_count_dict[event_id] = 1 772 773 if event_id in self.trace_event_mem_dict: 774 self.trace_event_mem_dict[event_id] += len(segment) 775 else: 776 self.trace_event_mem_dict[event_id] = len(segment) 777 778 event_format = self.events_format.get(event_id, "") 779 if event_format == "": 780 # current event format is not found in trace file format data. 781 self.format_miss_cnt = self.format_miss_cnt + 1 782 self.format_miss_set.add(event_id) 783 return 784 785 fields = event_format["fields"] 786 one_event = {} 787 one_event["id"] = event_id 788 one_event["name"] = event_format["name"] 789 one_event["print_fmt"] = event_format["print_fmt"] 790 one_event["fields"] = {} 791 for field in fields: 792 offset = field["offset"] 793 size = field["size"] 794 one_event["fields"][field["name"]] = segment[offset:offset + size] 795 796 systrace = self.generate_one_event_str(segment, core_id, timestamp, one_event) 797 self.systrace.append([timestamp, systrace]) 798 pass 799 800 def generate_one_event_str(self, data: List, cpu_id: int, time_stamp: int, one_event: dict) -> str: 801 pid = int.from_bytes(one_event["fields"]["common_pid"], byteorder='little') 802 event_str = "" 803 804 cmd_line = self.cmd_lines.get(pid, "") 805 if pid == 0: 806 event_str += "<idle>" 807 elif cmd_line != "": 808 event_str += cmd_line 809 else: 810 event_str += "<...>" 811 event_str = event_str.rjust(SysTraceViewer.COMM_STR_MAX) 812 event_str += "-" 813 814 event_str += str(pid).ljust(SysTraceViewer.PID_STR_MAX) 815 816 tgid = self.tgids.get(pid) 817 if tgid is not None: 818 tgid = "%d" % tgid 819 event_str += "(" + tgid.rjust(SysTraceViewer.TGID_STR_MAX) + ")" 820 else: 821 event_str += "(-----)" 822 823 event_str += " [" + str(cpu_id).zfill(SysTraceViewer.CPU_STR_MAX) + "] " 824 825 flags = int.from_bytes(one_event["fields"]["common_flags"], byteorder='little') 826 preempt_count = int.from_bytes(one_event["fields"]["common_preempt_count"], byteorder='little') 827 if flags | preempt_count != 0: 828 event_str += self.trace_flags_to_str(flags, preempt_count) + " " 829 else: 830 event_str += ".... " 831 832 if time_stamp % 1000 >= 500: 833 time_stamp_str = str((time_stamp // 1000) + 1) 834 else: 835 time_stamp_str = str(time_stamp // 1000) 836 ts_secs = time_stamp_str[:-6].rjust(SysTraceViewer.TS_SECS_MIN) 837 ts_micro_secs = time_stamp_str[-6:] 838 event_str += ts_secs + "." + ts_micro_secs + ": " 839 840 parse_result = parse_functions.parse(one_event["print_fmt"], data, one_event) 841 if parse_result is None: 842 self.get_not_found_format.add(str(one_event["name"])) 843 else: 844 event_str += str(one_event["name"]) + ": " + parse_result 845 846 return event_str 847 848 def trace_flags_to_str(self, flags: int, preempt_count: int) -> str: 849 result = "" 850 irqs_off = '.' 851 if flags & SysTraceViewer.TRACE_FLAG_IRQS_OFF != 0: 852 irqs_off = 'd' 853 elif flags & SysTraceViewer.TRACE_FLAG_IRQS_NOSUPPORT != 0: 854 irqs_off = 'X' 855 result += irqs_off 856 857 need_resched = '.' 858 is_need_resched = flags & SysTraceViewer.TRACE_FLAG_NEED_RESCHED 859 is_preempt_resched = flags & SysTraceViewer.TRACE_FLAG_PREEMPT_RESCHED 860 if is_need_resched != 0 and is_preempt_resched != 0: 861 need_resched = 'N' 862 elif is_need_resched != 0: 863 need_resched = 'n' 864 elif is_preempt_resched != 0: 865 need_resched = 'p' 866 result += need_resched 867 868 nmi_flag = flags & SysTraceViewer.TRACE_FLAG_NMI 869 hard_irq = flags & SysTraceViewer.TRACE_FLAG_HARDIRQ 870 soft_irq = flags & SysTraceViewer.TRACE_FLAG_SOFTIRQ 871 irq_char = '.' 872 if nmi_flag != 0 and hard_irq != 0: 873 irq_char = 'Z' 874 elif nmi_flag != 0: 875 irq_char = 'z' 876 elif hard_irq != 0 and soft_irq != 0: 877 irq_char = 'H' 878 elif hard_irq != 0: 879 irq_char = 'h' 880 elif soft_irq != 0: 881 irq_char = 's' 882 result += irq_char 883 884 if preempt_count != 0: 885 result += "0123456789abcdef"[preempt_count & 0x0F] 886 else: 887 result += "." 888 889 return result 890 891 def show(self) -> None: 892 self.save_systrace() 893 self.show_stat() 894 pass 895 896 def save_systrace(self) -> None: 897 outfile_flags = os.O_RDWR | os.O_CREAT 898 outfile_mode = stat.S_IRUSR | stat.S_IWUSR 899 900 result = sorted(self.systrace, key=lambda x: x[0]) 901 outfile = os.fdopen(os.open(self.out_file, outfile_flags, outfile_mode), 'w', encoding="utf-8") 902 outfile.write(TRACE_TXT_HEADER_FORMAT) 903 for line in result: 904 outfile.write("{}\n".format(line[1])) 905 outfile.close() 906 907 def show_stat(self) -> None: 908 for name in self.get_not_found_format: 909 print("Error: function parse_%s not found" % name) 910 print("Trace format miss count: %d" % self.format_miss_cnt) 911 print("Trace format id missed set:") 912 print(self.format_miss_set) 913 914 count_total = sum(self.trace_event_count_dict.values()) 915 mem_total = sum(self.trace_event_mem_dict.values()) / 1024 # KB 916 print(f"Trace counter: total count({count_total}), total mem({mem_total:.3f}KB)") 917 for format_id, count in self.trace_event_count_dict.items(): 918 count_percentage = count / count_total * 100 919 mem = self.trace_event_mem_dict.get(format_id, 0) / 1024 920 mem_percentage = mem / mem_total * 100 921 print(f"ID {format_id}: count={count}, count percentage={count_percentage:.5f}%, mem={mem:.3f}KB, mem percentage={mem_percentage:.5f}%") 922 923 924class TraceViewer(TraceViewerInterface): 925 def __init__(self, viewers: List) -> None: 926 self.trace_events = [] 927 self.viewers = viewers 928 pass 929 930 def append_trace_event(self, trace_event: List) -> None: 931 self.trace_events.append(trace_event) 932 pass 933 934 def calculate(self, context: TraceParseContext): 935 for viewer in self.viewers: 936 viewer.set_context(context) 937 938 for timestamp, core_id, event_id, segment in self.trace_events: 939 for viewer in self.viewers: 940 viewer.calculate(timestamp, core_id, event_id, segment, context) 941 942 for viewer in self.viewers: 943 viewer.show() 944 pass 945 946 947class TraceFileParser(TraceFileParserInterface): 948 def __init__(self, file: TraceFile, vformat: TraceFileFormat, viewer: TraceViewer, context: TraceParseContext) -> None: 949 self.trace_file = file 950 self.trace_format = vformat 951 self.trace_viewer = viewer 952 self.context = context 953 pass 954 955 def parse_simple_field(self, vformat: str, data: List) -> tuple: 956 if data is None: 957 return {} 958 if DataType.get_data_bytes(vformat) != len(data): 959 return {} 960 unpack_value = struct.unpack(vformat, data) 961 return unpack_value 962 963 def parse_event_format(self, data: List) -> dict: 964 def parse_events_format_field(field_line): 965 field_info = field_line.split(";") 966 field_info[0] = field_info[0].lstrip() 967 field_info[1] = field_info[1].lstrip() 968 field_info[2] = field_info[2].lstrip() 969 field_info[3] = field_info[3].lstrip() 970 971 field = {} 972 type_name_pos = field_info[0].rfind(" ") 973 field["type"] = field_info[0][len("field:"):type_name_pos] 974 field["name"] = field_info[0][type_name_pos + 1:] 975 field["offset"] = int(field_info[1][len("offset:"):]) 976 field["size"] = int(field_info[2][len("size:"):]) 977 field["signed"] = field_info[3][len("signed:"):] 978 return field 979 980 events_format = {} 981 name_line_prefix = "name: " 982 id_line_prefix = "ID: " 983 field_line_prefix = "field:" 984 print_fmt_line_prefix = "print fmt: " 985 986 events_format_lines = data.decode('utf-8').split("\n") 987 event_format = {"fields": []} 988 for line in events_format_lines: 989 line = line.lstrip() 990 if line.startswith(name_line_prefix): 991 event_format["name"] = line[len(name_line_prefix):] 992 elif line.startswith(id_line_prefix): 993 event_format["id"] = int(line[len(id_line_prefix):]) 994 elif line.startswith(field_line_prefix): 995 event_format["fields"].append(parse_events_format_field(line)) 996 elif line.startswith(print_fmt_line_prefix): 997 event_format["print_fmt"] = line[len(print_fmt_line_prefix):] 998 events_format[event_format["id"]] = event_format 999 event_format = {"fields": []} 1000 return events_format 1001 1002 def parse_cmd_lines(self, data: List) -> dict: 1003 cmd_lines = {} 1004 1005 if isinstance(data, bytes): 1006 cmd_lines_list = data.decode('utf-8').split("\n") 1007 elif isinstance(data, list): 1008 if all(isinstance(item, bytes) for item in data): 1009 cmd_lines_list = b''.join(data).decode('utf-8').split("\n") 1010 else: 1011 cmd_lines_list = ''.join(data).split("\n") 1012 else: 1013 raise ValueError("Unsupported data type: must be bytes or list of bytes/str") 1014 1015 for cmd_line in cmd_lines_list: 1016 pos = cmd_line.find(" ") 1017 if pos == -1: 1018 continue 1019 cmd_lines[int(cmd_line[:pos])] = cmd_line[pos + 1:] 1020 return cmd_lines 1021 1022 def parse_tid_groups(self, data: List) -> dict: 1023 tgids = {} 1024 tgids_lines_list = data.decode('utf-8').split("\n") 1025 for tgids_line in tgids_lines_list: 1026 pos = tgids_line.find(" ") 1027 if pos == -1: 1028 continue 1029 tgids[int(tgids_line[:pos])] = int(tgids_line[pos + 1:]) 1030 return tgids 1031 1032 def get_segment_data(self, segment_size) -> List: 1033 return self.trace_file.read_data(segment_size) 1034 1035 def get_context(self) -> TraceParseContext: 1036 return self.context 1037 1038 def get_viewer(self) -> TraceViewerInterface: 1039 return self.trace_viewer 1040 1041 def parse(self) -> None: 1042 self.trace_format.accept(self) 1043 self.trace_viewer.calculate(self.context) 1044 pass 1045 1046 1047def parse_binary_trace_file() -> None: 1048 global binary_file 1049 global out_file 1050 file = TraceFile(binary_file) 1051 vformat = TraceFileFormat() 1052 viewer = TraceViewer([ 1053 SysTraceViewer(out_file) 1054 ]) 1055 context = TraceParseContext() 1056 parser = TraceFileParser(file, vformat, viewer, context) 1057 parser.parse() 1058 pass 1059 1060 1061def main() -> None: 1062 parse_options() 1063 1064 if file_dir == '': 1065 if text_file != '': 1066 parse_text_trace_file() 1067 else: 1068 parse_binary_trace_file() 1069 else: 1070 for file in os.listdir(file_dir): 1071 if file.find('.sys') != -1: 1072 print(file) 1073 global binary_file 1074 global out_file 1075 binary_file = os.path.join(file_dir, file) 1076 out_file = os.path.join(os.path.split(binary_file)[0], 1077 '%s%s' % (os.path.split(binary_file)[-1].split('.')[0], '.ftrace')) 1078 parse_binary_trace_file() 1079 1080 1081if __name__ == '__main__': 1082 main() 1083