1#!/usr/bin/env python3 2 3# 4# Copyright (C) 2019 The Android Open Source Project 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18 19# 20# Dependencies: 21# 22# $> sudo apt-get install python3-pip 23# $> pip3 install --user protobuf sqlalchemy sqlite3 24# 25 26import optparse 27import os 28import re 29import sys 30import tempfile 31from pathlib import Path 32from datetime import timedelta 33from typing import Iterable, Optional, List 34 35DIR = os.path.abspath(os.path.dirname(__file__)) 36sys.path.append(os.path.dirname(DIR)) 37from iorap.generated.TraceFile_pb2 import * 38from iorap.lib.inode2filename import Inode2Filename 39 40parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 41sys.path.append(parent_dir_name) 42from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \ 43 RawFtraceEntry 44import lib.cmd_utils as cmd_utils 45 46_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes. 47ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3] 48TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath( 49 'external/perfetto/tools/traceconv') 50 51class PageRun: 52 """ 53 Intermediate representation for a run of one or more pages. 54 """ 55 def __init__(self, device_number: int, inode: int, offset: int, length: int): 56 self.device_number = device_number 57 self.inode = inode 58 self.offset = offset 59 self.length = length 60 61 def __str__(self): 62 return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \ 63 %(self.device_number, self.inode, self.offset, self.length) 64 65def debug_print(msg): 66 #print(msg) 67 pass 68 69UNDER_LAUNCH = False 70 71def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]): 72 global _PAGE_SIZE 73 74 runs = [ 75 PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs, 76 length=_PAGE_SIZE) 77 for pg_entry in page_cache_entries 78 ] 79 80 for r in runs: 81 debug_print(r) 82 83 print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE)) 84 85 return runs 86 87def optimize_page_runs(page_runs): 88 new_entries = [] 89 last_entry = None 90 for pg_entry in page_runs: 91 if last_entry: 92 if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode: 93 # we are dealing with a run for the same exact file as a previous run. 94 if pg_entry.offset == last_entry.offset + last_entry.length: 95 # trivially contiguous entries. merge them together. 96 last_entry.length += pg_entry.length 97 continue 98 # Default: Add the run without merging it to a previous run. 99 last_entry = pg_entry 100 new_entries.append(pg_entry) 101 return new_entries 102 103def is_filename_matching_filter(file_name, filters=[]): 104 """ 105 Blacklist-style regular expression filters. 106 107 :return: True iff file_name has an RE match in one of the filters. 108 """ 109 for filt in filters: 110 res = re.search(filt, file_name) 111 if res: 112 return True 113 114 return False 115 116def build_protobuf(page_runs, inode2filename, filters=[]): 117 trace_file = TraceFile() 118 trace_file_index = trace_file.index 119 120 file_id_counter = 0 121 file_id_map = {} # filename -> id 122 123 stats_length_total = 0 124 filename_stats = {} # filename -> total size 125 126 skipped_inode_map = {} 127 filtered_entry_map = {} # filename -> count 128 129 for pg_entry in page_runs: 130 fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode) 131 if not fn: 132 skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1 133 continue 134 135 filename = fn 136 137 if filters and not is_filename_matching_filter(filename, filters): 138 filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1 139 continue 140 141 file_id = file_id_map.get(filename) 142 # file_id could 0, which satisfies "if file_id" and causes duplicate 143 # filename for file id 0. 144 if file_id is None: 145 file_id = file_id_counter 146 file_id_map[filename] = file_id_counter 147 file_id_counter = file_id_counter + 1 148 149 file_index_entry = trace_file_index.entries.add() 150 file_index_entry.id = file_id 151 file_index_entry.file_name = filename 152 153 # already in the file index, add the file entry. 154 file_entry = trace_file.list.entries.add() 155 file_entry.index_id = file_id 156 file_entry.file_length = pg_entry.length 157 stats_length_total += file_entry.file_length 158 file_entry.file_offset = pg_entry.offset 159 160 filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length 161 162 for inode, count in skipped_inode_map.items(): 163 print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count)) 164 165 print("Stats: Sum of lengths %d" %(stats_length_total)) 166 167 if filters: 168 print("Filter: %d total files removed." %(len(filtered_entry_map))) 169 170 for fn, count in filtered_entry_map.items(): 171 print("Filter: File '%s' removed '%d' entries." %(fn, count)) 172 173 for filename, file_size in filename_stats.items(): 174 print("%s,%s" %(filename, file_size)) 175 176 return trace_file 177 178def calc_trace_end_time(trace2db: Trace2Db, 179 trace_duration: Optional[timedelta]) -> float: 180 """ 181 Calculates the end time based on the trace duration. 182 The start time is the first receiving mm file map event. 183 The end time is the start time plus the trace duration. 184 All of them are in milliseconds. 185 """ 186 # If the duration is not set, assume all time is acceptable. 187 if trace_duration is None: 188 # float('inf') 189 return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf') 190 191 first_event = trace2db.session.query(MmFilemapAddToPageCache).join( 192 MmFilemapAddToPageCache.raw_ftrace_entry).order_by( 193 RawFtraceEntry.timestamp).first() 194 195 # total_seconds() will return a float number. 196 return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds() 197 198def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]): 199 end_time = calc_trace_end_time(trace2db, trace_duration) 200 # SELECT * FROM tbl ORDER BY id; 201 return trace2db.session.query(MmFilemapAddToPageCache).join( 202 MmFilemapAddToPageCache.raw_ftrace_entry).filter( 203 RawFtraceEntry.timestamp <= end_time).order_by( 204 MmFilemapAddToPageCache.id).all() 205 206def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str, 207 path_to_tmp_systrace: str) -> None: 208 """ Transforms the systrace file from perfetto trace. """ 209 cmd_utils.run_command_nofail([str(TRACECONV_BIN), 210 'systrace', 211 path_to_perfetto_trace, 212 path_to_tmp_systrace]) 213 214 215def run(sql_db_path:str, 216 trace_file:str, 217 trace_duration:Optional[timedelta], 218 output_file:str, 219 inode_table:str, 220 filter:List[str]) -> int: 221 trace2db = Trace2Db(sql_db_path) 222 # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache. 223 trace2db.set_raw_ftrace_entry_filter(\ 224 lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache') 225 # TODO: parse multiple trace files here. 226 parse_count = trace2db.parse_file_into_db(trace_file) 227 228 mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db, 229 trace_duration) 230 print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows))) 231 232 page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows) 233 print("DONE. Converted %d entries" %(len(page_runs))) 234 235 # TODO: flags to select optimizations. 236 optimized_page_runs = optimize_page_runs(page_runs) 237 print("DONE. Optimized down to %d entries" %(len(optimized_page_runs))) 238 239 print("Build protobuf...") 240 trace_file = build_protobuf(optimized_page_runs, inode_table, filter) 241 242 print("Write protobuf to file...") 243 output_file = open(output_file, 'wb') 244 output_file.write(trace_file.SerializeToString()) 245 output_file.close() 246 247 print("DONE") 248 249 # TODO: Silent running mode [no output except on error] for build runs. 250 251 return 0 252 253def main(argv): 254 parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb") 255 parser.add_option('-i', dest='inode_data_file', metavar='FILE', 256 help='Read cached inode data from a file saved earlier with pagecache.py -d') 257 parser.add_option('-t', dest='trace_file', metavar='FILE', 258 help='Path to systrace file (trace.html) that will be parsed') 259 parser.add_option('--perfetto-trace', dest='perfetto_trace_file', 260 metavar='FILE', 261 help='Path to perfetto trace that will be parsed') 262 263 parser.add_option('--db', dest='sql_db', metavar='FILE', 264 help='Path to intermediate sqlite3 database [default: in-memory].') 265 266 parser.add_option('-f', dest='filter', action="append", default=[], 267 help="Add file filter. All file entries not matching one of the filters are discarded.") 268 269 parser.add_option('-l', dest='launch_lock', action="store_true", default=False, 270 help="Exclude all events not inside launch_lock") 271 272 parser.add_option('-o', dest='output_file', metavar='FILE', 273 help='Output protobuf file') 274 275 parser.add_option('--duration', dest='trace_duration', action="store", 276 type=int, help='The duration of trace in milliseconds.') 277 278 options, categories = parser.parse_args(argv[1:]) 279 280 # TODO: OptionParser should have some flags to make these mandatory. 281 if not options.inode_data_file: 282 parser.error("-i is required") 283 if not options.trace_file and not options.perfetto_trace_file: 284 parser.error("one of -t or --perfetto-trace is required") 285 if options.trace_file and options.perfetto_trace_file: 286 parser.error("please enter either -t or --perfetto-trace, not both") 287 if not options.output_file: 288 parser.error("-o is required") 289 290 if options.launch_lock: 291 print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.") 292 293 inode_table = Inode2Filename.new_from_filename(options.inode_data_file) 294 295 sql_db_path = ":memory:" 296 if options.sql_db: 297 sql_db_path = options.sql_db 298 299 trace_duration = timedelta(milliseconds=options.trace_duration) if \ 300 options.trace_duration is not None else None 301 302 # if the input is systrace 303 if options.trace_file: 304 return run(sql_db_path, 305 options.trace_file, 306 trace_duration, 307 options.output_file, 308 inode_table, 309 options.filter) 310 311 # if the input is perfetto trace 312 # TODO python 3.7 switch to using nullcontext 313 with tempfile.NamedTemporaryFile() as trace_file: 314 transform_perfetto_trace_to_systrace(options.perfetto_trace_file, 315 trace_file.name) 316 return run(sql_db_path, 317 trace_file.name, 318 trace_duration, 319 options.output_file, 320 inode_table, 321 options.filter) 322 323if __name__ == '__main__': 324 print(sys.argv) 325 sys.exit(main(sys.argv)) 326