1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from __future__ import absolute_import 6from __future__ import division 7from __future__ import print_function 8import collections 9import gzip 10import json 11import logging 12import os 13import platform 14import shutil 15import subprocess 16import tempfile 17import time 18import traceback 19import six 20 21 22try: 23 StringTypes = six.string_types # pylint: disable=invalid-name 24except NameError: 25 StringTypes = str 26 27 28_TRACING_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 29 os.path.pardir, os.path.pardir) 30_TRACE2HTML_PATH = os.path.join(_TRACING_DIR, 'bin', 'trace2html') 31 32MIB = 1024 * 1024 33 34class TraceDataPart(object): 35 """Trace data can come from a variety of tracing agents. 36 37 Data from each agent is collected into a trace "part" and accessed by the 38 following fixed field names. 39 """ 40 def __init__(self, raw_field_name): 41 self._raw_field_name = raw_field_name 42 43 def __repr__(self): 44 return 'TraceDataPart("%s")' % self._raw_field_name 45 46 @property 47 def raw_field_name(self): 48 return self._raw_field_name 49 50 def __eq__(self, other): 51 return self.raw_field_name == other.raw_field_name 52 53 def __hash__(self): 54 return hash(self.raw_field_name) 55 56 57ANDROID_PROCESS_DATA_PART = TraceDataPart('androidProcessDump') 58ATRACE_PART = TraceDataPart('systemTraceEvents') 59ATRACE_PROCESS_DUMP_PART = TraceDataPart('atraceProcessDump') 60CHROME_TRACE_PART = TraceDataPart('traceEvents') 61CPU_TRACE_DATA = TraceDataPart('cpuSnapshots') 62TELEMETRY_PART = TraceDataPart('telemetry') 63WALT_TRACE_PART = TraceDataPart('waltTraceEvents') 64CGROUP_TRACE_PART = TraceDataPart('cgroupDump') 65 66ALL_TRACE_PARTS = {ANDROID_PROCESS_DATA_PART, 67 ATRACE_PART, 68 ATRACE_PROCESS_DUMP_PART, 69 CHROME_TRACE_PART, 70 CPU_TRACE_DATA, 71 TELEMETRY_PART} 72 73 74class _TraceData(object): 75 """Provides read access to traces collected from multiple tracing agents. 76 77 Instances are created by calling the AsData() method on a TraceDataWriter. 78 """ 79 def __init__(self, raw_data): 80 self._raw_data = raw_data 81 82 def HasTracesFor(self, part): 83 return bool(self.GetTracesFor(part)) 84 85 def GetTracesFor(self, part): 86 """Return the list of traces for |part| in string or dictionary forms.""" 87 if not isinstance(part, TraceDataPart): 88 raise TypeError('part must be a TraceDataPart instance') 89 return self._raw_data.get(part.raw_field_name, []) 90 91 def GetTraceFor(self, part): 92 traces = self.GetTracesFor(part) 93 assert len(traces) == 1 94 return traces[0] 95 96 97_TraceItem = collections.namedtuple( 98 '_TraceItem', ['part_name', 'handle']) 99 100 101class TraceDataException(Exception): 102 """Exception raised by TraceDataBuilder via RecordTraceDataException().""" 103 104 105class TraceDataBuilder(object): 106 """TraceDataBuilder helps build up a trace from multiple trace agents. 107 108 Note: the collected trace data is maintained in a set of temporary files to 109 be later processed e.g. by the Serialize() method. To ensure proper clean up 110 of such files clients must call the CleanUpTraceData() method or, even easier, 111 use the context manager API, e.g.: 112 113 with trace_data.TraceDataBuilder() as builder: 114 builder.AddTraceFor(trace_part, data) 115 builder.Serialize(output_file) 116 """ 117 def __init__(self): 118 self._traces = [] 119 self._frozen = False 120 self._temp_dir = tempfile.mkdtemp() 121 self._exceptions = [] 122 123 def __enter__(self): 124 return self 125 126 def __exit__(self, *args): 127 self.CleanUpTraceData() 128 129 def OpenTraceHandleFor(self, part, suffix): 130 """Open a file handle for writing trace data into it. 131 132 Args: 133 part: A TraceDataPart instance. 134 suffix: A string used as file extension and identifier for the format 135 of the trace contents, e.g. '.json'. Can also append '.gz' to 136 indicate gzipped content, e.g. '.json.gz'. 137 """ 138 if not isinstance(part, TraceDataPart): 139 raise TypeError('part must be a TraceDataPart instance') 140 if self._frozen: 141 raise RuntimeError('trace data builder is no longer open for writing') 142 trace = _TraceItem( 143 part_name=part.raw_field_name, 144 handle=tempfile.NamedTemporaryFile( 145 delete=False, dir=self._temp_dir, suffix=suffix)) 146 self._traces.append(trace) 147 return trace.handle 148 149 def AddTraceFileFor(self, part, trace_file): 150 """Move a file with trace data into this builder. 151 152 This is useful for situations where a client might want to start collecting 153 trace data into a file, even before the TraceDataBuilder itself is created. 154 155 Args: 156 part: A TraceDataPart instance. 157 trace_file: A path to a file containing trace data. Note: for efficiency 158 the file is moved rather than copied into the builder. Therefore the 159 source file will no longer exist after calling this method; and the 160 lifetime of the trace data will thereafter be managed by this builder. 161 """ 162 _, suffix = os.path.splitext(trace_file) 163 with self.OpenTraceHandleFor(part, suffix) as handle: 164 pass 165 if os.name == 'nt': 166 # On windows os.rename won't overwrite, so the destination path needs to 167 # be removed first. 168 os.remove(handle.name) 169 os.rename(trace_file, handle.name) 170 171 def AddTraceFor(self, part, data, allow_unstructured=False): 172 """Record new trace data into this builder. 173 174 Args: 175 part: A TraceDataPart instance. 176 data: The trace data to write: a json-serializable dict, or unstructured 177 text data as a string. 178 allow_unstructured: This must be set to True to allow passing 179 unstructured text data as input. Note: the use of this flag is 180 discouraged and only exists to support legacy clients; new tracing 181 agents should all produce structured trace data (e.g. proto or json). 182 """ 183 if isinstance(data, StringTypes): 184 if not allow_unstructured: 185 raise ValueError('must pass allow_unstructured=True for text data') 186 do_write = lambda d, f: f.write(d) 187 suffix = '.txt' # Used for atrace and systrace data. 188 elif isinstance(data, dict): 189 do_write = json.dump 190 suffix = '.json' 191 else: 192 raise TypeError('invalid trace data type') 193 with self.OpenTraceHandleFor(part, suffix) as handle: 194 do_write(data, handle) 195 196 def Freeze(self): 197 """Do not allow writing any more data into this builder.""" 198 self._frozen = True 199 return self 200 201 def CleanUpTraceData(self): 202 """Clean up resources used by the data builder. 203 204 Will also re-raise any exceptions previously added by 205 RecordTraceCollectionException(). 206 """ 207 if self._traces is None: 208 return # Already cleaned up. 209 self.Freeze() 210 for trace in self._traces: 211 # Make sure all trace handles are closed. It's fine if we close some 212 # of them multiple times. 213 trace.handle.close() 214 shutil.rmtree(self._temp_dir) 215 self._temp_dir = None 216 self._traces = None 217 218 if self._exceptions: 219 raise TraceDataException( 220 'Exceptions raised during trace data collection:\n' + 221 '\n'.join(self._exceptions)) 222 223 def Serialize(self, file_path, trace_title=None): 224 """Serialize the trace data to a file in HTML format.""" 225 self.Freeze() 226 assert self._traces, 'trace data has already been cleaned up' 227 228 trace_files = [trace.handle.name for trace in self._traces] 229 SerializeAsHtml(trace_files, file_path, trace_title) 230 231 def AsData(self): 232 """Allow in-memory access to read the collected JSON trace data. 233 234 This method is only provided for writing tests which require read access 235 to the collected trace data (e.g. for tracing agents to test they correctly 236 write data), and to support legacy TBMv1 metric computation. Only traces 237 in JSON format are supported. 238 239 Be careful: this may require a lot of memory if the traces to process are 240 very large. This has lead in the past to OOM errors (e.g. crbug/672097). 241 242 TODO(crbug/928278): Ideally, this method should be removed when it can be 243 entirely replaced by calls to an external trace processor. 244 """ 245 self.Freeze() 246 assert self._traces, 'trace data has already been cleaned up' 247 248 raw_data = {} 249 for trace in self._traces: 250 is_compressed_json = trace.handle.name.endswith('.json.gz') 251 is_json = trace.handle.name.endswith('.json') or is_compressed_json 252 if is_json: 253 traces_for_part = raw_data.setdefault(trace.part_name, []) 254 opener = gzip.open if is_compressed_json else open 255 with opener(trace.handle.name, 'rb') as f: 256 traces_for_part.append(json.load(f)) 257 else: 258 logging.info('Skipping over non-json trace: %s', trace.handle.name) 259 return _TraceData(raw_data) 260 261 def IterTraceParts(self): 262 """Iterates over trace parts. 263 264 Return value: iterator over pairs (part_name, file_path). 265 """ 266 for trace in self._traces: 267 yield trace.part_name, trace.handle.name 268 269 def RecordTraceDataException(self): 270 """Records the most recent exception to be re-raised during cleanup. 271 272 Exceptions raised during trace data collection can be stored temporarily 273 in the builder. They will be re-raised when the builder is cleaned up later. 274 This way, any collected trace data can still be retained before the 275 benchmark is aborted. 276 277 This method is intended to be called from within an "except" handler, e.g.: 278 try: 279 # Collect trace data. 280 except Exception: # pylint: disable=broad-except 281 builder.RecordTraceDataException() 282 """ 283 self._exceptions.append(traceback.format_exc()) 284 285 286def CreateTestTrace(number=1): 287 """Convenient helper method to create trace data objects for testing. 288 289 Objects are created via the usual trace data writing route, so clients are 290 also responsible for cleaning up trace data themselves. 291 292 Clients are meant to treat these test traces as opaque. No guarantees are 293 made about their contents, which they shouldn't try to read. 294 """ 295 builder = TraceDataBuilder() 296 builder.AddTraceFor(CHROME_TRACE_PART, {'traceEvents': [{'test': number}]}) 297 return builder.Freeze() 298 299 300def CreateFromRawChromeEvents(events): 301 """Convenient helper to create trace data objects from raw Chrome events. 302 303 This bypasses trace data writing, going directly to the in-memory json trace 304 representation, so there is no need for trace file cleanup. 305 306 This is used only for testing legacy clients that still read trace data. 307 """ 308 assert isinstance(events, list) 309 return _TraceData({ 310 CHROME_TRACE_PART.raw_field_name: [{'traceEvents': events}]}) 311 312 313def SerializeAsHtml(trace_files, html_file, trace_title=None): 314 """Serialize a set of traces to a single file in HTML format. 315 316 Args: 317 trace_files: a list of file names, each containing a trace from 318 one of the tracing agents. 319 html_file: a name of the output file. 320 trace_title: optional. A title for the resulting trace. 321 """ 322 if not trace_files: 323 raise ValueError('trace files list is empty') 324 325 input_size = sum(os.path.getsize(trace_file) for trace_file in trace_files) 326 327 cmd = [] 328 if platform.system() == 'Windows': 329 version_cmd = ['python', '-c', 330 'import sys\nprint(sys.version_info.major)'] 331 version = subprocess.check_output(version_cmd) 332 if version.strip() == '3': 333 raise RuntimeError('trace2html cannot run with python 3.') 334 cmd.append('python') 335 cmd.append(_TRACE2HTML_PATH) 336 cmd.extend(trace_files) 337 cmd.extend(['--output', html_file]) 338 if trace_title is not None: 339 cmd.extend(['--title', trace_title]) 340 341 start_time = time.time() 342 subprocess.check_output(cmd) 343 elapsed_time = time.time() - start_time 344 logging.info('trace2html processed %.01f MiB of trace data in %.02f seconds.', 345 1.0 * input_size / MIB, elapsed_time) 346