1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.postprocessor; 18 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.OptionClass; 21 import com.android.tradefed.log.LogUtil.CLog; 22 import com.android.tradefed.metrics.proto.MetricMeasurement.DataType; 23 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 24 import com.android.tradefed.result.LogFile; 25 import com.android.tradefed.result.TestDescription; 26 import com.android.tradefed.util.FileUtil; 27 import com.android.tradefed.util.ZipUtil; 28 import com.android.tradefed.util.ZipUtil2; 29 import com.android.tradefed.util.proto.TfMetricProtoUtil; 30 31 import com.google.common.annotations.VisibleForTesting; 32 import com.google.gson.Gson; 33 import com.google.gson.JsonElement; 34 import com.google.gson.JsonObject; 35 import com.google.gson.JsonSyntaxException; 36 import com.google.protobuf.Descriptors.FieldDescriptor; 37 import com.google.protobuf.Message; 38 import com.google.protobuf.TextFormat; 39 import com.google.protobuf.TextFormat.ParseException; 40 41 import org.apache.commons.compress.archivers.zip.ZipFile; 42 43 import java.io.BufferedReader; 44 import java.io.File; 45 import java.io.FileInputStream; 46 import java.io.FileReader; 47 import java.io.IOException; 48 import java.util.ArrayList; 49 import java.util.HashMap; 50 import java.util.HashSet; 51 import java.util.LinkedHashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Map.Entry; 55 import java.util.Optional; 56 import java.util.Set; 57 import java.util.concurrent.TimeUnit; 58 import java.util.regex.Pattern; 59 import java.util.stream.Collectors; 60 61 import perfetto.protos.PerfettoMergedMetrics.TraceMetrics; 62 import perfetto.protos.File.TraceSummary; 63 import perfetto.protos.V2Metric.TraceMetricV2; 64 import perfetto.protos.V2Metric.TraceMetricV2.MetricRow; 65 import perfetto.protos.V2Metric.TraceMetricV2.MetricRow.Dimension; 66 67 /** 68 * A post processor that processes text/binary metric perfetto proto file into key-value pairs by 69 * recursively expanding the proto messages and fields with string values until the field with 70 * numeric value is encountered. Treats enum and boolean as string values while constructing the 71 * keys. 72 * 73 * <p>It optionally supports indexing list fields when there are duplicates while constructing the 74 * keys. For example 75 * 76 * <p>"perfetto-indexed-list-field" - perfetto.protos.AndroidStartupMetric.Startup 77 * 78 * <p>"perfetto-prefix-key-field" - perfetto.protos.ProcessRenderInfo.process_name 79 * 80 * <p>android_startup-startup#1-package_name-com.calculator-to_first_frame-dur_ns: 300620342 81 * android_startup-startup#2-package_name-com.nexuslauncher-to_first_frame-dur_ns: 49257713 82 * android_startup-startup#3-package_name-com.calculator-to_first_frame-dur_ns: 261382005 83 */ 84 @OptionClass(alias = "perfetto-generic-processor") 85 public class PerfettoGenericPostProcessor extends BasePostProcessor { 86 87 private static final TextFormat.Parser ALLOW_UNKNOWN_FIELD = 88 TextFormat.Parser.newBuilder().setAllowUnknownFields(true).build(); 89 private static final String METRIC_SEP = "-"; 90 @VisibleForTesting static final String RUNTIME_METRIC_KEY = "perfetto_post_processor_runtime"; 91 92 public enum METRIC_FILE_FORMAT { 93 text, 94 binary, 95 json, 96 } 97 98 public enum AlternativeParseFormat { 99 json, 100 none, 101 } 102 103 @Option( 104 name = "perfetto-proto-file-prefix", 105 description = "Prefix for identifying a perfetto metric file name.") 106 private Set<String> mPerfettoProtoMetricFilePrefix = new HashSet<>(); 107 108 @Option( 109 name = "perfetto-indexed-list-field", 110 description = "List fields in perfetto proto metric file that has to be indexed.") 111 private Set<String> mPerfettoIndexedListFields = new HashSet<>(); 112 113 @Option( 114 name = "perfetto-prefix-key-field", 115 description = 116 "String value field need to be prefixed with the all the other" 117 + "numeric value field keys in the proto message.") 118 private Set<String> mPerfettoPrefixKeyFields = new HashSet<>(); 119 120 @Option( 121 name = "perfetto-prefix-inner-message-key-field", 122 description = 123 "String value field need to be prefixed with the all the other" 124 + "numeric value field keys outside of the current proto message.") 125 private Set<String> mPerfettoPrefixInnerMessagePrefixFields = new HashSet<>(); 126 127 @Option( 128 name = "perfetto-include-all-metrics", 129 description = 130 "If this flag is turned on, all the metrics parsed from the perfetto file will" 131 + " be included in the final result map and ignores the regex passed" 132 + " in the filters.") 133 private boolean mPerfettoIncludeAllMetrics = false; 134 135 @Option( 136 name = "perfetto-metric-filter-regex", 137 description = 138 "Regular expression that will be used for filtering the metrics parsed" 139 + " from the perfetto proto metric file.") 140 private Set<String> mPerfettoMetricFilterRegEx = new HashSet<>(); 141 142 @Option( 143 name = "trace-processor-output-format", 144 description = "Trace processor output format. One of [binary|text|json]") 145 private METRIC_FILE_FORMAT mTraceProcessorOutputFormat = METRIC_FILE_FORMAT.text; 146 147 @Option( 148 name = "decompress-perfetto-timeout", 149 description = "Timeout to decompress perfetto compressed file.", 150 isTimeVal = true) 151 private long mDecompressTimeoutMs = TimeUnit.MINUTES.toMillis(20); 152 153 @Deprecated 154 @Option( 155 name = "processed-metric", 156 description = 157 "True if the metric is final and shouldn't be processed any more," 158 + " false if the metric can be handled by another post-processor.") 159 private boolean mProcessedMetric = true; 160 161 @Option( 162 name = "perfetto-metric-replace-prefix", 163 description = 164 "Replace the prefix in metricsfrom the metric proto file. Key is the prefix to" 165 + " look for in the metrickeys parsed and value is be the replacement" 166 + " string.") 167 private Map<String, String> mReplacePrefixMap = new LinkedHashMap<String, String>(); 168 169 @Option( 170 name = "perfetto-all-metric-prefix", 171 description = 172 "Prefix to be used with the metrics collected from perfetto." 173 + "This will be applied before any other prefixes to metrics.") 174 private String mAllMetricPrefix = "perfetto"; 175 176 @Option( 177 name = "perfetto-alternative-parse-format", 178 description = 179 "Parse the metrics as key/value pair or JSON when corresponding proto " 180 + "definition is not found. One of [json|none]") 181 private AlternativeParseFormat mAlternativeParseFormat = AlternativeParseFormat.none; 182 183 // Matches 1.73, 1.73E+2 184 private Pattern mNumberWithExponentPattern = 185 Pattern.compile("[-+]?[0-9]*[\\.]?[0-9]+([eE][-+]?[0-9]+)?"); 186 187 // Matches numbers without exponent format. 188 private Pattern mNumberPattern = Pattern.compile("[-+]?[0-9]*[\\.]?[0-9]+"); 189 190 private List<Pattern> mMetricPatterns = new ArrayList<>(); 191 192 private String mPrefixFromInnerMessage = ""; 193 194 @Override processTestMetricsAndLogs( TestDescription testDescription, HashMap<String, Metric> testMetrics, Map<String, LogFile> testLogs)195 public Map<String, Metric.Builder> processTestMetricsAndLogs( 196 TestDescription testDescription, 197 HashMap<String, Metric> testMetrics, 198 Map<String, LogFile> testLogs) { 199 buildMetricFilterPatterns(); 200 return processPerfettoMetrics(filterPerfeticMetricFiles(testLogs)); 201 } 202 203 @Override processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)204 public Map<String, Metric.Builder> processRunMetricsAndLogs( 205 HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) { 206 buildMetricFilterPatterns(); 207 return processPerfettoMetrics(filterPerfeticMetricFiles(runLogs)); 208 } 209 210 /** 211 * Filter the perfetto metric file based on the prefix. 212 * 213 * @param logs 214 * @return files matched the prefix. 215 */ filterPerfeticMetricFiles(Map<String, LogFile> logs)216 private List<File> filterPerfeticMetricFiles(Map<String, LogFile> logs) { 217 List<File> perfettoMetricFiles = new ArrayList<>(); 218 for (String key : logs.keySet()) { 219 Optional<String> reportPrefix = 220 mPerfettoProtoMetricFilePrefix.stream() 221 .filter(prefix -> key.startsWith(prefix)) 222 .findAny(); 223 224 if (!reportPrefix.isPresent()) { 225 continue; 226 } 227 CLog.i("Adding perfetto metric file: " + logs.get(key).getPath()); 228 perfettoMetricFiles.add(new File(logs.get(key).getPath())); 229 } 230 return perfettoMetricFiles; 231 } 232 233 /** 234 * Process perfetto metric files into key, value pairs. 235 * 236 * @param perfettoMetricFiles perfetto metric files to be processed. 237 * @return key, value pairs processed from the metrics. 238 */ processPerfettoMetrics(List<File> perfettoMetricFiles)239 private Map<String, Metric.Builder> processPerfettoMetrics(List<File> perfettoMetricFiles) { 240 Map<String, Metric.Builder> parsedMetrics = new HashMap<>(); 241 long startTime = System.currentTimeMillis(); 242 File uncompressedDir = null; 243 for (File perfettoMetricFile : perfettoMetricFiles) { 244 // Text files by default are compressed before uploading. Decompress the text proto 245 // file before post processing. 246 try { 247 if (!(mTraceProcessorOutputFormat == METRIC_FILE_FORMAT.binary) 248 && ZipUtil.isZipFileValid(perfettoMetricFile, true)) { 249 ZipFile perfettoZippedFile = new ZipFile(perfettoMetricFile); 250 uncompressedDir = FileUtil.createTempDir("uncompressed_perfetto_metric"); 251 ZipUtil2.extractZip(perfettoZippedFile, uncompressedDir); 252 perfettoMetricFile = uncompressedDir.listFiles()[0]; 253 perfettoZippedFile.close(); 254 } 255 } catch (IOException e) { 256 CLog.e( 257 "IOException happened when unzipping the perfetto metric proto" 258 + " file." 259 + e.getMessage()); 260 } 261 262 // Parse the perfetto proto file. 263 try (BufferedReader bufferedReader = 264 new BufferedReader(new FileReader(perfettoMetricFile))) { 265 switch (mTraceProcessorOutputFormat) { 266 case text: 267 if (perfettoMetricFile.getName().contains("v2")) { 268 TraceSummary.Builder builderV2 = TraceSummary.newBuilder(); 269 ALLOW_UNKNOWN_FIELD.merge(bufferedReader, builderV2); 270 parsedMetrics.putAll(convertPerfettoProtoMessageV2(builderV2.build())); 271 } else { 272 TraceMetrics.Builder builder = TraceMetrics.newBuilder(); 273 ALLOW_UNKNOWN_FIELD.merge(bufferedReader, builder); 274 parsedMetrics.putAll( 275 handlePrefixForProcessedMetrics( 276 convertPerfettoProtoMessage(builder.build()))); 277 } 278 break; 279 case binary: 280 if (perfettoMetricFile.getName().contains("v2")) { 281 TraceSummary traceSummary = 282 TraceSummary.parseFrom(new FileInputStream(perfettoMetricFile)); 283 parsedMetrics.putAll(convertPerfettoProtoMessageV2(traceSummary)); 284 } else { 285 TraceMetrics metricProto = null; 286 metricProto = 287 TraceMetrics.parseFrom(new FileInputStream(perfettoMetricFile)); 288 parsedMetrics.putAll( 289 handlePrefixForProcessedMetrics( 290 convertPerfettoProtoMessage(metricProto))); 291 } 292 break; 293 case json: 294 CLog.w("JSON perfetto metric file processing not supported."); 295 } 296 } catch (ParseException e) { 297 if (AlternativeParseFormat.none == mAlternativeParseFormat) { 298 CLog.e("Failed to merge the perfetto metric file. " + e.getMessage()); 299 } else { 300 CLog.w("Failed to merge the perfetto metric file, trying alternative"); 301 parsedMetrics.putAll( 302 handlePrefixForProcessedMetrics( 303 processPerfettoMetricsWithAlternativeMethods( 304 perfettoMetricFile))); 305 } 306 } catch (IOException ioe) { 307 CLog.e( 308 "IOException happened when reading the perfetto metric file. " 309 + ioe.getMessage()); 310 } finally { 311 // Delete the uncompressed perfetto metric proto file directory. 312 FileUtil.recursiveDelete(uncompressedDir); 313 } 314 } 315 316 if (parsedMetrics.size() > 0) { 317 parsedMetrics.put( 318 RUNTIME_METRIC_KEY, 319 TfMetricProtoUtil.stringToMetric( 320 Long.toString(System.currentTimeMillis() - startTime)) 321 .toBuilder()); 322 } 323 324 return parsedMetrics; 325 } 326 327 /** 328 * Process perfetto metric files that does not have proto defined in TraceMetrics into key, 329 * value pairs. 330 * 331 * @param perfettoMetricFile perfetto metric file to be processed. 332 * @return key, value pairs processed from the metrics. 333 */ processPerfettoMetricsWithAlternativeMethods( File perfettoMetricFile)334 private Map<String, Metric.Builder> processPerfettoMetricsWithAlternativeMethods( 335 File perfettoMetricFile) { 336 CLog.w("Entering processPerfettoMetricsWithAlternativeMethods"); 337 Map<String, Metric.Builder> result = new HashMap<>(); 338 try (BufferedReader bufferedReader = 339 new BufferedReader(new FileReader(perfettoMetricFile))) { 340 if (AlternativeParseFormat.json == mAlternativeParseFormat) { 341 JsonObject node = new Gson().fromJson(bufferedReader, JsonObject.class); 342 node.entrySet().forEach(nested -> flattenJson(result, nested, new ArrayList<>())); 343 return result; 344 } 345 } catch (JsonSyntaxException jse) { 346 CLog.e( 347 "JsonSyntaxException happened when parsing perfetto metric file. " 348 + jse.getMessage()); 349 } catch (IOException ioe) { 350 CLog.e( 351 "IOException happened when reading the perfetto metric file. " 352 + ioe.getMessage()); 353 } 354 355 return result; 356 } 357 358 /** 359 * Flatten a json into key, value pairs where key is the concatenation of keys in each level of 360 * json, and the value is the value of the leaf node. 361 * 362 * @param result the map to store the result 363 * @param node the node to process from 364 * @names the list of the names that has been added so far above the current node. 365 * @return key, value pairs of the flattened json. 366 */ flattenJson( Map<String, Metric.Builder> result, Entry<String, JsonElement> node, List<String> names)367 private Map<String, Metric.Builder> flattenJson( 368 Map<String, Metric.Builder> result, 369 Entry<String, JsonElement> node, 370 List<String> names) { 371 names.add(node.getKey()); 372 if (node.getValue().isJsonObject()) { 373 node.getValue() 374 .getAsJsonObject() 375 .entrySet() 376 .forEach(nested -> flattenJson(result, nested, new ArrayList<>(names))); 377 } else { 378 String name = names.stream().collect(Collectors.joining(METRIC_SEP)); 379 result.put( 380 name, 381 TfMetricProtoUtil.stringToMetric(node.getValue().getAsString()).toBuilder()); 382 } 383 384 return result; 385 } 386 handlePrefixForProcessedMetrics( Map<String, Metric.Builder> processedMetrics)387 private Map<String, Metric.Builder> handlePrefixForProcessedMetrics( 388 Map<String, Metric.Builder> processedMetrics) { 389 Map<String, Metric.Builder> result = new HashMap<>(); 390 result.putAll(filterMetrics(processedMetrics)); 391 replacePrefix(result); 392 // Generic prefix string is applied to all the metrics parsed from perfetto trace file. 393 replaceAllMetricPrefix(result); 394 return result; 395 } 396 397 /** 398 * Replace the prefix in the metric key parsed from the proto file with the given string. 399 * 400 * @param processPerfettoMetrics metrics parsed from the perfetto proto file. 401 */ replacePrefix(Map<String, Metric.Builder> processPerfettoMetrics)402 private void replacePrefix(Map<String, Metric.Builder> processPerfettoMetrics) { 403 if (mReplacePrefixMap.isEmpty()) { 404 return; 405 } 406 Map<String, Metric.Builder> finalMetrics = new HashMap<String, Metric.Builder>(); 407 for (Map.Entry<String, Metric.Builder> metric : processPerfettoMetrics.entrySet()) { 408 boolean isReplaced = false; 409 for (Map.Entry<String, String> replaceEntry : mReplacePrefixMap.entrySet()) { 410 if (metric.getKey().startsWith(replaceEntry.getKey())) { 411 String newKey = 412 metric.getKey() 413 .replaceFirst(replaceEntry.getKey(), replaceEntry.getValue()); 414 finalMetrics.put(newKey, metric.getValue()); 415 isReplaced = true; 416 break; 417 } 418 } 419 // If key is not replaced put the original key and value in the final metrics. 420 if (!isReplaced) { 421 finalMetrics.put(metric.getKey(), metric.getValue()); 422 } 423 } 424 processPerfettoMetrics.clear(); 425 processPerfettoMetrics.putAll(finalMetrics); 426 } 427 428 /** 429 * Prefix all the metrics key with given string. 430 * 431 * @param processPerfettoMetrics metrics parsed from the perfetto proto file. 432 */ replaceAllMetricPrefix(Map<String, Metric.Builder> processPerfettoMetrics)433 private void replaceAllMetricPrefix(Map<String, Metric.Builder> processPerfettoMetrics) { 434 if (mAllMetricPrefix == null || mAllMetricPrefix.isEmpty()) { 435 return; 436 } 437 Map<String, Metric.Builder> finalMetrics = new HashMap<String, Metric.Builder>(); 438 for (Map.Entry<String, Metric.Builder> metric : processPerfettoMetrics.entrySet()) { 439 String newKey = String.format("%s_%s", mAllMetricPrefix, metric.getKey()); 440 finalMetrics.put(newKey, metric.getValue()); 441 CLog.d("Perfetto trace metric: key: %s value: %s", newKey, metric.getValue()); 442 } 443 processPerfettoMetrics.clear(); 444 processPerfettoMetrics.putAll(finalMetrics); 445 } 446 447 /** 448 * Expands the metric proto file as tree structure and converts it into key, value pairs by 449 * recursively constructing the key using the message name, proto fields with string values 450 * until the numeric proto field is encountered. 451 * 452 * <p>android_startup-startup-package_name-com.calculator-to_first_frame-dur_ns: 300620342 453 * android_startup-startup-package_name-com.nexuslauncher-to_first_frame-dur_ns: 49257713 454 * 455 * <p>It also supports indexing the list proto fields optionally. This will be used if the list 456 * generates duplicate key's when recursively expanding the messages to prevent overriding the 457 * results. 458 * 459 * <p>"perfetto-indexed-list-field" - perfetto.protos.AndroidStartupMetric.Startup 460 * 461 * <p><android_startup-startup#1-package_name-com.calculator-to_first_frame-dur_ns: 300620342 462 * android_startup-startup#2-package_name-com.nexuslauncher-to_first_frame-dur_ns: 49257713 463 * android_startup-startup#3-package_name-com.calculator-to_first_frame-dur_ns: 261382005 464 * 465 * <p>"perfetto-prefix-key-field" - perfetto.protos.ProcessRenderInfo.process_name 466 * android_hwui_metric-process_info-process_name-system_server-cache_miss_avg 467 */ convertPerfettoProtoMessage(Message reportMessage)468 private Map<String, Metric.Builder> convertPerfettoProtoMessage(Message reportMessage) { 469 Map<FieldDescriptor, Object> fields = reportMessage.getAllFields(); 470 Map<String, Metric.Builder> convertedMetrics = new HashMap<String, Metric.Builder>(); 471 List<String> keyPrefixes = new ArrayList<String>(); 472 473 // Key that will be used to prefix the other keys in the same proto message. 474 String keyPrefixOtherFields = ""; 475 // If the flag is set then the prefix is set from the current message. Used 476 // to clear the prefix text after all the metrics are prefixed. 477 boolean prefixSetInCurrentMessage = false; 478 479 // TODO(b/15014555): Cleanup the parsing logic. 480 for (Entry<FieldDescriptor, Object> entry : fields.entrySet()) { 481 if (!(entry.getValue() instanceof Message) && !(entry.getValue() instanceof List)) { 482 if (isNumeric(entry.getValue().toString())) { 483 // Check if the current field has to be used as prefix for other fields 484 // and add it to the list of prefixes. 485 if (mPerfettoPrefixKeyFields.contains(entry.getKey().toString())) { 486 if (!keyPrefixOtherFields.isEmpty()) { 487 keyPrefixOtherFields = keyPrefixOtherFields.concat("-"); 488 } 489 keyPrefixOtherFields = 490 keyPrefixOtherFields.concat( 491 String.format( 492 "%s-%s", 493 entry.getKey().getName().toString(), 494 entry.getValue().toString())); 495 continue; 496 } 497 498 if (mPerfettoPrefixInnerMessagePrefixFields.contains( 499 entry.getKey().toString())) { 500 if (!mPrefixFromInnerMessage.isEmpty()) { 501 mPrefixFromInnerMessage = mPrefixFromInnerMessage.concat("-"); 502 } 503 mPrefixFromInnerMessage = 504 mPrefixFromInnerMessage.concat( 505 String.format( 506 "%s-%s", 507 entry.getKey().getName().toString(), 508 entry.getValue().toString())); 509 prefixSetInCurrentMessage = true; 510 continue; 511 } 512 513 // Otherwise treat this numeric field as metric. 514 if (mNumberPattern.matcher(entry.getValue().toString()).matches()) { 515 convertedMetrics.put( 516 entry.getKey().getName(), 517 TfMetricProtoUtil.stringToMetric(entry.getValue().toString()) 518 .toBuilder()); 519 } else { 520 // Parse the exponent notation of string before adding it to metric. 521 convertedMetrics.put( 522 entry.getKey().getName(), 523 TfMetricProtoUtil.stringToMetric( 524 Long.toString( 525 Double.valueOf(entry.getValue().toString()) 526 .longValue())) 527 .toBuilder()); 528 } 529 } else { 530 // Add to prefix list if string value is encountered. 531 keyPrefixes.add( 532 String.join( 533 METRIC_SEP, 534 entry.getKey().getName().toString(), 535 entry.getValue().toString())); 536 if (mPerfettoPrefixKeyFields.contains(entry.getKey().toString())) { 537 if (!keyPrefixOtherFields.isEmpty()) { 538 keyPrefixOtherFields = keyPrefixOtherFields.concat("-"); 539 } 540 keyPrefixOtherFields = 541 keyPrefixOtherFields.concat( 542 String.format( 543 "%s-%s", 544 entry.getKey().getName().toString(), 545 entry.getValue().toString())); 546 } 547 548 if (mPerfettoPrefixInnerMessagePrefixFields.contains( 549 entry.getKey().toString())) { 550 if (!mPrefixFromInnerMessage.isEmpty()) { 551 mPrefixFromInnerMessage = mPrefixFromInnerMessage.concat("-"); 552 } 553 mPrefixFromInnerMessage = 554 mPrefixFromInnerMessage.concat( 555 String.format( 556 "%s-%s", 557 entry.getKey().getName().toString(), 558 entry.getValue().toString())); 559 prefixSetInCurrentMessage = true; 560 continue; 561 } 562 } 563 } 564 } 565 566 // Recursively expand the proto messages and repeated fields(i.e list). 567 // Recursion when there are no messages or list with in the current message. 568 // Used to cache the message prefix. 569 String innerMessagePrefix = ""; 570 for (Entry<FieldDescriptor, Object> entry : fields.entrySet()) { 571 if (entry.getValue() instanceof Message) { 572 Map<String, Metric.Builder> messageMetrics = 573 convertPerfettoProtoMessage((Message) entry.getValue()); 574 if (!mPrefixFromInnerMessage.isEmpty()) { 575 innerMessagePrefix = mPrefixFromInnerMessage; 576 } 577 for (Entry<String, Metric.Builder> metricEntry : messageMetrics.entrySet()) { 578 // Add prefix to the metrics parsed from this message. 579 for (String prefix : keyPrefixes) { 580 convertedMetrics.put( 581 String.join( 582 METRIC_SEP, 583 prefix, 584 entry.getKey().getName(), 585 metricEntry.getKey()), 586 metricEntry.getValue()); 587 } 588 if (keyPrefixes.isEmpty()) { 589 convertedMetrics.put( 590 String.join( 591 METRIC_SEP, entry.getKey().getName(), metricEntry.getKey()), 592 metricEntry.getValue()); 593 } 594 } 595 } else if (entry.getValue() instanceof List) { 596 List<? extends Object> listMetrics = (List) entry.getValue(); 597 for (int i = 0; i < listMetrics.size(); i++) { 598 String metricKeyRoot; 599 // Use indexing if the current field is chosen for indexing. 600 // Use it if metrics keys generated has duplicates to prevent overriding. 601 if (mPerfettoIndexedListFields.contains(entry.getKey().toString())) { 602 metricKeyRoot = 603 String.join( 604 METRIC_SEP, 605 entry.getKey().getName(), 606 String.valueOf(i + 1)); 607 } else { 608 metricKeyRoot = String.join(METRIC_SEP, entry.getKey().getName()); 609 } 610 if (listMetrics.get(i) instanceof Message) { 611 Map<String, Metric.Builder> messageMetrics = 612 convertPerfettoProtoMessage((Message) listMetrics.get(i)); 613 if (!mPrefixFromInnerMessage.isEmpty()) { 614 innerMessagePrefix = mPrefixFromInnerMessage; 615 } 616 for (Entry<String, Metric.Builder> metricEntry : 617 messageMetrics.entrySet()) { 618 for (String prefix : keyPrefixes) { 619 convertedMetrics.put( 620 String.join( 621 METRIC_SEP, 622 prefix, 623 metricKeyRoot, 624 metricEntry.getKey()), 625 metricEntry.getValue()); 626 } 627 if (keyPrefixes.isEmpty()) { 628 convertedMetrics.put( 629 String.join( 630 METRIC_SEP, metricKeyRoot, metricEntry.getKey()), 631 metricEntry.getValue()); 632 } 633 } 634 } else { 635 convertedMetrics.put( 636 metricKeyRoot, 637 TfMetricProtoUtil.stringToMetric(listMetrics.get(i).toString()) 638 .toBuilder()); 639 } 640 } 641 } 642 } 643 644 // Add prefix key to all the keys in current proto message which has numeric values. 645 Map<String, Metric.Builder> additionalConvertedMetrics = 646 new HashMap<String, Metric.Builder>(); 647 if (!keyPrefixOtherFields.isEmpty()) { 648 for (Map.Entry<String, Metric.Builder> currentMetric : convertedMetrics.entrySet()) { 649 additionalConvertedMetrics.put( 650 String.format("%s-%s", keyPrefixOtherFields, currentMetric.getKey()), 651 currentMetric.getValue()); 652 } 653 } 654 655 if (!mPrefixFromInnerMessage.isEmpty() || !innerMessagePrefix.isEmpty()) { 656 String prefixToUse = 657 !mPrefixFromInnerMessage.isEmpty() 658 ? mPrefixFromInnerMessage 659 : innerMessagePrefix; 660 for (Map.Entry<String, Metric.Builder> currentMetric : convertedMetrics.entrySet()) { 661 additionalConvertedMetrics.put( 662 String.format("%s-%s", prefixToUse, currentMetric.getKey()), 663 currentMetric.getValue()); 664 } 665 } 666 667 if (!prefixSetInCurrentMessage) { 668 mPrefixFromInnerMessage = ""; 669 } 670 671 // Not cleaning up the other metrics without prefix fields. 672 convertedMetrics.putAll(additionalConvertedMetrics); 673 674 return convertedMetrics; 675 } 676 677 /** 678 * Expands the metric v2 proto file as tree structure and converts it into key, value pairs by 679 * recursively constructing the key using the id and dimensions string values, proto fields with 680 * string values until the numeric proto field is encountered. 681 * 682 * <p>memory_per_process-avg_rss_and_swap-.ShannonImsService: 121380864.000000 683 * memory_per_process-avg_rss_and_swap-/apex/com.android.adbd/bin/adbd: 10464441.000000 684 */ convertPerfettoProtoMessageV2(TraceSummary reportMessage)685 private Map<String, Metric.Builder> convertPerfettoProtoMessageV2(TraceSummary reportMessage) { 686 CLog.d("convertPerfettoProtoMessageV2 reportMessage : " + reportMessage); 687 Map<String, Metric.Builder> convertedMetrics = new HashMap<String, Metric.Builder>(); 688 for (TraceMetricV2 metric : reportMessage.getMetricList()) { 689 for (MetricRow row : metric.getRowList()) { 690 String rowKey = metric.getSpec().getId(); 691 double rowValue = row.getValue(); 692 for (Dimension dimension : row.getDimensionList()) { 693 Map<FieldDescriptor, Object> fields = dimension.getAllFields(); 694 for (Entry<FieldDescriptor, Object> entry : fields.entrySet()) { 695 if (!(entry.getValue() instanceof Message)) { 696 rowKey = rowKey.concat("-").concat(entry.getValue().toString()); 697 } 698 } 699 } 700 convertedMetrics.put( 701 rowKey, 702 TfMetricProtoUtil.stringToMetric(String.format("%f", rowValue)) 703 .toBuilder()); 704 CLog.d( 705 "Perfetto trace v2 metric: key: %s value: %s", 706 rowKey, Double.toString(rowValue)); 707 } 708 } 709 return convertedMetrics; 710 } 711 712 /** 713 * Check if the given string is number. It matches the string with exponent notation as well. 714 * 715 * <p>For example returns true for Return true for 1.73, 1.73E+2 716 */ isNumeric(String strNum)717 private boolean isNumeric(String strNum) { 718 if (strNum == null) { 719 return false; 720 } 721 return mNumberWithExponentPattern.matcher(strNum).matches(); 722 } 723 724 /** Build regular expression patterns to filter the metrics. */ buildMetricFilterPatterns()725 private void buildMetricFilterPatterns() { 726 if (!mPerfettoMetricFilterRegEx.isEmpty() && mMetricPatterns.isEmpty()) { 727 for (String regEx : mPerfettoMetricFilterRegEx) { 728 mMetricPatterns.add(Pattern.compile(regEx)); 729 } 730 } 731 } 732 733 /** 734 * Filter parsed metrics from the proto metric files based on the regular expression. If 735 * "mPerfettoIncludeAllMetrics" is enabled then filters will be ignored and returns all the 736 * parsed metrics. 737 */ filterMetrics(Map<String, Metric.Builder> parsedMetrics)738 private Map<String, Metric.Builder> filterMetrics(Map<String, Metric.Builder> parsedMetrics) { 739 if (mPerfettoIncludeAllMetrics) { 740 return parsedMetrics; 741 } 742 Map<String, Metric.Builder> filteredMetrics = new HashMap<>(); 743 for (Entry<String, Metric.Builder> metricEntry : parsedMetrics.entrySet()) { 744 for (Pattern pattern : mMetricPatterns) { 745 if (pattern.matcher(metricEntry.getKey()).matches()) { 746 filteredMetrics.put(metricEntry.getKey(), metricEntry.getValue()); 747 break; 748 } 749 } 750 } 751 return filteredMetrics; 752 } 753 754 /** Set the metric type to RAW metric. */ 755 @Override getMetricType()756 protected DataType getMetricType() { 757 return DataType.RAW; 758 } 759 } 760