1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.postprocessor; 17 18 import com.android.os.StatsLog.ConfigMetricsReportList; 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.OptionClass; 21 import com.android.tradefed.log.LogUtil.CLog; 22 import com.android.tradefed.metrics.proto.MetricMeasurement.DataType; 23 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 24 import com.android.tradefed.result.ByteArrayInputStreamSource; 25 import com.android.tradefed.result.LogDataType; 26 import com.android.tradefed.result.LogFile; 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.util.proto.TfMetricProtoUtil; 29 30 import com.google.common.annotations.VisibleForTesting; 31 import com.google.common.collect.ImmutableList; 32 import com.google.protobuf.Descriptors.FieldDescriptor; 33 import com.google.protobuf.Message; 34 import com.google.protobuf.TextFormat; 35 import com.google.protobuf.util.JsonFormat; 36 37 import java.io.File; 38 import java.io.FileInputStream; 39 import java.io.FileNotFoundException; 40 import java.io.IOException; 41 import java.util.HashMap; 42 import java.util.HashSet; 43 import java.util.List; 44 import java.util.Map; 45 import java.util.Map.Entry; 46 import java.util.Optional; 47 import java.util.Set; 48 import java.util.stream.Collectors; 49 50 import com.android.tradefed.util.ExtensionAtomsRegistry; 51 52 /** 53 * A post processor that processes binary proto statsd reports into key-value pairs by expanding the 54 * report as a tree structure. 55 * 56 * <p>This processor is agnostic to the type of metric reports it encounters. It also serves as the 57 * base class for other statsd post processors by including common code to retrieve and read statsd 58 * reports. 59 */ 60 @OptionClass(alias = "statsd-generic-processor") 61 public class StatsdGenericPostProcessor extends BasePostProcessor { 62 // TODO(harrytczhang): Add ability to dump the results out to a JSON file once saving files from 63 // post processors is possible. 64 65 @Option( 66 name = "statsd-report-data-prefix", 67 description = 68 "Prefix for identifying a data name that points to a statsd report. " 69 + "Can be repeated. " 70 + "Will also be prepended to the outcoming metric to \"namespace\" them." 71 ) 72 private Set<String> mReportPrefixes = new HashSet<>(); 73 74 @Option( 75 name = "output-statsd-report-proto", 76 description = 77 "Output the human-readable proto of the statsd metric reports in the specified " 78 + "format. Can be repeated. Empty (off) by default." 79 + "Only TEXTPB and JSON are supported at the moment.") 80 private Set<LogDataType> mOutputStatsdReportProtoFormats = new HashSet<>(); 81 82 @VisibleForTesting static final String METRIC_SEP = "-"; 83 @VisibleForTesting static final String INDEX_SEP = "#"; 84 85 // A few fields to skip as they are large and not currently used. 86 private static final String UID_MAP_FIELD_NAME = "uid_map"; 87 private static final String STRINGS_FIELD_NAME = "strings"; 88 private static final ImmutableList<String> FIELDS_TO_SKIP = 89 ImmutableList.of(UID_MAP_FIELD_NAME, STRINGS_FIELD_NAME); 90 91 private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer(); 92 93 @Override processTestMetricsAndLogs( TestDescription testDescription, HashMap<String, Metric> testMetrics, Map<String, LogFile> testLogs)94 public Map<String, Metric.Builder> processTestMetricsAndLogs( 95 TestDescription testDescription, 96 HashMap<String, Metric> testMetrics, 97 Map<String, LogFile> testLogs) { 98 return processStatsdReportsFromLogs(testLogs, testDescription.toString()); 99 } 100 101 @Override processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)102 public Map<String, Metric.Builder> processRunMetricsAndLogs( 103 HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) { 104 return processStatsdReportsFromLogs(runLogs, "TestRun"); 105 } 106 107 /** 108 * Parse metrics from a {@link ConfigMetricsReportList} read from a statsd report proto. 109 * 110 * <p>This is the main interface for subclasses of this statsd post processor. 111 */ parseMetricsFromReportList( ConfigMetricsReportList reportList)112 protected Map<String, Metric.Builder> parseMetricsFromReportList( 113 ConfigMetricsReportList reportList) { 114 return convertProtoMessage(reportList); 115 } 116 117 /** 118 * Uses the metrics to locate the statsd report files and then call into the processing method 119 * to parse the reports into metrics. 120 */ processStatsdReportsFromLogs( Map<String, LogFile> logs, String testOrRunName)121 private Map<String, Metric.Builder> processStatsdReportsFromLogs( 122 Map<String, LogFile> logs, String testOrRunName) { 123 Map<String, Metric.Builder> parsedMetrics = new HashMap<>(); 124 125 for (String key : logs.keySet()) { 126 // Go through the logs and match them to the list of prefixes. Only proceed when a match 127 // is found. 128 Optional<String> reportPrefix = 129 mReportPrefixes.stream().filter(prefix -> key.startsWith(prefix)).findAny(); 130 if (!reportPrefix.isPresent()) { 131 continue; 132 } 133 134 File reportFile = new File(logs.get(key).getPath()); 135 try (FileInputStream reportStream = new FileInputStream(reportFile)) { 136 ConfigMetricsReportList reportList = 137 ConfigMetricsReportList.parseFrom( 138 reportStream, ExtensionAtomsRegistry.registry); 139 if (reportList.getReportsList().isEmpty()) { 140 CLog.i("No reports collected for %s.", key); 141 continue; 142 } 143 for (LogDataType format : mOutputStatsdReportProtoFormats) { 144 String dataName = 145 String.format("%s_%s_report", reportPrefix.get(), testOrRunName); 146 switch (format) { 147 case TEXTPB: 148 testLog( 149 dataName, 150 format, 151 new ByteArrayInputStreamSource( 152 TextFormat.printToString(reportList).getBytes())); 153 break; 154 case JSON: 155 testLog( 156 dataName, 157 format, 158 new ByteArrayInputStreamSource( 159 JSON_PRINTER.print(reportList).getBytes())); 160 break; 161 default: 162 CLog.e( 163 "Cannot output statsd report proto with unsupported format %s.", 164 format); 165 } 166 } 167 Map<String, Metric.Builder> metricsForReport = 168 parseMetricsFromReportList(reportList); 169 // Prepend the report prefix to serve as top-level organization of the outputted 170 // metrics. Please see comment on the prepend method for details. 171 parsedMetrics.putAll(addPrefixToMetrics(metricsForReport, reportPrefix.get())); 172 } catch (FileNotFoundException e) { 173 CLog.e("Report file does not exist at supposed path %s.", logs.get(key).getPath()); 174 } catch (IOException e) { 175 CLog.i("Failed to read report file due to error %s.", e.toString()); 176 } 177 } 178 179 return parsedMetrics; 180 } 181 182 /** 183 * Add a prefix to the metrics out of a single report. 184 * 185 * <p>This is used to add top-level organization for the metric output as due to the nesting 186 * nature of the metrics the actual metric key can be hard to read at a glance without the 187 * prefixes added here. As an example, if we use "statsd-app-start" as a prefix, then some 188 * of the metrics would read: 189 * 190 * <pre> 191 * statsd-app-start-reports-metrics-event_metrics-data-atom-app_start_occurred-type: WARM 192 * statsd-app-start-reports-metrics-event_metrics-data-atom-app_start_occurred-uid: 10149 193 * statsd-app-start-reports-metrics-event_metrics-data-elapsed_timestamp_nanos: 1449806339931935 194 */ addPrefixToMetrics( Map<String, Metric.Builder> metrics, String prefix)195 private Map<String, Metric.Builder> addPrefixToMetrics( 196 Map<String, Metric.Builder> metrics, String prefix) { 197 return metrics.entrySet() 198 .stream() 199 .collect( 200 Collectors.toMap( 201 e -> String.join(METRIC_SEP, prefix, e.getKey()), 202 e -> e.getValue())); 203 } 204 205 /** 206 * Flatten a proto message to a set of key-value pairs which become metrics. 207 * 208 * <p>It treats a message as a tree and uses the concatenated path from the root to a 209 * non-message value as the key, while the non-message value becomes the metric value. Nodes 210 * from repeated fields are distinguished by having a 1-based index number appended to all 211 * elements after the first element. The first element is not appended as in most cases only one 212 * element is in the list field and having it appear as-is is easier to read. 213 * 214 * <p>TODO(b/140432161): Separate this out into a utility should the need arise. 215 */ convertProtoMessage(Message reportMessage)216 protected Map<String, Metric.Builder> convertProtoMessage(Message reportMessage) { 217 Map<FieldDescriptor, Object> fields = reportMessage.getAllFields(); 218 Map<String, Metric.Builder> convertedMetrics = new HashMap<String, Metric.Builder>(); 219 for (Entry<FieldDescriptor, Object> entry : fields.entrySet()) { 220 if (FIELDS_TO_SKIP.contains(entry.getKey().getName())) { 221 continue; 222 } 223 if (entry.getValue() instanceof Message) { 224 Map<String, Metric.Builder> messageMetrics = 225 convertProtoMessage((Message) entry.getValue()); 226 for (Entry<String, Metric.Builder> metricEntry : messageMetrics.entrySet()) { 227 convertedMetrics.put( 228 String.join(METRIC_SEP, entry.getKey().getName(), metricEntry.getKey()), 229 metricEntry.getValue()); 230 } 231 } else if (entry.getValue() instanceof List) { 232 List<? extends Object> listMetrics = (List) entry.getValue(); 233 for (int i = 0; i < listMetrics.size(); i++) { 234 String metricKeyRoot = 235 (i == 0 236 ? entry.getKey().getName() 237 : String.join( 238 INDEX_SEP, 239 entry.getKey().getName(), 240 String.valueOf(i + 1))); 241 if (listMetrics.get(i) instanceof Message) { 242 Map<String, Metric.Builder> messageMetrics = 243 convertProtoMessage((Message) listMetrics.get(i)); 244 for (Entry<String, Metric.Builder> metricEntry : 245 messageMetrics.entrySet()) { 246 convertedMetrics.put( 247 String.join(METRIC_SEP, metricKeyRoot, metricEntry.getKey()), 248 metricEntry.getValue()); 249 } 250 } else { 251 convertedMetrics.put( 252 metricKeyRoot, 253 TfMetricProtoUtil.stringToMetric(listMetrics.get(i).toString()) 254 .toBuilder()); 255 } 256 } 257 } else { 258 convertedMetrics.put( 259 entry.getKey().getName(), 260 TfMetricProtoUtil.stringToMetric(entry.getValue().toString()).toBuilder()); 261 } 262 } 263 return convertedMetrics; 264 } 265 266 /** 267 * Set the metric type to RAW metric. 268 */ 269 @Override getMetricType()270 protected DataType getMetricType() { 271 return DataType.RAW; 272 } 273 } 274