1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.postprocessor; 17 18 import com.android.os.AtomsProto.Atom; 19 import com.android.os.StatsLog.EventMetricData; 20 import com.android.os.StatsLog.ConfigMetricsReport; 21 import com.android.os.StatsLog.ConfigMetricsReportList; 22 import com.android.os.StatsLog.StatsLogReport; 23 import com.android.tradefed.config.Option; 24 import com.android.tradefed.config.OptionClass; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 27 import com.android.tradefed.util.MultiMap; 28 import com.android.tradefed.util.ProtoUtil; 29 import com.android.tradefed.util.proto.TfMetricProtoUtil; 30 31 import com.google.protobuf.Descriptors.FieldDescriptor; 32 import com.google.protobuf.Message; 33 34 import java.util.ArrayList; 35 import java.util.Arrays; 36 import java.util.HashMap; 37 import java.util.List; 38 import java.util.Map; 39 import java.util.regex.Matcher; 40 import java.util.regex.Pattern; 41 import java.util.stream.Collectors; 42 43 /** 44 * A post processor that processes event metrics in statsd reports into key-value pairs, using the 45 * formatters specified on the processor. 46 */ 47 @OptionClass(alias = "statsd-event-metric-processor") 48 public class StatsdEventMetricPostProcessor extends StatsdGenericPostProcessor { 49 @Option( 50 name = "metric-formatter", 51 description = 52 "A formatter to format a statsd atom into a key-value pair for a metric." 53 + " Format: Use the atom field name as key and a 'key=value' string as" 54 + " value, and enclose atom field reference in square brackets, where they" 55 + " will be substituted with the field values in the atom. Example: key:" 56 + " app_start_occurred, value:" 57 + " [type]_startup_[pkg_name]=[windows_drawn_delay_millis]. Additionally," 58 + " use [_elapsed_timestamp_nanos] for the elapsed_timestamp_nanos field" 59 + " that records when the event occurred. At most one reference to" 60 + " repeated fields in each formatter is supported. Field definitions can" 61 + " be found in the atoms.proto file under frameworks/proto_logging/stats" 62 + " in the source tree.") 63 private MultiMap<String, String> mMetricFormatters = new MultiMap<>(); 64 65 // Corresponds to a field reference, e.g., "[field1_name.field2_name.field3_name]". 66 private static final Pattern FIELD_REF_PATTERN = 67 Pattern.compile("\\[(?:[a-zA-Z_]+\\.)*(?:[a-zA-Z_]+)\\]"); 68 69 /** 70 * Parse the event metrics from the {@link ConfigMetricsReportList} using the atom formatters. 71 * 72 * <p>Event metrics resulting in duplicate keys will be stored as comma separated values. 73 */ 74 @Override parseMetricsFromReportList( ConfigMetricsReportList reportList)75 protected Map<String, Metric.Builder> parseMetricsFromReportList( 76 ConfigMetricsReportList reportList) { 77 // A multimap is used to store metrics with multiple potential values. 78 MultiMap<String, String> parsedMetrics = new MultiMap<>(); 79 for (ConfigMetricsReport report : reportList.getReportsList()) { 80 for (StatsLogReport metric : report.getMetricsList()) { 81 if (!metric.hasEventMetrics()) { 82 continue; 83 } 84 // Look through each EventMetricData object's atom's every field, and extract 85 // metrics when the atom field name matches a set of configured metric formatters. 86 List<EventMetricData> dataItems = metric.getEventMetrics().getDataList(); 87 for (EventMetricData data : dataItems) { 88 Message atomParent = data; 89 Atom atom = null; 90 if (data.hasAtom()) { 91 atom = data.getAtom(); 92 } else if (data.hasAggregatedAtomInfo()) { 93 // In this case, the message housing the "elapsed_timestamp_nanos" field 94 // becomes the aggregated_atom_info field. 95 atomParent = data.getAggregatedAtomInfo(); 96 atom = data.getAggregatedAtomInfo().getAtom(); 97 } 98 if (atom == null) { 99 continue; 100 } 101 Map<FieldDescriptor, Object> atomFields = atom.getAllFields(); 102 for (FieldDescriptor field : atomFields.keySet()) { 103 if (mMetricFormatters.containsKey(field.getName())) { 104 parsedMetrics.putAll( 105 getMetricsByFormatters( 106 atomParent, 107 atom, 108 field, 109 mMetricFormatters.get(field.getName()))); 110 } 111 } 112 } 113 } 114 } 115 // Convert the multimap to a normal map with list of values turned into comma separated 116 // values. 117 Map<String, Metric.Builder> finalMetrics = new HashMap<>(); 118 for (String key : parsedMetrics.keySet()) { 119 // TODO(b/140434593): Move to repeated String fields and make sure that is supported in 120 // other post processors as well. 121 String value = String.join(",", parsedMetrics.get(key)); 122 finalMetrics.put(key, TfMetricProtoUtil.stringToMetric(value).toBuilder()); 123 } 124 return finalMetrics; 125 } 126 127 /** 128 * Helper method to get metrics from an {@link Atom} and its parent using the desired atom field 129 * and metric formatters. 130 */ getMetricsByFormatters( Message atomParent, Atom atom, FieldDescriptor atomField, List<String> formatters)131 private MultiMap<String, String> getMetricsByFormatters( 132 Message atomParent, Atom atom, FieldDescriptor atomField, List<String> formatters) { 133 MultiMap<String, String> metrics = new MultiMap<>(); 134 Message atomContent = (Message) atom.getField(atomField); 135 for (String formatter : formatters) { 136 String keyFormatter = formatter.split("=")[0]; 137 String valueFormatter = formatter.split("=")[1]; 138 List<String> metricKeys = fillInPlaceholders(keyFormatter, atomParent, atomContent); 139 List<String> metricValues = fillInPlaceholders(valueFormatter, atomParent, atomContent); 140 if (metricKeys.size() > 1 && metricValues.size() > 1) { 141 // If a repeated field is referenced in more than one location in the same 142 // formatter, it would be hard to determine whether there is a "pairing" relation 143 // between the two fields with the current field reference syntax. Specifically, one 144 // might want repeated_field.field1 and repeated_field.field2 to only appear paired 145 // within the same repeated_field instance, but the current logic produces a cross 146 // product between all repeated_field.field1 and repeated_field.field2 values, hence 147 // the warning below. 148 CLog.w( 149 "Found repeated fields in both metric key and value in formatting pair " 150 + "%s: %s. This is currently unsupported and could result in " 151 + "meaningless data. Skipping reporting on this pair.", 152 atomField.getName(), formatter); 153 continue; 154 } 155 for (String metricKey : metricKeys) { 156 for (String metricValue : metricValues) { 157 metrics.put(metricKey, metricValue); 158 } 159 } 160 } 161 return metrics; 162 } 163 164 /** Fill in the placeholders in the formatter using the proto message as source. */ fillInPlaceholders( String formatter, Message atomParent, Message atomContent)165 private List<String> fillInPlaceholders( 166 String formatter, Message atomParent, Message atomContent) { 167 Matcher matcher = FIELD_REF_PATTERN.matcher(formatter); 168 List<String> results = Arrays.asList(formatter); 169 while (matcher.find()) { 170 String placeholder = matcher.group(); 171 // Strip the brackets. 172 String fieldReference = placeholder.substring(1, placeholder.length() - 1); 173 List<String> actual = new ArrayList<>(); 174 if (fieldReference.startsWith("_")) { 175 actual.addAll( 176 ProtoUtil.getNestedFieldFromMessageAsStrings( 177 atomParent, 178 Arrays.asList(fieldReference.substring(1).split("\\.")))); 179 } else { 180 actual.addAll( 181 ProtoUtil.getNestedFieldFromMessageAsStrings( 182 atomContent, Arrays.asList(fieldReference.split("\\.")))); 183 } 184 // If both the existing expansion results and newly expanded results have multiple 185 // entries, then both the existing expansion and new expansion referred to repeated 186 // fields. 187 if (results.size() > 1 && actual.size() > 1) { 188 // If a repeated field is referenced in more than one location in the same 189 // formatter, it would be hard to determine whether there is a "pairing" relation 190 // between the two fields with the current field reference syntax. Specifically, one 191 // might want repeated_field.field1 and repeated_field.field2 to only appear paired 192 // within the same repeated_field instance, but the current logic produces a cross 193 // product between all repeated_field.field1 and repeated_field.field2 values, hence 194 // the warning below. 195 CLog.w( 196 "Found repeated fields in both metric key and value in formatter %s. This " 197 + "is currently unsupported and could result in meaningless data. " 198 + "Skipping reporting on this formatter.", 199 formatter); 200 return new ArrayList<>(); 201 } 202 List<String> updatedResults = 203 results.stream() 204 .flatMap(r -> actual.stream().map(a -> r.replace(placeholder, a))) 205 .collect(Collectors.toList()); 206 results = updatedResults; 207 } 208 return results; 209 } 210 } 211