1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef SRC_TRACE_PROCESSOR_UTIL_PROTO_TO_ARGS_PARSER_H_ 18 #define SRC_TRACE_PROCESSOR_UTIL_PROTO_TO_ARGS_PARSER_H_ 19 20 #include <cstddef> 21 #include <cstdint> 22 #include <functional> 23 #include <optional> 24 #include <string> 25 #include <type_traits> 26 #include <unordered_map> 27 #include <vector> 28 29 #include "perfetto/base/status.h" 30 #include "perfetto/protozero/field.h" 31 #include "perfetto/protozero/proto_utils.h" 32 #include "src/trace_processor/util/descriptors.h" 33 #include "src/trace_processor/util/interned_message_view.h" 34 35 #include "protos/perfetto/trace/interned_data/interned_data.pbzero.h" 36 37 namespace perfetto::trace_processor { 38 39 // TODO(altimin): Move InternedMessageView into trace_processor/util. 40 class InternedMessageView; 41 class PacketSequenceStateGeneration; 42 43 namespace util { 44 45 // ProtoToArgsParser encapsulates the process of taking an arbitrary proto and 46 // parsing it into key-value arg pairs. This is done by traversing 47 // the proto using reflection (with descriptors from |descriptor_pool|) 48 // and passing the parsed data to |Delegate| callbacks. 49 // 50 // E.g. given a proto like 51 // 52 // package perfetto.protos; 53 // message SubMessage { 54 // optional int32 field = 1; 55 // } 56 // message MainMessage { 57 // optional int32 field1 = 1; 58 // optional string field2 = 2; 59 // optional SubMessage field3 = 3; 60 // } 61 // 62 // We will get the args set columns "field1", "field2", "field3.field" and will 63 // store the values found inside as the result. 64 // 65 // Usage of this is as follows: 66 // 67 // DescriptorPool pool; 68 // ProtoToArgsParser parser(pool); 69 // pool.AddProtoFileDescriptor( 70 // /* provide descriptor generated by tools/gen_binary_descriptors */); 71 // parser.ParseMessage(const_bytes, ".perfetto.protos.MainMessage", 72 // /* fields */, /* delegate */); 73 class ProtoToArgsParser { 74 public: 75 explicit ProtoToArgsParser(const DescriptorPool& descriptor_pool); 76 77 struct Key { 78 Key(const std::string& flat_key, const std::string& key); 79 explicit Key(const std::string& key); 80 Key(); 81 ~Key(); 82 83 std::string flat_key; 84 std::string key; 85 }; 86 87 class Delegate { 88 public: 89 virtual ~Delegate(); 90 91 virtual void AddInteger(const Key& key, int64_t value) = 0; 92 virtual void AddUnsignedInteger(const Key& key, uint64_t value) = 0; 93 virtual void AddString(const Key& key, 94 const protozero::ConstChars& value) = 0; 95 virtual void AddString(const Key& key, const std::string& value) = 0; 96 virtual void AddDouble(const Key& key, double value) = 0; 97 virtual void AddPointer(const Key& key, uint64_t value) = 0; 98 virtual void AddBoolean(const Key& key, bool value) = 0; AddBytes(const Key & key,const protozero::ConstBytes & value)99 virtual void AddBytes(const Key& key, const protozero::ConstBytes& value) { 100 // In the absence of a better implementation default to showing 101 // bytes as string with the size: 102 std::string msg = "<bytes size=" + std::to_string(value.size) + ">"; 103 AddString(key, msg); 104 } 105 // Returns whether an entry was added or not. 106 virtual bool AddJson(const Key& key, 107 const protozero::ConstChars& value) = 0; 108 virtual void AddNull(const Key& key) = 0; 109 110 virtual size_t GetArrayEntryIndex(const std::string& array_key) = 0; 111 virtual size_t IncrementArrayEntryIndex(const std::string& array_key) = 0; 112 113 virtual PacketSequenceStateGeneration* seq_state() = 0; 114 packet_timestamp()115 virtual int64_t packet_timestamp() { return 0; } 116 117 template <typename FieldMetadata> GetInternedMessage(FieldMetadata,uint64_t iid)118 typename FieldMetadata::cpp_field_type::Decoder* GetInternedMessage( 119 FieldMetadata, 120 uint64_t iid) { 121 static_assert(std::is_base_of<protozero::proto_utils::FieldMetadataBase, 122 FieldMetadata>::value, 123 "Field metadata should be a subclass of FieldMetadataBase"); 124 static_assert(std::is_same<typename FieldMetadata::message_type, 125 protos::pbzero::InternedData>::value, 126 "Field should belong to InternedData proto"); 127 auto* interned_message_view = 128 GetInternedMessageView(FieldMetadata::kFieldId, iid); 129 if (!interned_message_view) 130 return nullptr; 131 return interned_message_view->template GetOrCreateDecoder< 132 typename FieldMetadata::cpp_field_type>(); 133 } 134 ShouldAddDefaultArg(const Key &)135 virtual bool ShouldAddDefaultArg(const Key&) { return true; } 136 137 protected: 138 virtual InternedMessageView* GetInternedMessageView(uint32_t field_id, 139 uint64_t iid) = 0; 140 }; 141 142 // Given a view of bytes that represent a serialized protozero message of 143 // |type| we will parse each field. 144 // 145 // Returns on any error with a status describing the problem. However any 146 // added values before encountering the error will be parsed and forwarded to 147 // the delegate. 148 // 149 // Fields with ids given in |fields| are parsed using reflection, as well 150 // as known (previously registered) extension fields. If |allowed_fields| is a 151 // nullptr, all fields are going to be parsed. 152 // 153 // Note: 154 // |type| must be the fully qualified name, but with a '.' added to the 155 // beginning. I.E. ".perfetto.protos.TrackEvent". And must match one of the 156 // descriptors already added through |AddProtoFileDescriptor|. 157 base::Status ParseMessage(const protozero::ConstBytes& cb, 158 const std::string& type, 159 const std::vector<uint32_t>* allowed_fields, 160 Delegate& delegate, 161 int* unknown_extensions = nullptr, 162 bool add_defaults = false); 163 164 // This class is responsible for resetting the current key prefix to the old 165 // value when deleted or reset. 166 struct ScopedNestedKeyContext { 167 public: 168 ~ScopedNestedKeyContext(); 169 ScopedNestedKeyContext(ScopedNestedKeyContext&&) noexcept; 170 ScopedNestedKeyContext(const ScopedNestedKeyContext&) = delete; 171 ScopedNestedKeyContext& operator=(const ScopedNestedKeyContext&) = delete; 172 keyScopedNestedKeyContext173 const Key& key() const { return key_; } 174 175 // Clear this context, which strips the latest suffix from |key_| and sets 176 // it to the state before the nested context was created. 177 void RemoveFieldSuffix(); 178 179 private: 180 friend class ProtoToArgsParser; 181 182 explicit ScopedNestedKeyContext(Key& old_value); 183 184 struct ScopedStringAppender; 185 186 Key& key_; 187 std::optional<size_t> old_flat_key_length_ = std::nullopt; 188 std::optional<size_t> old_key_length_ = std::nullopt; 189 }; 190 191 // These methods can be called from parsing overrides to enter nested 192 // contexts. The contexts are left when the returned scope is destroyed or 193 // RemoveFieldSuffix() is called. 194 ScopedNestedKeyContext EnterDictionary(const std::string& key); 195 ScopedNestedKeyContext EnterArray(size_t index); 196 197 using ParsingOverrideForField = 198 std::function<std::optional<base::Status>(const protozero::Field&, 199 Delegate& delegate)>; 200 201 // Installs an override for the field at the specified path. We will invoke 202 // |parsing_override| when the field is encountered. 203 // 204 // The return value of |parsing_override| indicates whether the override 205 // parsed the sub-message and ProtoToArgsParser should skip it (std::nullopt) 206 // or the sub-message should continue to be parsed by ProtoToArgsParser using 207 // the descriptor (base::Status). 208 // 209 // Note |field_path| must be the full path separated by periods. I.E. in the 210 // proto 211 // 212 // message SubMessage { 213 // optional int32 field = 1; 214 // } 215 // message MainMessage { 216 // optional SubMessage field1 = 1; 217 // optional SubMessage field2 = 2; 218 // } 219 // 220 // To override the handling of both SubMessage fields you must add two parsing 221 // overrides. One with a |field_path| == "field1.field" and another with 222 // "field2.field". 223 void AddParsingOverrideForField(const std::string& field_path, 224 ParsingOverrideForField parsing_override); 225 226 using ParsingOverrideForType = std::function<std::optional<base::Status>( 227 ScopedNestedKeyContext& key, 228 const protozero::ConstBytes& data, 229 Delegate& delegate)>; 230 231 // Installs an override for all fields with the given type. We will invoke 232 // |parsing_override| when a field with the given message type is encountered. 233 // Note that the path-based overrides take precedence over type overrides. 234 // 235 // The return value of |parsing_override| indicates whether the override 236 // parsed the sub-message and ProtoToArgsParser should skip it (std::nullopt) 237 // or the sub-message should continue to be parsed by ProtoToArgsParser using 238 // the descriptor (base::Status). 239 // 240 // 241 // For example, given the following protos and a type override for SubMessage, 242 // all three fields will be parsed using this override. 243 // 244 // message SubMessage { 245 // optional int32 value = 1; 246 // } 247 // 248 // message MainMessage1 { 249 // optional SubMessage field1 = 1; 250 // optional SubMessage field2 = 2; 251 // } 252 // 253 // message MainMessage2 { 254 // optional SubMessage field3 = 1; 255 // } 256 void AddParsingOverrideForType(const std::string& message_type, 257 ParsingOverrideForType parsing_override); 258 259 private: 260 base::Status ParseField(const FieldDescriptor& field_descriptor, 261 int repeated_field_number, 262 protozero::Field field, 263 Delegate& delegate, 264 int* unknown_extensions, 265 bool add_defaults); 266 267 base::Status ParsePackedField( 268 const FieldDescriptor& field_descriptor, 269 std::unordered_map<size_t, int>& repeated_field_index, 270 protozero::Field field, 271 Delegate& delegate, 272 int* unknown_extensions, 273 bool add_defaults); 274 275 std::optional<base::Status> MaybeApplyOverrideForField( 276 const protozero::Field&, 277 Delegate& delegate); 278 279 std::optional<base::Status> MaybeApplyOverrideForType( 280 const std::string& message_type, 281 ScopedNestedKeyContext& key, 282 const protozero::ConstBytes& data, 283 Delegate& delegate); 284 285 // A type override can call |key.RemoveFieldSuffix()| if it wants to exclude 286 // the overriden field's name from the parsed args' keys. 287 base::Status ParseMessageInternal(ScopedNestedKeyContext& key, 288 const protozero::ConstBytes& cb, 289 const std::string& type, 290 const std::vector<uint32_t>* fields, 291 Delegate& delegate, 292 int* unknown_extensions, 293 bool add_defaults = false); 294 295 base::Status ParseSimpleField(const FieldDescriptor& desciptor, 296 const protozero::Field& field, 297 Delegate& delegate); 298 299 base::Status AddDefault(const FieldDescriptor& desciptor, Delegate& delegate); 300 301 base::Status AddEnum(const FieldDescriptor& descriptor, 302 int32_t value, 303 Delegate& delegate); 304 305 std::unordered_map<std::string, ParsingOverrideForField> field_overrides_; 306 std::unordered_map<std::string, ParsingOverrideForType> type_overrides_; 307 const DescriptorPool& pool_; 308 Key key_prefix_; 309 }; 310 311 } // namespace util 312 } // namespace perfetto::trace_processor 313 314 #endif // SRC_TRACE_PROCESSOR_UTIL_PROTO_TO_ARGS_PARSER_H_ 315