1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 18 #define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 19 20 #include <cstdint> 21 #include <vector> 22 23 #include "perfetto/base/status.h" 24 #include "perfetto/protozero/proto_utils.h" 25 #include "perfetto/public/compiler.h" 26 #include "perfetto/trace_processor/status.h" 27 #include "perfetto/trace_processor/trace_blob.h" 28 #include "perfetto/trace_processor/trace_blob_view.h" 29 #include "src/trace_processor/util/gzip_utils.h" 30 #include "src/trace_processor/util/status_macros.h" 31 32 #include "protos/perfetto/trace/trace.pbzero.h" 33 #include "protos/perfetto/trace/trace_packet.pbzero.h" 34 35 namespace perfetto { 36 namespace trace_processor { 37 38 // Reads a protobuf trace in chunks and extracts boundaries of trace packets 39 // (or subfields, for the case of ftrace) with their timestamps. 40 class ProtoTraceTokenizer { 41 public: 42 ProtoTraceTokenizer(); 43 44 template <typename Callback = util::Status(TraceBlobView)> Tokenize(TraceBlobView blob,Callback callback)45 util::Status Tokenize(TraceBlobView blob, Callback callback) { 46 const uint8_t* data = blob.data(); 47 size_t size = blob.size(); 48 if (!partial_buf_.empty()) { 49 // It takes ~5 bytes for a proto preamble + the varint size. 50 const size_t kHeaderBytes = 5; 51 if (PERFETTO_UNLIKELY(partial_buf_.size() < kHeaderBytes)) { 52 size_t missing_len = std::min(kHeaderBytes - partial_buf_.size(), size); 53 partial_buf_.insert(partial_buf_.end(), &data[0], &data[missing_len]); 54 if (partial_buf_.size() < kHeaderBytes) 55 return util::OkStatus(); 56 data += missing_len; 57 size -= missing_len; 58 } 59 60 // At this point we have enough data in |partial_buf_| to read at least 61 // the field header and know the size of the next TracePacket. 62 const uint8_t* pos = &partial_buf_[0]; 63 uint8_t proto_field_tag = *pos; 64 uint64_t field_size = 0; 65 // We cannot do &partial_buf_[partial_buf_.size()] because that crashes 66 // on MSVC STL debug builds, so does &*partial_buf_.end(). 67 const uint8_t* next = protozero::proto_utils::ParseVarInt( 68 ++pos, &partial_buf_.front() + partial_buf_.size(), &field_size); 69 bool parse_failed = next == pos; 70 pos = next; 71 if (proto_field_tag != kTracePacketTag || field_size == 0 || 72 parse_failed) { 73 return util::ErrStatus( 74 "Failed parsing a TracePacket from the partial buffer"); 75 } 76 77 // At this point we know how big the TracePacket is. 78 size_t hdr_size = static_cast<size_t>(pos - &partial_buf_[0]); 79 size_t size_incl_header = static_cast<size_t>(field_size + hdr_size); 80 PERFETTO_DCHECK(size_incl_header > partial_buf_.size()); 81 82 // There is a good chance that between the |partial_buf_| and the new 83 // |data| of the current call we have enough bytes to parse a TracePacket. 84 if (partial_buf_.size() + size >= size_incl_header) { 85 // Create a new buffer for the whole TracePacket and copy into that: 86 // 1) The beginning of the TracePacket (including the proto header) from 87 // the partial buffer. 88 // 2) The rest of the TracePacket from the current |data| buffer (note 89 // that we might have consumed already a few bytes form |data| 90 // earlier in this function, hence we need to keep |off| into 91 // account). 92 TraceBlob glued = TraceBlob::Allocate(size_incl_header); 93 memcpy(glued.data(), partial_buf_.data(), partial_buf_.size()); 94 // |size_missing| is the number of bytes for the rest of the TracePacket 95 // in |data|. 96 size_t size_missing = size_incl_header - partial_buf_.size(); 97 memcpy(glued.data() + partial_buf_.size(), &data[0], size_missing); 98 data += size_missing; 99 size -= size_missing; 100 partial_buf_.clear(); 101 RETURN_IF_ERROR( 102 ParseInternal(TraceBlobView(std::move(glued)), callback)); 103 } else { 104 partial_buf_.insert(partial_buf_.end(), data, &data[size]); 105 return util::OkStatus(); 106 } 107 } 108 return ParseInternal(blob.slice(data, size), callback); 109 } 110 111 private: 112 static constexpr uint8_t kTracePacketTag = 113 protozero::proto_utils::MakeTagLengthDelimited( 114 protos::pbzero::Trace::kPacketFieldNumber); 115 116 template <typename Callback = util::Status(TraceBlobView)> ParseInternal(TraceBlobView whole_buf,Callback callback)117 util::Status ParseInternal(TraceBlobView whole_buf, Callback callback) { 118 static constexpr auto kLengthDelimited = 119 protozero::proto_utils::ProtoWireType::kLengthDelimited; 120 const uint8_t* const start = whole_buf.data(); 121 protos::pbzero::Trace::Decoder decoder(whole_buf.data(), whole_buf.size()); 122 for (auto it = decoder.packet(); it; ++it) { 123 if (PERFETTO_UNLIKELY(it->type() != kLengthDelimited)) { 124 return base::ErrStatus("Failed to parse TracePacket bounds"); 125 } 126 protozero::ConstBytes packet = *it; 127 TraceBlobView sliced = whole_buf.slice(packet.data, packet.size); 128 RETURN_IF_ERROR(ParsePacket(std::move(sliced), callback)); 129 } 130 131 const size_t bytes_left = decoder.bytes_left(); 132 if (bytes_left > 0) { 133 PERFETTO_DCHECK(partial_buf_.empty()); 134 partial_buf_.insert(partial_buf_.end(), &start[decoder.read_offset()], 135 &start[decoder.read_offset() + bytes_left]); 136 } 137 return util::OkStatus(); 138 } 139 140 template <typename Callback = util::Status(TraceBlobView)> ParsePacket(TraceBlobView packet,Callback callback)141 util::Status ParsePacket(TraceBlobView packet, Callback callback) { 142 protos::pbzero::TracePacket::Decoder decoder(packet.data(), 143 packet.length()); 144 if (decoder.has_compressed_packets()) { 145 if (!util::IsGzipSupported()) { 146 return util::Status( 147 "Cannot decode compressed packets. Zlib not enabled"); 148 } 149 150 protozero::ConstBytes field = decoder.compressed_packets(); 151 TraceBlobView compressed_packets = packet.slice(field.data, field.size); 152 TraceBlobView packets; 153 154 RETURN_IF_ERROR(Decompress(std::move(compressed_packets), &packets)); 155 156 const uint8_t* start = packets.data(); 157 const uint8_t* end = packets.data() + packets.length(); 158 const uint8_t* ptr = start; 159 while ((end - ptr) > 2) { 160 const uint8_t* packet_outer = ptr; 161 if (PERFETTO_UNLIKELY(*ptr != kTracePacketTag)) 162 return util::ErrStatus("Expected TracePacket tag"); 163 uint64_t packet_size = 0; 164 ptr = protozero::proto_utils::ParseVarInt(++ptr, end, &packet_size); 165 const uint8_t* packet_start = ptr; 166 ptr += packet_size; 167 if (PERFETTO_UNLIKELY((ptr - packet_outer) < 2 || ptr > end)) 168 return util::ErrStatus("Invalid packet size"); 169 170 TraceBlobView sliced = 171 packets.slice(packet_start, static_cast<size_t>(packet_size)); 172 RETURN_IF_ERROR(ParsePacket(std::move(sliced), callback)); 173 } 174 return util::OkStatus(); 175 } 176 return callback(std::move(packet)); 177 } 178 179 util::Status Decompress(TraceBlobView input, TraceBlobView* output); 180 181 // Used to glue together trace packets that span across two (or more) 182 // Parse() boundaries. 183 std::vector<uint8_t> partial_buf_; 184 185 // Allows support for compressed trace packets. 186 util::GzipDecompressor decompressor_; 187 }; 188 189 } // namespace trace_processor 190 } // namespace perfetto 191 192 #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 193