1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "perfetto/trace_processor/read_trace.h"
18
19 #include "perfetto/base/logging.h"
20 #include "perfetto/ext/base/file_utils.h"
21 #include "perfetto/ext/base/scoped_file.h"
22 #include "perfetto/ext/base/utils.h"
23 #include "perfetto/protozero/proto_utils.h"
24 #include "perfetto/trace_processor/trace_processor.h"
25
26 #include "perfetto/trace_processor/trace_blob.h"
27 #include "perfetto/trace_processor/trace_blob_view.h"
28 #include "src/trace_processor/forwarding_trace_parser.h"
29 #include "src/trace_processor/importers/gzip/gzip_trace_parser.h"
30 #include "src/trace_processor/importers/proto/proto_trace_tokenizer.h"
31 #include "src/trace_processor/util/gzip_utils.h"
32 #include "src/trace_processor/util/status_macros.h"
33
34 #include "protos/perfetto/trace/trace.pbzero.h"
35 #include "protos/perfetto/trace/trace_packet.pbzero.h"
36
37 #if TRACE_PROCESSOR_HAS_MMAP()
38 #include <stdlib.h>
39 #include <sys/mman.h>
40 #include <unistd.h>
41 #endif
42
43 namespace perfetto {
44 namespace trace_processor {
45 namespace {
46
47 // 1MB chunk size seems the best tradeoff on a MacBook Pro 2013 - i7 2.8 GHz.
48 constexpr size_t kChunkSize = 1024 * 1024;
49
ReadTraceUsingRead(TraceProcessor * tp,int fd,uint64_t * file_size,const std::function<void (uint64_t parsed_size)> & progress_callback)50 util::Status ReadTraceUsingRead(
51 TraceProcessor* tp,
52 int fd,
53 uint64_t* file_size,
54 const std::function<void(uint64_t parsed_size)>& progress_callback) {
55 // Load the trace in chunks using ordinary read().
56 for (int i = 0;; i++) {
57 if (progress_callback && i % 128 == 0)
58 progress_callback(*file_size);
59
60 TraceBlob blob = TraceBlob::Allocate(kChunkSize);
61 auto rsize = base::Read(fd, blob.data(), blob.size());
62 if (rsize == 0)
63 break;
64
65 if (rsize < 0) {
66 return util::ErrStatus("Reading trace file failed (errno: %d, %s)", errno,
67 strerror(errno));
68 }
69
70 *file_size += static_cast<uint64_t>(rsize);
71 TraceBlobView blob_view(std::move(blob), 0, static_cast<size_t>(rsize));
72 RETURN_IF_ERROR(tp->Parse(std::move(blob_view)));
73 }
74 return util::OkStatus();
75 }
76
77 class SerializingProtoTraceReader : public ChunkedTraceReader {
78 public:
SerializingProtoTraceReader(std::vector<uint8_t> * output)79 explicit SerializingProtoTraceReader(std::vector<uint8_t>* output)
80 : output_(output) {}
81
Parse(TraceBlobView blob)82 util::Status Parse(TraceBlobView blob) override {
83 return tokenizer_.Tokenize(std::move(blob), [this](TraceBlobView packet) {
84 uint8_t buffer[protozero::proto_utils::kMaxSimpleFieldEncodedSize];
85
86 uint8_t* pos = buffer;
87 pos = protozero::proto_utils::WriteVarInt(kTracePacketTag, pos);
88 pos = protozero::proto_utils::WriteVarInt(packet.length(), pos);
89 output_->insert(output_->end(), buffer, pos);
90
91 output_->insert(output_->end(), packet.data(),
92 packet.data() + packet.length());
93 return util::OkStatus();
94 });
95 }
96
NotifyEndOfFile()97 void NotifyEndOfFile() override {}
98
99 private:
100 static constexpr uint8_t kTracePacketTag =
101 protozero::proto_utils::MakeTagLengthDelimited(
102 protos::pbzero::Trace::kPacketFieldNumber);
103
104 ProtoTraceTokenizer tokenizer_;
105 std::vector<uint8_t>* output_;
106 };
107
108 } // namespace
109
ReadTrace(TraceProcessor * tp,const char * filename,const std::function<void (uint64_t parsed_size)> & progress_callback)110 util::Status ReadTrace(
111 TraceProcessor* tp,
112 const char* filename,
113 const std::function<void(uint64_t parsed_size)>& progress_callback) {
114 base::ScopedFile fd(base::OpenFile(filename, O_RDONLY));
115 if (!fd)
116 return util::ErrStatus("Could not open trace file (path: %s)", filename);
117
118 uint64_t bytes_read = 0;
119
120 #if TRACE_PROCESSOR_HAS_MMAP()
121 char* no_mmap = getenv("TRACE_PROCESSOR_NO_MMAP");
122 uint64_t whole_size_64 = static_cast<uint64_t>(lseek(*fd, 0, SEEK_END));
123 lseek(*fd, 0, SEEK_SET);
124 bool use_mmap = !no_mmap || *no_mmap != '1';
125 if (sizeof(size_t) < 8 && whole_size_64 > 2147483648ULL)
126 use_mmap = false; // Cannot use mmap on 32-bit systems for files > 2GB.
127
128 if (use_mmap) {
129 const size_t whole_size = static_cast<size_t>(whole_size_64);
130 void* file_mm = mmap(nullptr, whole_size, PROT_READ, MAP_PRIVATE, *fd, 0);
131 if (file_mm != MAP_FAILED) {
132 TraceBlobView whole_mmap(TraceBlob::FromMmap(file_mm, whole_size));
133 // Parse the file in chunks so we get some status update on stdio.
134 static constexpr size_t kMmapChunkSize = 128ul * 1024 * 1024;
135 while (bytes_read < whole_size_64) {
136 progress_callback(bytes_read);
137 const size_t bytes_read_z = static_cast<size_t>(bytes_read);
138 size_t slice_size = std::min(whole_size - bytes_read_z, kMmapChunkSize);
139 TraceBlobView slice = whole_mmap.slice_off(bytes_read_z, slice_size);
140 RETURN_IF_ERROR(tp->Parse(std::move(slice)));
141 bytes_read += slice_size;
142 } // while (slices)
143 } // if (!MAP_FAILED)
144 } // if (use_mmap)
145 if (bytes_read == 0)
146 PERFETTO_LOG("Cannot use mmap on this system. Falling back on read()");
147 #endif // TRACE_PROCESSOR_HAS_MMAP()
148 if (bytes_read == 0) {
149 RETURN_IF_ERROR(
150 ReadTraceUsingRead(tp, *fd, &bytes_read, progress_callback));
151 }
152 tp->NotifyEndOfFile();
153 tp->SetCurrentTraceName(filename);
154
155 if (progress_callback)
156 progress_callback(bytes_read);
157 return util::OkStatus();
158 }
159
DecompressTrace(const uint8_t * data,size_t size,std::vector<uint8_t> * output)160 util::Status DecompressTrace(const uint8_t* data,
161 size_t size,
162 std::vector<uint8_t>* output) {
163 TraceType type = GuessTraceType(data, size);
164 if (type != TraceType::kGzipTraceType && type != TraceType::kProtoTraceType) {
165 return util::ErrStatus(
166 "Only GZIP and proto trace types are supported by DecompressTrace");
167 }
168
169 if (type == TraceType::kGzipTraceType) {
170 std::unique_ptr<ChunkedTraceReader> reader(
171 new SerializingProtoTraceReader(output));
172 GzipTraceParser parser(std::move(reader));
173
174 RETURN_IF_ERROR(parser.ParseUnowned(data, size));
175 if (parser.needs_more_input())
176 return util::ErrStatus("Cannot decompress partial trace file");
177
178 parser.NotifyEndOfFile();
179 return util::OkStatus();
180 }
181
182 PERFETTO_CHECK(type == TraceType::kProtoTraceType);
183
184 protos::pbzero::Trace::Decoder decoder(data, size);
185 util::GzipDecompressor decompressor;
186 if (size > 0 && !decoder.packet()) {
187 return util::ErrStatus("Trace does not contain valid packets");
188 }
189 for (auto it = decoder.packet(); it; ++it) {
190 protos::pbzero::TracePacket::Decoder packet(*it);
191 if (!packet.has_compressed_packets()) {
192 it->SerializeAndAppendTo(output);
193 continue;
194 }
195
196 // Make sure that to reset the stream between the gzip streams.
197 auto bytes = packet.compressed_packets();
198 decompressor.Reset();
199 using ResultCode = util::GzipDecompressor::ResultCode;
200 ResultCode ret = decompressor.FeedAndExtract(
201 bytes.data, bytes.size, [&output](const uint8_t* buf, size_t buf_len) {
202 output->insert(output->end(), buf, buf + buf_len);
203 });
204 if (ret == ResultCode::kError || ret == ResultCode::kNeedsMoreInput) {
205 return util::ErrStatus("Failed while decompressing stream");
206 }
207 }
208 return util::OkStatus();
209 }
210
211 } // namespace trace_processor
212 } // namespace perfetto
213