1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "RecordReadThread.h"
18
19 #include <sys/resource.h>
20 #include <unistd.h>
21
22 #include <algorithm>
23 #include <unordered_map>
24
25 #include "environment.h"
26 #include "event_type.h"
27 #include "record.h"
28 #include "utils.h"
29
30 namespace simpleperf {
31
32 static constexpr size_t kDefaultLowBufferLevel = 10 * 1024 * 1024u;
33 static constexpr size_t kDefaultCriticalBufferLevel = 5 * 1024 * 1024u;
34
RecordBuffer(size_t buffer_size)35 RecordBuffer::RecordBuffer(size_t buffer_size)
36 : read_head_(0), write_head_(0), buffer_size_(buffer_size), buffer_(new char[buffer_size]) {}
37
GetFreeSize() const38 size_t RecordBuffer::GetFreeSize() const {
39 size_t write_head = write_head_.load(std::memory_order_relaxed);
40 size_t read_head = read_head_.load(std::memory_order_relaxed);
41 size_t write_tail = read_head > 0 ? read_head - 1 : buffer_size_ - 1;
42 if (write_head <= write_tail) {
43 return write_tail - write_head;
44 }
45 return buffer_size_ - write_head + write_tail;
46 }
47
AllocWriteSpace(size_t record_size)48 char* RecordBuffer::AllocWriteSpace(size_t record_size) {
49 size_t write_head = write_head_.load(std::memory_order_relaxed);
50 size_t read_head = read_head_.load(std::memory_order_acquire);
51 size_t write_tail = read_head > 0 ? read_head - 1 : buffer_size_ - 1;
52 cur_write_record_size_ = record_size;
53 if (write_head < write_tail) {
54 if (write_head + record_size > write_tail) {
55 return nullptr;
56 }
57 } else if (write_head + record_size > buffer_size_) {
58 // Not enough space at the end of the buffer, need to wrap to the start of the buffer.
59 if (write_tail < record_size) {
60 return nullptr;
61 }
62 if (buffer_size_ - write_head >= sizeof(perf_event_header)) {
63 // Set the size field in perf_event_header to 0. So GetCurrentRecord() can wrap to the start
64 // of the buffer when size is 0.
65 memset(buffer_.get() + write_head, 0, sizeof(perf_event_header));
66 }
67 cur_write_record_size_ += buffer_size_ - write_head;
68 write_head = 0;
69 }
70 return buffer_.get() + write_head;
71 }
72
FinishWrite()73 void RecordBuffer::FinishWrite() {
74 size_t write_head = write_head_.load(std::memory_order_relaxed);
75 write_head = (write_head + cur_write_record_size_) % buffer_size_;
76 write_head_.store(write_head, std::memory_order_release);
77 }
78
GetCurrentRecord()79 char* RecordBuffer::GetCurrentRecord() {
80 size_t write_head = write_head_.load(std::memory_order_acquire);
81 size_t read_head = read_head_.load(std::memory_order_relaxed);
82 if (read_head == write_head) {
83 return nullptr;
84 }
85 perf_event_header header;
86 if (read_head > write_head) {
87 if (buffer_size_ - read_head < sizeof(header) ||
88 (memcpy(&header, buffer_.get() + read_head, sizeof(header)) && header.size == 0)) {
89 // Need to wrap to the start of the buffer.
90 cur_read_record_size_ += buffer_size_ - read_head;
91 read_head = 0;
92 memcpy(&header, buffer_.get(), sizeof(header));
93 }
94 } else {
95 memcpy(&header, buffer_.get() + read_head, sizeof(header));
96 }
97 cur_read_record_size_ += header.size;
98 return buffer_.get() + read_head;
99 }
100
MoveToNextRecord()101 void RecordBuffer::MoveToNextRecord() {
102 size_t read_head = read_head_.load(std::memory_order_relaxed);
103 read_head = (read_head + cur_read_record_size_) % buffer_size_;
104 read_head_.store(read_head, std::memory_order_release);
105 cur_read_record_size_ = 0;
106 }
107
RecordParser(const perf_event_attr & attr)108 RecordParser::RecordParser(const perf_event_attr& attr)
109 : sample_type_(attr.sample_type),
110 sample_regs_count_(__builtin_popcountll(attr.sample_regs_user)) {
111 size_t pos = sizeof(perf_event_header);
112 uint64_t mask = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_IP;
113 pos += __builtin_popcountll(sample_type_ & mask) * sizeof(uint64_t);
114 if (sample_type_ & PERF_SAMPLE_TID) {
115 pid_pos_in_sample_records_ = pos;
116 pos += sizeof(uint64_t);
117 }
118 if (sample_type_ & PERF_SAMPLE_TIME) {
119 time_pos_in_sample_records_ = pos;
120 pos += sizeof(uint64_t);
121 }
122 mask = PERF_SAMPLE_ADDR | PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | PERF_SAMPLE_CPU |
123 PERF_SAMPLE_PERIOD;
124 pos += __builtin_popcountll(sample_type_ & mask) * sizeof(uint64_t);
125 callchain_pos_in_sample_records_ = pos;
126 if ((sample_type_ & PERF_SAMPLE_TIME) && attr.sample_id_all) {
127 mask = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | PERF_SAMPLE_ID;
128 time_rpos_in_non_sample_records_ =
129 (__builtin_popcountll(sample_type_ & mask) + 1) * sizeof(uint64_t);
130 }
131 }
132
GetTimePos(const perf_event_header & header) const133 size_t RecordParser::GetTimePos(const perf_event_header& header) const {
134 if (header.type == PERF_RECORD_SAMPLE) {
135 return time_pos_in_sample_records_;
136 }
137 if (time_rpos_in_non_sample_records_ != 0u &&
138 time_rpos_in_non_sample_records_ < header.size - sizeof(perf_event_header)) {
139 return header.size - time_rpos_in_non_sample_records_;
140 }
141 return 0;
142 }
143
GetStackSizePos(const std::function<void (size_t,size_t,void *)> & read_record_fn) const144 size_t RecordParser::GetStackSizePos(
145 const std::function<void(size_t, size_t, void*)>& read_record_fn) const {
146 size_t pos = callchain_pos_in_sample_records_;
147 if (sample_type_ & PERF_SAMPLE_CALLCHAIN) {
148 uint64_t ip_nr;
149 read_record_fn(pos, sizeof(ip_nr), &ip_nr);
150 pos += (ip_nr + 1) * sizeof(uint64_t);
151 }
152 if (sample_type_ & PERF_SAMPLE_RAW) {
153 uint32_t size;
154 read_record_fn(pos, sizeof(size), &size);
155 pos += size + sizeof(uint32_t);
156 }
157 if (sample_type_ & PERF_SAMPLE_BRANCH_STACK) {
158 uint64_t stack_nr;
159 read_record_fn(pos, sizeof(stack_nr), &stack_nr);
160 pos += sizeof(uint64_t) + stack_nr * sizeof(BranchStackItemType);
161 }
162 if (sample_type_ & PERF_SAMPLE_REGS_USER) {
163 uint64_t abi;
164 read_record_fn(pos, sizeof(abi), &abi);
165 pos += (1 + (abi == 0 ? 0 : sample_regs_count_)) * sizeof(uint64_t);
166 }
167 return (sample_type_ & PERF_SAMPLE_STACK_USER) ? pos : 0;
168 }
169
KernelRecordReader(EventFd * event_fd)170 KernelRecordReader::KernelRecordReader(EventFd* event_fd) : event_fd_(event_fd) {
171 size_t buffer_size;
172 buffer_ = event_fd_->GetMappedBuffer(buffer_size);
173 buffer_mask_ = buffer_size - 1;
174 }
175
GetDataFromKernelBuffer()176 bool KernelRecordReader::GetDataFromKernelBuffer() {
177 data_size_ = event_fd_->GetAvailableMmapDataSize(data_pos_);
178 if (data_size_ == 0) {
179 return false;
180 }
181 init_data_size_ = data_size_;
182 record_header_.size = 0;
183 return true;
184 }
185
ReadRecord(size_t pos,size_t size,void * dest)186 void KernelRecordReader::ReadRecord(size_t pos, size_t size, void* dest) {
187 pos = (pos + data_pos_) & buffer_mask_;
188 size_t copy_size = std::min(size, buffer_mask_ + 1 - pos);
189 memcpy(dest, buffer_ + pos, copy_size);
190 if (copy_size < size) {
191 memcpy(static_cast<char*>(dest) + copy_size, buffer_, size - copy_size);
192 }
193 }
194
MoveToNextRecord(const RecordParser & parser)195 bool KernelRecordReader::MoveToNextRecord(const RecordParser& parser) {
196 data_pos_ = (data_pos_ + record_header_.size) & buffer_mask_;
197 data_size_ -= record_header_.size;
198 if (data_size_ == 0) {
199 event_fd_->DiscardMmapData(init_data_size_);
200 init_data_size_ = 0;
201 return false;
202 }
203 ReadRecord(0, sizeof(record_header_), &record_header_);
204 size_t time_pos = parser.GetTimePos(record_header_);
205 if (time_pos != 0) {
206 ReadRecord(time_pos, sizeof(record_time_), &record_time_);
207 }
208 return true;
209 }
210
RecordReadThread(size_t record_buffer_size,const perf_event_attr & attr,size_t min_mmap_pages,size_t max_mmap_pages,size_t aux_buffer_size,bool allow_cutting_samples,bool exclude_perf)211 RecordReadThread::RecordReadThread(size_t record_buffer_size, const perf_event_attr& attr,
212 size_t min_mmap_pages, size_t max_mmap_pages,
213 size_t aux_buffer_size, bool allow_cutting_samples,
214 bool exclude_perf)
215 : record_buffer_(record_buffer_size),
216 record_parser_(attr),
217 attr_(attr),
218 min_mmap_pages_(min_mmap_pages),
219 max_mmap_pages_(max_mmap_pages),
220 aux_buffer_size_(aux_buffer_size) {
221 if (attr.sample_type & PERF_SAMPLE_STACK_USER) {
222 stack_size_in_sample_record_ = attr.sample_stack_user;
223 }
224 record_buffer_low_level_ = std::min(record_buffer_size / 4, kDefaultLowBufferLevel);
225 record_buffer_critical_level_ = std::min(record_buffer_size / 6, kDefaultCriticalBufferLevel);
226 if (!allow_cutting_samples) {
227 record_buffer_low_level_ = record_buffer_critical_level_;
228 }
229 if (exclude_perf) {
230 exclude_pid_ = getpid();
231 }
232 }
233
~RecordReadThread()234 RecordReadThread::~RecordReadThread() {
235 if (read_thread_) {
236 StopReadThread();
237 }
238 }
239
RegisterDataCallback(IOEventLoop & loop,const std::function<bool ()> & data_callback)240 bool RecordReadThread::RegisterDataCallback(IOEventLoop& loop,
241 const std::function<bool()>& data_callback) {
242 int cmd_fd[2];
243 int data_fd[2];
244 if (pipe2(cmd_fd, O_CLOEXEC) != 0 || pipe2(data_fd, O_CLOEXEC) != 0) {
245 PLOG(ERROR) << "pipe2";
246 return false;
247 }
248 read_cmd_fd_.reset(cmd_fd[0]);
249 write_cmd_fd_.reset(cmd_fd[1]);
250 cmd_ = NO_CMD;
251 read_data_fd_.reset(data_fd[0]);
252 write_data_fd_.reset(data_fd[1]);
253 has_data_notification_ = false;
254 if (!loop.AddReadEvent(read_data_fd_, data_callback)) {
255 return false;
256 }
257 read_thread_.reset(new std::thread([&]() { RunReadThread(); }));
258 return true;
259 }
260
AddEventFds(const std::vector<EventFd * > & event_fds)261 bool RecordReadThread::AddEventFds(const std::vector<EventFd*>& event_fds) {
262 return SendCmdToReadThread(CMD_ADD_EVENT_FDS, const_cast<std::vector<EventFd*>*>(&event_fds));
263 }
264
RemoveEventFds(const std::vector<EventFd * > & event_fds)265 bool RecordReadThread::RemoveEventFds(const std::vector<EventFd*>& event_fds) {
266 return SendCmdToReadThread(CMD_REMOVE_EVENT_FDS, const_cast<std::vector<EventFd*>*>(&event_fds));
267 }
268
SyncKernelBuffer()269 bool RecordReadThread::SyncKernelBuffer() {
270 return SendCmdToReadThread(CMD_SYNC_KERNEL_BUFFER, nullptr);
271 }
272
StopReadThread()273 bool RecordReadThread::StopReadThread() {
274 bool result = true;
275 if (read_thread_ != nullptr) {
276 result = SendCmdToReadThread(CMD_STOP_THREAD, nullptr);
277 if (result) {
278 read_thread_->join();
279 read_thread_ = nullptr;
280 }
281 }
282 return result;
283 }
284
SendCmdToReadThread(Cmd cmd,void * cmd_arg)285 bool RecordReadThread::SendCmdToReadThread(Cmd cmd, void* cmd_arg) {
286 {
287 std::lock_guard<std::mutex> lock(cmd_mutex_);
288 cmd_ = cmd;
289 cmd_arg_ = cmd_arg;
290 }
291 char unused = 0;
292 if (TEMP_FAILURE_RETRY(write(write_cmd_fd_, &unused, 1)) != 1) {
293 return false;
294 }
295 std::unique_lock<std::mutex> lock(cmd_mutex_);
296 while (cmd_ != NO_CMD) {
297 cmd_finish_cond_.wait(lock);
298 }
299 return cmd_result_;
300 }
301
GetRecord()302 std::unique_ptr<Record> RecordReadThread::GetRecord() {
303 record_buffer_.MoveToNextRecord();
304 char* p = record_buffer_.GetCurrentRecord();
305 if (p != nullptr) {
306 std::unique_ptr<Record> r = ReadRecordFromBuffer(attr_, p);
307 if (r->type() == PERF_RECORD_AUXTRACE) {
308 auto auxtrace = static_cast<AuxTraceRecord*>(r.get());
309 record_buffer_.AddCurrentRecordSize(auxtrace->data->aux_size);
310 auxtrace->location.addr = r->Binary() + r->size();
311 }
312 return r;
313 }
314 if (has_data_notification_) {
315 char unused;
316 TEMP_FAILURE_RETRY(read(read_data_fd_, &unused, 1));
317 has_data_notification_ = false;
318 }
319 return nullptr;
320 }
321
RunReadThread()322 void RecordReadThread::RunReadThread() {
323 IncreaseThreadPriority();
324 IOEventLoop loop;
325 CHECK(loop.AddReadEvent(read_cmd_fd_, [&]() { return HandleCmd(loop); }));
326 loop.RunLoop();
327 }
328
IncreaseThreadPriority()329 void RecordReadThread::IncreaseThreadPriority() {
330 // TODO: use real time priority for root.
331 rlimit rlim;
332 int result = getrlimit(RLIMIT_NICE, &rlim);
333 if (result == 0 && rlim.rlim_cur == 40) {
334 result = setpriority(PRIO_PROCESS, gettid(), -20);
335 if (result == 0) {
336 LOG(VERBOSE) << "Priority of record read thread is increased";
337 }
338 }
339 }
340
GetCmd()341 RecordReadThread::Cmd RecordReadThread::GetCmd() {
342 std::lock_guard<std::mutex> lock(cmd_mutex_);
343 return cmd_;
344 }
345
HandleCmd(IOEventLoop & loop)346 bool RecordReadThread::HandleCmd(IOEventLoop& loop) {
347 char unused;
348 TEMP_FAILURE_RETRY(read(read_cmd_fd_, &unused, 1));
349 bool result = true;
350 switch (GetCmd()) {
351 case CMD_ADD_EVENT_FDS:
352 result = HandleAddEventFds(loop, *static_cast<std::vector<EventFd*>*>(cmd_arg_));
353 break;
354 case CMD_REMOVE_EVENT_FDS:
355 result = HandleRemoveEventFds(*static_cast<std::vector<EventFd*>*>(cmd_arg_));
356 break;
357 case CMD_SYNC_KERNEL_BUFFER:
358 result = ReadRecordsFromKernelBuffer();
359 break;
360 case CMD_STOP_THREAD:
361 result = loop.ExitLoop();
362 break;
363 default:
364 LOG(ERROR) << "Unknown cmd: " << GetCmd();
365 result = false;
366 break;
367 }
368 std::lock_guard<std::mutex> lock(cmd_mutex_);
369 cmd_ = NO_CMD;
370 cmd_result_ = result;
371 cmd_finish_cond_.notify_one();
372 return true;
373 }
374
HandleAddEventFds(IOEventLoop & loop,const std::vector<EventFd * > & event_fds)375 bool RecordReadThread::HandleAddEventFds(IOEventLoop& loop,
376 const std::vector<EventFd*>& event_fds) {
377 std::unordered_map<int, EventFd*> cpu_map;
378 for (size_t pages = max_mmap_pages_; pages >= min_mmap_pages_; pages >>= 1) {
379 bool success = true;
380 bool report_error = pages == min_mmap_pages_;
381 for (EventFd* fd : event_fds) {
382 auto it = cpu_map.find(fd->Cpu());
383 if (it == cpu_map.end()) {
384 if (!fd->CreateMappedBuffer(pages, report_error)) {
385 success = false;
386 break;
387 }
388 if (IsEtmEventType(fd->attr().type)) {
389 if (!fd->CreateAuxBuffer(aux_buffer_size_, report_error)) {
390 fd->DestroyMappedBuffer();
391 success = false;
392 break;
393 }
394 }
395 cpu_map[fd->Cpu()] = fd;
396 } else {
397 if (!fd->ShareMappedBuffer(*(it->second), pages == min_mmap_pages_)) {
398 success = false;
399 break;
400 }
401 }
402 }
403 if (success) {
404 LOG(VERBOSE) << "Each kernel buffer is " << pages << " pages.";
405 break;
406 }
407 for (auto& pair : cpu_map) {
408 pair.second->DestroyMappedBuffer();
409 pair.second->DestroyAuxBuffer();
410 }
411 cpu_map.clear();
412 }
413 if (cpu_map.empty()) {
414 return false;
415 }
416 for (auto& pair : cpu_map) {
417 if (!pair.second->StartPolling(loop, [this]() { return ReadRecordsFromKernelBuffer(); })) {
418 return false;
419 }
420 kernel_record_readers_.emplace_back(pair.second);
421 }
422 return true;
423 }
424
HandleRemoveEventFds(const std::vector<EventFd * > & event_fds)425 bool RecordReadThread::HandleRemoveEventFds(const std::vector<EventFd*>& event_fds) {
426 for (auto& event_fd : event_fds) {
427 if (event_fd->HasMappedBuffer()) {
428 auto it = std::find_if(
429 kernel_record_readers_.begin(), kernel_record_readers_.end(),
430 [&](const KernelRecordReader& reader) { return reader.GetEventFd() == event_fd; });
431 if (it != kernel_record_readers_.end()) {
432 kernel_record_readers_.erase(it);
433 event_fd->StopPolling();
434 event_fd->DestroyMappedBuffer();
435 event_fd->DestroyAuxBuffer();
436 }
437 }
438 }
439 return true;
440 }
441
CompareRecordTime(KernelRecordReader * r1,KernelRecordReader * r2)442 static bool CompareRecordTime(KernelRecordReader* r1, KernelRecordReader* r2) {
443 return r1->RecordTime() > r2->RecordTime();
444 }
445
446 // When reading from mmap buffers, we prefer reading from all buffers at once rather than reading
447 // one buffer at a time. Because by reading all buffers at once, we can merge records from
448 // different buffers easily in memory. Otherwise, we have to sort records with greater effort.
ReadRecordsFromKernelBuffer()449 bool RecordReadThread::ReadRecordsFromKernelBuffer() {
450 do {
451 std::vector<KernelRecordReader*> readers;
452 for (auto& reader : kernel_record_readers_) {
453 if (reader.GetDataFromKernelBuffer()) {
454 readers.push_back(&reader);
455 }
456 }
457 bool has_data = false;
458 if (!readers.empty()) {
459 has_data = true;
460 if (readers.size() == 1u) {
461 // Only one buffer has data, process it directly.
462 while (readers[0]->MoveToNextRecord(record_parser_)) {
463 PushRecordToRecordBuffer(readers[0]);
464 }
465 } else {
466 // Use a binary heap to merge records from different buffers. As records from the same
467 // buffer are already ordered by time, we only need to merge the first record from all
468 // buffers. And each time a record is popped from the heap, we put the next record from its
469 // buffer into the heap.
470 for (auto& reader : readers) {
471 reader->MoveToNextRecord(record_parser_);
472 }
473 std::make_heap(readers.begin(), readers.end(), CompareRecordTime);
474 size_t size = readers.size();
475 while (size > 0) {
476 std::pop_heap(readers.begin(), readers.begin() + size, CompareRecordTime);
477 PushRecordToRecordBuffer(readers[size - 1]);
478 if (readers[size - 1]->MoveToNextRecord(record_parser_)) {
479 std::push_heap(readers.begin(), readers.begin() + size, CompareRecordTime);
480 } else {
481 size--;
482 }
483 }
484 }
485 }
486 ReadAuxDataFromKernelBuffer(&has_data);
487 if (!has_data) {
488 break;
489 }
490 if (!SendDataNotificationToMainThread()) {
491 return false;
492 }
493 // If there are no commands, we can loop until there is no more data from the kernel.
494 } while (GetCmd() == NO_CMD);
495 return true;
496 }
497
PushRecordToRecordBuffer(KernelRecordReader * kernel_record_reader)498 void RecordReadThread::PushRecordToRecordBuffer(KernelRecordReader* kernel_record_reader) {
499 const perf_event_header& header = kernel_record_reader->RecordHeader();
500 if (header.type == PERF_RECORD_SAMPLE && exclude_pid_ != -1) {
501 uint32_t pid;
502 kernel_record_reader->ReadRecord(record_parser_.GetPidPosInSampleRecord(), sizeof(pid), &pid);
503 if (pid == exclude_pid_) {
504 return;
505 }
506 }
507 if (header.type == PERF_RECORD_SAMPLE && stack_size_in_sample_record_ > 1024) {
508 size_t free_size = record_buffer_.GetFreeSize();
509 if (free_size < record_buffer_critical_level_) {
510 // When the free size in record buffer is below critical level, drop sample records to save
511 // space for more important records (like mmap or fork records).
512 stat_.lost_samples++;
513 return;
514 }
515 size_t stack_size_limit = stack_size_in_sample_record_;
516 if (free_size < record_buffer_low_level_) {
517 // When the free size in record buffer is below low level, cut the stack data in sample
518 // records to 1K. This makes the unwinder unwind only part of the callchains, but hopefully
519 // the call chain joiner can complete the callchains.
520 stack_size_limit = 1024;
521 }
522 size_t stack_size_pos =
523 record_parser_.GetStackSizePos([&](size_t pos, size_t size, void* dest) {
524 return kernel_record_reader->ReadRecord(pos, size, dest);
525 });
526 uint64_t stack_size;
527 kernel_record_reader->ReadRecord(stack_size_pos, sizeof(stack_size), &stack_size);
528 if (stack_size > 0) {
529 size_t dyn_stack_size_pos = stack_size_pos + sizeof(stack_size) + stack_size;
530 uint64_t dyn_stack_size;
531 kernel_record_reader->ReadRecord(dyn_stack_size_pos, sizeof(dyn_stack_size), &dyn_stack_size);
532 if (dyn_stack_size == 0) {
533 // If stack_user_data.dyn_size == 0, it may be because the kernel misses the patch to
534 // update dyn_size, like in N9 (See b/22612370). So assume all stack data is valid if
535 // dyn_size == 0.
536 // TODO: Add cts test.
537 dyn_stack_size = stack_size;
538 }
539 // When simpleperf requests the kernel to dump 64K stack per sample, it will allocate 64K
540 // space in each sample to store stack data. However, a thread may use less stack than 64K.
541 // So not all the 64K stack data in a sample is valid, and we only need to keep valid stack
542 // data, whose size is dyn_stack_size.
543 uint64_t new_stack_size = std::min<uint64_t>(dyn_stack_size, stack_size_limit);
544 if (stack_size > new_stack_size) {
545 // Remove part of the stack data.
546 perf_event_header new_header = header;
547 new_header.size -= stack_size - new_stack_size;
548 char* p = record_buffer_.AllocWriteSpace(new_header.size);
549 if (p != nullptr) {
550 memcpy(p, &new_header, sizeof(new_header));
551 size_t pos = sizeof(new_header);
552 kernel_record_reader->ReadRecord(pos, stack_size_pos - pos, p + pos);
553 memcpy(p + stack_size_pos, &new_stack_size, sizeof(uint64_t));
554 pos = stack_size_pos + sizeof(uint64_t);
555 kernel_record_reader->ReadRecord(pos, new_stack_size, p + pos);
556 memcpy(p + pos + new_stack_size, &new_stack_size, sizeof(uint64_t));
557 record_buffer_.FinishWrite();
558 if (new_stack_size < dyn_stack_size) {
559 stat_.cut_stack_samples++;
560 }
561 } else {
562 stat_.lost_samples++;
563 }
564 return;
565 }
566 }
567 }
568 char* p = record_buffer_.AllocWriteSpace(header.size);
569 if (p != nullptr) {
570 kernel_record_reader->ReadRecord(0, header.size, p);
571 record_buffer_.FinishWrite();
572 } else {
573 if (header.type == PERF_RECORD_SAMPLE) {
574 stat_.lost_samples++;
575 } else {
576 stat_.lost_non_samples++;
577 }
578 }
579 }
580
ReadAuxDataFromKernelBuffer(bool * has_data)581 void RecordReadThread::ReadAuxDataFromKernelBuffer(bool* has_data) {
582 for (auto& reader : kernel_record_readers_) {
583 EventFd* event_fd = reader.GetEventFd();
584 if (event_fd->HasAuxBuffer()) {
585 char* buf[2];
586 size_t size[2];
587 uint64_t offset = event_fd->GetAvailableAuxData(&buf[0], &size[0], &buf[1], &size[1]);
588 size_t aux_size = size[0] + size[1];
589 if (aux_size == 0) {
590 continue;
591 }
592 *has_data = true;
593 AuxTraceRecord auxtrace(Align(aux_size, 8), offset, event_fd->Cpu(), 0, event_fd->Cpu());
594 size_t alloc_size = auxtrace.size() + auxtrace.data->aux_size;
595 if (record_buffer_.GetFreeSize() < alloc_size + record_buffer_critical_level_) {
596 stat_.lost_aux_data_size += aux_size;
597 } else {
598 char* p = record_buffer_.AllocWriteSpace(alloc_size);
599 CHECK(p != nullptr);
600 MoveToBinaryFormat(auxtrace.Binary(), auxtrace.size(), p);
601 MoveToBinaryFormat(buf[0], size[0], p);
602 if (size[1] != 0) {
603 MoveToBinaryFormat(buf[1], size[1], p);
604 }
605 size_t pad_size = auxtrace.data->aux_size - aux_size;
606 if (pad_size != 0) {
607 uint64_t pad = 0;
608 memcpy(p, &pad, pad_size);
609 }
610 record_buffer_.FinishWrite();
611 stat_.aux_data_size += aux_size;
612 LOG(DEBUG) << "record aux data " << aux_size << " bytes";
613 }
614 event_fd->DiscardAuxData(aux_size);
615 }
616 }
617 }
618
SendDataNotificationToMainThread()619 bool RecordReadThread::SendDataNotificationToMainThread() {
620 if (!has_data_notification_.load(std::memory_order_relaxed)) {
621 has_data_notification_ = true;
622 char unused = 0;
623 if (TEMP_FAILURE_RETRY(write(write_data_fd_, &unused, 1)) != 1) {
624 PLOG(ERROR) << "write";
625 return false;
626 }
627 }
628 return true;
629 }
630
631 } // namespace simpleperf
632