1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/profiling/memory/shared_ring_buffer.h"
18
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <sys/mman.h>
22 #include <sys/stat.h>
23 #include <unistd.h>
24
25 #include <atomic>
26 #include <cinttypes>
27 #include <type_traits>
28
29 #include "perfetto/base/build_config.h"
30 #include "perfetto/ext/base/scoped_file.h"
31 #include "perfetto/ext/base/temp_file.h"
32 #include "src/profiling/memory/scoped_spinlock.h"
33
34 #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID)
35 #include <linux/memfd.h>
36 #include <sys/syscall.h>
37 #endif
38
39 namespace perfetto {
40 namespace profiling {
41
42 namespace {
43
44 constexpr auto kMetaPageSize = base::kPageSize;
45 constexpr auto kAlignment = 8; // 64 bits to use aligned memcpy().
46 constexpr auto kHeaderSize = kAlignment;
47 constexpr auto kGuardSize = base::kPageSize * 1024 * 16; // 64 MB.
48 #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID)
49 constexpr auto kFDSeals = F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_SEAL;
50 #endif
51
52 } // namespace
53
SharedRingBuffer(CreateFlag,size_t size)54 SharedRingBuffer::SharedRingBuffer(CreateFlag, size_t size) {
55 size_t size_with_meta = size + kMetaPageSize;
56 base::ScopedFile fd;
57 #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID)
58 bool is_memfd = false;
59 fd.reset(static_cast<int>(syscall(__NR_memfd_create, "heapprofd_ringbuf",
60 MFD_CLOEXEC | MFD_ALLOW_SEALING)));
61 is_memfd = !!fd;
62
63 if (!fd) {
64 #if PERFETTO_BUILDFLAG(PERFETTO_ANDROID_BUILD)
65 // In-tree builds only allow mem_fd, so we can inspect the seals to verify
66 // the fd is appropriately sealed.
67 PERFETTO_ELOG("memfd_create() failed");
68 return;
69 #else
70 PERFETTO_DPLOG("memfd_create() failed");
71 #endif
72 }
73 #endif
74
75 if (!fd)
76 fd = base::TempFile::CreateUnlinked().ReleaseFD();
77
78 PERFETTO_CHECK(fd);
79 int res = ftruncate(fd.get(), static_cast<off_t>(size_with_meta));
80 PERFETTO_CHECK(res == 0);
81 #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID)
82 if (is_memfd) {
83 res = fcntl(*fd, F_ADD_SEALS, kFDSeals);
84 if (res != 0) {
85 PERFETTO_PLOG("Failed to seal FD.");
86 return;
87 }
88 }
89 #endif
90 Initialize(std::move(fd));
91 if (!is_valid())
92 return;
93
94 new (meta_) MetadataPage();
95 }
96
~SharedRingBuffer()97 SharedRingBuffer::~SharedRingBuffer() {
98 static_assert(std::is_trivially_constructible<MetadataPage>::value,
99 "MetadataPage must be trivially constructible");
100 static_assert(std::is_trivially_destructible<MetadataPage>::value,
101 "MetadataPage must be trivially destructible");
102
103 if (is_valid()) {
104 size_t outer_size = kMetaPageSize + size_ * 2 + kGuardSize;
105 munmap(meta_, outer_size);
106 }
107
108 // This is work-around for code like the following:
109 // https://android.googlesource.com/platform/libcore/+/4ecb71f94378716f88703b9f7548b5d24839262f/ojluni/src/main/native/UNIXProcess_md.c#427
110 // They fork, close all fds by iterating over /proc/self/fd using opendir.
111 // Unfortunately closedir calls free, which detects the fork, and then tries
112 // to destruct the Client that holds this SharedRingBuffer.
113 //
114 // ScopedResource crashes on failure to close, so we explicitly ignore
115 // failures here.
116 int fd = mem_fd_.release();
117 if (fd != -1)
118 close(fd);
119 }
120
Initialize(base::ScopedFile mem_fd)121 void SharedRingBuffer::Initialize(base::ScopedFile mem_fd) {
122 #if PERFETTO_BUILDFLAG(PERFETTO_ANDROID_BUILD)
123 int seals = fcntl(*mem_fd, F_GET_SEALS);
124 if (seals == -1) {
125 PERFETTO_PLOG("Failed to get seals of FD.");
126 return;
127 }
128 if ((seals & kFDSeals) != kFDSeals) {
129 PERFETTO_ELOG("FD not properly sealed. Expected %x, got %x", kFDSeals,
130 seals);
131 return;
132 }
133 #endif
134
135 struct stat stat_buf = {};
136 int res = fstat(*mem_fd, &stat_buf);
137 if (res != 0 || stat_buf.st_size == 0) {
138 PERFETTO_PLOG("Could not attach to fd.");
139 return;
140 }
141 auto size_with_meta = static_cast<size_t>(stat_buf.st_size);
142 auto size = size_with_meta - kMetaPageSize;
143
144 // |size_with_meta| must be a power of two number of pages + 1 page (for
145 // metadata).
146 if (size_with_meta < 2 * base::kPageSize || size % base::kPageSize ||
147 (size & (size - 1))) {
148 PERFETTO_ELOG("SharedRingBuffer size is invalid (%zu)", size_with_meta);
149 return;
150 }
151
152 // First of all reserve the whole virtual region to fit the buffer twice
153 // + metadata page + red zone at the end.
154 size_t outer_size = kMetaPageSize + size * 2 + kGuardSize;
155 uint8_t* region = reinterpret_cast<uint8_t*>(
156 mmap(nullptr, outer_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
157 if (region == MAP_FAILED) {
158 PERFETTO_PLOG("mmap(PROT_NONE) failed");
159 return;
160 }
161
162 // Map first the whole buffer (including the initial metadata page) @ off=0.
163 void* reg1 = mmap(region, size_with_meta, PROT_READ | PROT_WRITE,
164 MAP_SHARED | MAP_FIXED, *mem_fd, 0);
165
166 // Then map again the buffer, skipping the metadata page. The final result is:
167 // [ METADATA ] [ RING BUFFER SHMEM ] [ RING BUFFER SHMEM ]
168 void* reg2 = mmap(region + size_with_meta, size, PROT_READ | PROT_WRITE,
169 MAP_SHARED | MAP_FIXED, *mem_fd,
170 /*offset=*/kMetaPageSize);
171
172 if (reg1 != region || reg2 != region + size_with_meta) {
173 PERFETTO_PLOG("mmap(MAP_SHARED) failed");
174 munmap(region, outer_size);
175 return;
176 }
177 set_size(size);
178 meta_ = reinterpret_cast<MetadataPage*>(region);
179 mem_ = region + kMetaPageSize;
180 mem_fd_ = std::move(mem_fd);
181 }
182
BeginWrite(const ScopedSpinlock & spinlock,size_t size)183 SharedRingBuffer::Buffer SharedRingBuffer::BeginWrite(
184 const ScopedSpinlock& spinlock,
185 size_t size) {
186 PERFETTO_DCHECK(spinlock.locked());
187 Buffer result;
188
189 std::optional<PointerPositions> opt_pos = GetPointerPositions();
190 if (!opt_pos) {
191 meta_->stats.num_writes_corrupt++;
192 errno = EBADF;
193 return result;
194 }
195 auto pos = opt_pos.value();
196
197 const uint64_t size_with_header =
198 base::AlignUp<kAlignment>(size + kHeaderSize);
199
200 // size_with_header < size is for catching overflow of size_with_header.
201 if (PERFETTO_UNLIKELY(size_with_header < size)) {
202 errno = EINVAL;
203 return result;
204 }
205
206 if (size_with_header > write_avail(pos)) {
207 meta_->stats.num_writes_overflow++;
208 errno = EAGAIN;
209 return result;
210 }
211
212 uint8_t* wr_ptr = at(pos.write_pos);
213
214 result.size = size;
215 result.data = wr_ptr + kHeaderSize;
216 result.bytes_free = write_avail(pos);
217 meta_->stats.bytes_written += size;
218 meta_->stats.num_writes_succeeded++;
219
220 // We can make this a relaxed store, as this gets picked up by the acquire
221 // load in GetPointerPositions (and the release store below).
222 reinterpret_cast<std::atomic<uint32_t>*>(wr_ptr)->store(
223 0, std::memory_order_relaxed);
224
225 // This needs to happen after the store above, so the reader never observes an
226 // incorrect byte count. This is matched by the acquire load in
227 // GetPointerPositions.
228 meta_->write_pos.fetch_add(size_with_header, std::memory_order_release);
229 return result;
230 }
231
EndWrite(Buffer buf)232 void SharedRingBuffer::EndWrite(Buffer buf) {
233 if (!buf)
234 return;
235 uint8_t* wr_ptr = buf.data - kHeaderSize;
236 PERFETTO_DCHECK(reinterpret_cast<uintptr_t>(wr_ptr) % kAlignment == 0);
237
238 // This needs to release to make sure the reader sees the payload written
239 // between the BeginWrite and EndWrite calls.
240 //
241 // This is matched by the acquire load in BeginRead where it reads the
242 // record's size.
243 reinterpret_cast<std::atomic<uint32_t>*>(wr_ptr)->store(
244 static_cast<uint32_t>(buf.size), std::memory_order_release);
245 }
246
BeginRead()247 SharedRingBuffer::Buffer SharedRingBuffer::BeginRead() {
248 std::optional<PointerPositions> opt_pos = GetPointerPositions();
249 if (!opt_pos) {
250 meta_->stats.num_reads_corrupt++;
251 errno = EBADF;
252 return Buffer();
253 }
254 auto pos = opt_pos.value();
255
256 size_t avail_read = read_avail(pos);
257
258 if (avail_read < kHeaderSize) {
259 meta_->stats.num_reads_nodata++;
260 errno = EAGAIN;
261 return Buffer(); // No data
262 }
263
264 uint8_t* rd_ptr = at(pos.read_pos);
265 PERFETTO_DCHECK(reinterpret_cast<uintptr_t>(rd_ptr) % kAlignment == 0);
266 const size_t size = reinterpret_cast<std::atomic<uint32_t>*>(rd_ptr)->load(
267 std::memory_order_acquire);
268 if (size == 0) {
269 meta_->stats.num_reads_nodata++;
270 errno = EAGAIN;
271 return Buffer();
272 }
273 const size_t size_with_header = base::AlignUp<kAlignment>(size + kHeaderSize);
274
275 if (size_with_header > avail_read) {
276 PERFETTO_ELOG(
277 "Corrupted header detected, size=%zu"
278 ", read_avail=%zu, rd=%" PRIu64 ", wr=%" PRIu64,
279 size, avail_read, pos.read_pos, pos.write_pos);
280 meta_->stats.num_reads_corrupt++;
281 errno = EBADF;
282 return Buffer();
283 }
284
285 rd_ptr += kHeaderSize;
286 PERFETTO_DCHECK(reinterpret_cast<uintptr_t>(rd_ptr) % kAlignment == 0);
287 return Buffer(rd_ptr, size, write_avail(pos));
288 }
289
EndRead(Buffer buf)290 size_t SharedRingBuffer::EndRead(Buffer buf) {
291 if (!buf)
292 return 0;
293 size_t size_with_header = base::AlignUp<kAlignment>(buf.size + kHeaderSize);
294 meta_->read_pos.fetch_add(size_with_header, std::memory_order_relaxed);
295 meta_->stats.num_reads_succeeded++;
296 return size_with_header;
297 }
298
IsCorrupt(const PointerPositions & pos)299 bool SharedRingBuffer::IsCorrupt(const PointerPositions& pos) {
300 if (pos.write_pos < pos.read_pos || pos.write_pos - pos.read_pos > size_ ||
301 pos.write_pos % kAlignment || pos.read_pos % kAlignment) {
302 PERFETTO_ELOG("Ring buffer corrupted, rd=%" PRIu64 ", wr=%" PRIu64
303 ", size=%zu",
304 pos.read_pos, pos.write_pos, size_);
305 return true;
306 }
307 return false;
308 }
309
SharedRingBuffer(SharedRingBuffer && other)310 SharedRingBuffer::SharedRingBuffer(SharedRingBuffer&& other) noexcept {
311 *this = std::move(other);
312 }
313
operator =(SharedRingBuffer && other)314 SharedRingBuffer& SharedRingBuffer::operator=(
315 SharedRingBuffer&& other) noexcept {
316 mem_fd_ = std::move(other.mem_fd_);
317 std::tie(meta_, mem_, size_, size_mask_) =
318 std::tie(other.meta_, other.mem_, other.size_, other.size_mask_);
319 std::tie(other.meta_, other.mem_, other.size_, other.size_mask_) =
320 std::make_tuple(nullptr, nullptr, 0, 0);
321 return *this;
322 }
323
324 // static
Create(size_t size)325 std::optional<SharedRingBuffer> SharedRingBuffer::Create(size_t size) {
326 auto buf = SharedRingBuffer(CreateFlag(), size);
327 if (!buf.is_valid())
328 return std::nullopt;
329 return std::make_optional(std::move(buf));
330 }
331
332 // static
Attach(base::ScopedFile mem_fd)333 std::optional<SharedRingBuffer> SharedRingBuffer::Attach(
334 base::ScopedFile mem_fd) {
335 auto buf = SharedRingBuffer(AttachFlag(), std::move(mem_fd));
336 if (!buf.is_valid())
337 return std::nullopt;
338 return std::make_optional(std::move(buf));
339 }
340
341 } // namespace profiling
342 } // namespace perfetto
343