1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #pragma once
18
19 #include <android-base/result.h>
20 #include <android-base/unique_fd.h>
21 #include <linux/bpf.h>
22 #include <sys/mman.h>
23 #include <utils/Log.h>
24
25 #include "bpf/BpfUtils.h"
26
27 namespace android {
28 namespace bpf {
29
30 // BpfRingbufBase contains the non-templated functionality of BPF ring buffers.
31 class BpfRingbufBase {
32 public:
~BpfRingbufBase()33 ~BpfRingbufBase() {
34 if (mConsumerPos) munmap(mConsumerPos, mConsumerSize);
35 if (mProducerPos) munmap(mProducerPos, mProducerSize);
36 mConsumerPos = nullptr;
37 mProducerPos = nullptr;
38 }
39
40 protected:
41 // Non-initializing constructor, used by Create.
BpfRingbufBase(size_t value_size)42 BpfRingbufBase(size_t value_size) : mValueSize(value_size) {}
43
44 // Full construction that aborts on error (use Create/Init to handle errors).
BpfRingbufBase(const char * path,size_t value_size)45 BpfRingbufBase(const char* path, size_t value_size) : mValueSize(value_size) {
46 if (auto status = Init(path); !status.ok()) {
47 ALOGE("BpfRingbuf init failed: %s", status.error().message().c_str());
48 abort();
49 }
50 }
51
52 // Delete copy constructor (class owns raw pointers).
53 BpfRingbufBase(const BpfRingbufBase&) = delete;
54
55 // Initialize the base ringbuffer components. Must be called exactly once.
56 base::Result<void> Init(const char* path);
57
58 // Consumes all messages from the ring buffer, passing them to the callback.
59 base::Result<int> ConsumeAll(
60 const std::function<void(const void*)>& callback);
61
62 // Replicates c-style void* "byte-wise" pointer addition.
63 template <typename Ptr>
pointerAddBytes(void * base,ssize_t offset_bytes)64 static Ptr pointerAddBytes(void* base, ssize_t offset_bytes) {
65 return reinterpret_cast<Ptr>(reinterpret_cast<char*>(base) + offset_bytes);
66 }
67
68 // Rounds len by clearing bitmask, adding header, and aligning to 8 bytes.
roundLength(uint32_t len)69 static uint32_t roundLength(uint32_t len) {
70 len &= ~(BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT);
71 len += BPF_RINGBUF_HDR_SZ;
72 return (len + 7) & ~7;
73 }
74
75 const size_t mValueSize;
76
77 size_t mConsumerSize;
78 size_t mProducerSize;
79 unsigned long mPosMask;
80 android::base::unique_fd mRingFd;
81
82 void* mDataPos = nullptr;
83 unsigned long* mConsumerPos = nullptr;
84 unsigned long* mProducerPos = nullptr;
85 };
86
87 // This is a class wrapper for eBPF ring buffers. An eBPF ring buffer is a
88 // special type of eBPF map used for sending messages from eBPF to userspace.
89 // The implementation relies on fast shared memory and atomics for the producer
90 // and consumer management. Ring buffers are a faster alternative to eBPF perf
91 // buffers.
92 //
93 // This class is thread compatible, but not thread safe.
94 //
95 // Note: A kernel eBPF ring buffer may be accessed by both kernel and userspace
96 // processes at the same time. However, the userspace consumers of a given ring
97 // buffer all share a single read pointer. There is no guarantee which readers
98 // will read which messages.
99 template <typename Value>
100 class BpfRingbuf : public BpfRingbufBase {
101 public:
102 using MessageCallback = std::function<void(const Value&)>;
103
104 // Creates a ringbuffer wrapper from a pinned path. This initialization will
105 // abort on error. To handle errors, initialize with Create instead.
BpfRingbuf(const char * path)106 BpfRingbuf(const char* path) : BpfRingbufBase(path, sizeof(Value)) {}
107
108 // Creates a ringbuffer wrapper from a pinned path. There are no guarantees
109 // that the ringbuf outputs messaged of type `Value`, only that they are the
110 // same size. Size is only checked in ConsumeAll.
111 static base::Result<std::unique_ptr<BpfRingbuf<Value>>> Create(
112 const char* path);
113
114 // Consumes all messages from the ring buffer, passing them to the callback.
115 // Returns the number of messages consumed or a non-ok result on error. If the
116 // ring buffer has no pending messages an OK result with count 0 is returned.
117 base::Result<int> ConsumeAll(const MessageCallback& callback);
118
119 private:
120 // Empty ctor for use by Create.
BpfRingbuf()121 BpfRingbuf() : BpfRingbufBase(sizeof(Value)) {}
122 };
123
124 #define ACCESS_ONCE(x) (*(volatile typeof(x)*)&(x))
125
126 #if defined(__i386__) || defined(__x86_64__)
127 #define smp_sync() asm volatile("" ::: "memory")
128 #elif defined(__aarch64__)
129 #define smp_sync() asm volatile("dmb ish" ::: "memory")
130 #else
131 #define smp_sync() __sync_synchronize()
132 #endif
133
134 #define smp_store_release(p, v) \
135 do { \
136 smp_sync(); \
137 ACCESS_ONCE(*(p)) = (v); \
138 } while (0)
139
140 #define smp_load_acquire(p) \
141 ({ \
142 auto ___p = ACCESS_ONCE(*(p)); \
143 smp_sync(); \
144 ___p; \
145 })
146
Init(const char * path)147 inline base::Result<void> BpfRingbufBase::Init(const char* path) {
148 if (sizeof(unsigned long) != 8) {
149 return android::base::Error()
150 << "BpfRingbuf does not support 32 bit architectures";
151 }
152 mRingFd.reset(mapRetrieveRW(path));
153 if (!mRingFd.ok()) {
154 return android::base::ErrnoError()
155 << "failed to retrieve ringbuffer at " << path;
156 }
157
158 int map_type = android::bpf::bpfGetFdMapType(mRingFd);
159 if (map_type != BPF_MAP_TYPE_RINGBUF) {
160 errno = EINVAL;
161 return android::base::ErrnoError()
162 << "bpf map has wrong type: want BPF_MAP_TYPE_RINGBUF ("
163 << BPF_MAP_TYPE_RINGBUF << ") got " << map_type;
164 }
165
166 int max_entries = android::bpf::bpfGetFdMaxEntries(mRingFd);
167 if (max_entries < 0) {
168 return android::base::ErrnoError()
169 << "failed to read max_entries from ringbuf";
170 }
171 if (max_entries == 0) {
172 errno = EINVAL;
173 return android::base::ErrnoError() << "max_entries must be non-zero";
174 }
175
176 mPosMask = max_entries - 1;
177 mConsumerSize = getpagesize();
178 mProducerSize = getpagesize() + 2 * max_entries;
179
180 {
181 void* ptr = mmap(NULL, mConsumerSize, PROT_READ | PROT_WRITE, MAP_SHARED,
182 mRingFd, 0);
183 if (ptr == MAP_FAILED) {
184 return android::base::ErrnoError()
185 << "failed to mmap ringbuf consumer pages";
186 }
187 mConsumerPos = reinterpret_cast<unsigned long*>(ptr);
188 }
189
190 {
191 void* ptr = mmap(NULL, mProducerSize, PROT_READ, MAP_SHARED, mRingFd,
192 mConsumerSize);
193 if (ptr == MAP_FAILED) {
194 return android::base::ErrnoError()
195 << "failed to mmap ringbuf producer page";
196 }
197 mProducerPos = reinterpret_cast<unsigned long*>(ptr);
198 }
199
200 mDataPos = pointerAddBytes<void*>(mProducerPos, getpagesize());
201 return {};
202 }
203
ConsumeAll(const std::function<void (const void *)> & callback)204 inline base::Result<int> BpfRingbufBase::ConsumeAll(
205 const std::function<void(const void*)>& callback) {
206 int64_t count = 0;
207 unsigned long cons_pos = smp_load_acquire(mConsumerPos);
208 unsigned long prod_pos = smp_load_acquire(mProducerPos);
209 while (cons_pos < prod_pos) {
210 // Find the start of the entry for this read (wrapping is done here).
211 void* start_ptr = pointerAddBytes<void*>(mDataPos, cons_pos & mPosMask);
212
213 // The entry has an 8 byte header containing the sample length.
214 uint32_t length = smp_load_acquire(reinterpret_cast<uint32_t*>(start_ptr));
215
216 // If the sample isn't committed, we're caught up with the producer.
217 if (length & BPF_RINGBUF_BUSY_BIT) return count;
218
219 cons_pos += roundLength(length);
220
221 if ((length & BPF_RINGBUF_DISCARD_BIT) == 0) {
222 if (length != mValueSize) {
223 smp_store_release(mConsumerPos, cons_pos);
224 errno = EMSGSIZE;
225 return android::base::ErrnoError()
226 << "BPF ring buffer message has unexpected size (want "
227 << mValueSize << " bytes, got " << length << " bytes)";
228 }
229 callback(pointerAddBytes<const void*>(start_ptr, BPF_RINGBUF_HDR_SZ));
230 count++;
231 }
232
233 smp_store_release(mConsumerPos, cons_pos);
234 }
235
236 return count;
237 }
238
239 template <typename Value>
240 inline base::Result<std::unique_ptr<BpfRingbuf<Value>>>
Create(const char * path)241 BpfRingbuf<Value>::Create(const char* path) {
242 auto rb = std::unique_ptr<BpfRingbuf>(new BpfRingbuf);
243 if (auto status = rb->Init(path); !status.ok()) return status.error();
244 return rb;
245 }
246
247 template <typename Value>
ConsumeAll(const MessageCallback & callback)248 inline base::Result<int> BpfRingbuf<Value>::ConsumeAll(
249 const MessageCallback& callback) {
250 return BpfRingbufBase::ConsumeAll([&](const void* value) {
251 callback(*reinterpret_cast<const Value*>(value));
252 });
253 }
254
255 #undef ACCESS_ONCE
256 #undef smp_sync
257 #undef smp_store_release
258 #undef smp_load_acquire
259
260 } // namespace bpf
261 } // namespace android
262