1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_RING_BUFFER_H
3 #define _LINUX_RING_BUFFER_H
4
5 #include <linux/mm.h>
6 #include <linux/poll.h>
7 #include <linux/seq_file.h>
8
9 #include <asm/local.h>
10
11 #include <uapi/linux/trace_mmap.h>
12
13 struct trace_buffer;
14 struct ring_buffer_iter;
15
16 /*
17 * Don't refer to this struct directly, use functions below.
18 */
19 struct ring_buffer_event {
20 u32 type_len:5, time_delta:27;
21
22 u32 array[];
23 };
24
25 #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
26
27 /**
28 * enum ring_buffer_type - internal ring buffer types
29 *
30 * @RINGBUF_TYPE_PADDING: Left over page padding or discarded event
31 * If time_delta is 0:
32 * array is ignored
33 * size is variable depending on how much
34 * padding is needed
35 * If time_delta is non zero:
36 * array[0] holds the actual length
37 * size = 4 + length (bytes)
38 *
39 * @RINGBUF_TYPE_TIME_EXTEND: Extend the time delta
40 * array[0] = time delta (28 .. 59)
41 * size = 8 bytes
42 *
43 * @RINGBUF_TYPE_TIME_STAMP: Absolute timestamp
44 * Same format as TIME_EXTEND except that the
45 * value is an absolute timestamp, not a delta
46 * event.time_delta contains bottom 27 bits
47 * array[0] = top (28 .. 59) bits
48 * size = 8 bytes
49 *
50 * <= @RINGBUF_TYPE_DATA_TYPE_LEN_MAX:
51 * Data record
52 * If type_len is zero:
53 * array[0] holds the actual length
54 * array[1..(length+3)/4] holds data
55 * size = 4 + length (bytes)
56 * else
57 * length = type_len << 2
58 * array[0..(length+3)/4-1] holds data
59 * size = 4 + length (bytes)
60 */
61 enum ring_buffer_type {
62 RINGBUF_TYPE_DATA_TYPE_LEN_MAX = 28,
63 RINGBUF_TYPE_PADDING,
64 RINGBUF_TYPE_TIME_EXTEND,
65 RINGBUF_TYPE_TIME_STAMP,
66 };
67
68 #define TS_SHIFT 27
69 #define TS_MASK ((1ULL << TS_SHIFT) - 1)
70 #define TS_DELTA_TEST (~TS_MASK)
71
72 /*
73 * We need to fit the time_stamp delta into 27 bits.
74 */
test_time_stamp(u64 delta)75 static inline bool test_time_stamp(u64 delta)
76 {
77 return !!(delta & TS_DELTA_TEST);
78 }
79
80 unsigned ring_buffer_event_length(struct ring_buffer_event *event);
81 void *ring_buffer_event_data(struct ring_buffer_event *event);
82 u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer,
83 struct ring_buffer_event *event);
84
85 #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
86
87 /* Max payload is BUF_PAGE_SIZE - header (8bytes) */
88 #define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
89
90 #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
91
92 #define RB_ALIGNMENT 4U
93 #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
94 #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */
95
96 #ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS
97 # define RB_FORCE_8BYTE_ALIGNMENT 0
98 # define RB_ARCH_ALIGNMENT RB_ALIGNMENT
99 #else
100 # define RB_FORCE_8BYTE_ALIGNMENT 1
101 # define RB_ARCH_ALIGNMENT 8U
102 #endif
103
104 #define RB_ALIGN_DATA __aligned(RB_ARCH_ALIGNMENT)
105
106 struct buffer_data_page {
107 u64 time_stamp; /* page time stamp */
108 local_t commit; /* write committed index */
109 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
110 };
111
112 /*
113 * ring_buffer_discard_commit will remove an event that has not
114 * been committed yet. If this is used, then ring_buffer_unlock_commit
115 * must not be called on the discarded event. This function
116 * will try to remove the event from the ring buffer completely
117 * if another event has not been written after it.
118 *
119 * Example use:
120 *
121 * if (some_condition)
122 * ring_buffer_discard_commit(buffer, event);
123 * else
124 * ring_buffer_unlock_commit(buffer, event);
125 */
126 void ring_buffer_discard_commit(struct trace_buffer *buffer,
127 struct ring_buffer_event *event);
128
129 struct ring_buffer_writer;
130
131 /*
132 * size is in bytes for each per CPU buffer.
133 */
134 struct trace_buffer *
135 __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *key,
136 struct ring_buffer_writer *writer);
137
138 struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flags,
139 int order, unsigned long start,
140 unsigned long range_size,
141 struct lock_class_key *key);
142
143 bool ring_buffer_last_boot_delta(struct trace_buffer *buffer, long *text,
144 long *data);
145
146 /*
147 * Because the ring buffer is generic, if other users of the ring buffer get
148 * traced by ftrace, it can produce lockdep warnings. We need to keep each
149 * ring buffer's lock class separate.
150 */
151 #define ring_buffer_alloc(size, flags) \
152 ({ \
153 static struct lock_class_key __key; \
154 __ring_buffer_alloc((size), (flags), &__key, NULL); \
155 })
156
157 /*
158 * Because the ring buffer is generic, if other users of the ring buffer get
159 * traced by ftrace, it can produce lockdep warnings. We need to keep each
160 * ring buffer's lock class separate.
161 */
162 #define ring_buffer_alloc_range(size, flags, order, start, range_size) \
163 ({ \
164 static struct lock_class_key __key; \
165 __ring_buffer_alloc_range((size), (flags), (order), (start), \
166 (range_size), &__key); \
167 })
168
169 typedef bool (*ring_buffer_cond_fn)(void *data);
170 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full,
171 ring_buffer_cond_fn cond, void *data);
172 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,
173 struct file *filp, poll_table *poll_table, int full);
174 void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu);
175
176 #define RING_BUFFER_ALL_CPUS -1
177
178 void ring_buffer_free(struct trace_buffer *buffer);
179
180 int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, int cpu);
181
182 void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val);
183
184 struct ring_buffer_event *ring_buffer_lock_reserve(struct trace_buffer *buffer,
185 unsigned long length);
186 int ring_buffer_unlock_commit(struct trace_buffer *buffer);
187 int ring_buffer_write(struct trace_buffer *buffer,
188 unsigned long length, void *data);
189
190 void ring_buffer_nest_start(struct trace_buffer *buffer);
191 void ring_buffer_nest_end(struct trace_buffer *buffer);
192
193 struct ring_buffer_event *
194 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts,
195 unsigned long *lost_events);
196 struct ring_buffer_event *
197 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts,
198 unsigned long *lost_events);
199
200 struct ring_buffer_iter *
201 ring_buffer_read_start(struct trace_buffer *buffer, int cpu, gfp_t flags);
202 void ring_buffer_read_finish(struct ring_buffer_iter *iter);
203
204 struct ring_buffer_event *
205 ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts);
206 void ring_buffer_iter_advance(struct ring_buffer_iter *iter);
207 void ring_buffer_iter_reset(struct ring_buffer_iter *iter);
208 int ring_buffer_iter_empty(struct ring_buffer_iter *iter);
209 bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter);
210
211 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu);
212 unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer);
213
214 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu);
215 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer);
216 void ring_buffer_reset(struct trace_buffer *buffer);
217
218 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
219 int ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
220 struct trace_buffer *buffer_b, int cpu);
221 #else
222 static inline int
ring_buffer_swap_cpu(struct trace_buffer * buffer_a,struct trace_buffer * buffer_b,int cpu)223 ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
224 struct trace_buffer *buffer_b, int cpu)
225 {
226 return -ENODEV;
227 }
228 #endif
229
230 bool ring_buffer_empty(struct trace_buffer *buffer);
231 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu);
232
233 void ring_buffer_record_disable(struct trace_buffer *buffer);
234 void ring_buffer_record_enable(struct trace_buffer *buffer);
235 void ring_buffer_record_off(struct trace_buffer *buffer);
236 void ring_buffer_record_on(struct trace_buffer *buffer);
237 bool ring_buffer_record_is_on(struct trace_buffer *buffer);
238 bool ring_buffer_record_is_set_on(struct trace_buffer *buffer);
239 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu);
240 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu);
241
242 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu);
243 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu);
244 unsigned long ring_buffer_entries(struct trace_buffer *buffer);
245 unsigned long ring_buffer_overruns(struct trace_buffer *buffer);
246 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu);
247 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu);
248 unsigned long ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu);
249 unsigned long ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu);
250 unsigned long ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu);
251
252 u64 ring_buffer_time_stamp(struct trace_buffer *buffer);
253 void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer,
254 int cpu, u64 *ts);
255 void ring_buffer_set_clock(struct trace_buffer *buffer,
256 u64 (*clock)(void));
257 void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs);
258 bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer);
259
260 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu);
261
262 struct buffer_data_read_page;
263 struct buffer_data_read_page *
264 ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu);
265 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu,
266 struct buffer_data_read_page *page);
267 int ring_buffer_read_page(struct trace_buffer *buffer,
268 struct buffer_data_read_page *data_page,
269 size_t len, int cpu, int full);
270 void *ring_buffer_read_page_data(struct buffer_data_read_page *page);
271
272 struct trace_seq;
273
274 int ring_buffer_print_entry_header(struct trace_seq *s);
275 int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s);
276
277 int ring_buffer_subbuf_order_get(struct trace_buffer *buffer);
278 int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order);
279 int ring_buffer_subbuf_size_get(struct trace_buffer *buffer);
280
281 enum ring_buffer_flags {
282 RB_FL_OVERWRITE = 1 << 0,
283 };
284
285 #ifdef CONFIG_RING_BUFFER
286 int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node);
287 #else
288 #define trace_rb_cpu_prepare NULL
289 #endif
290
291 int ring_buffer_map(struct trace_buffer *buffer, int cpu,
292 struct vm_area_struct *vma);
293 int ring_buffer_unmap(struct trace_buffer *buffer, int cpu);
294 int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu);
295
296 #define meta_pages_lost(__meta) \
297 ((__meta)->Reserved1)
298 #define meta_pages_touched(__meta) \
299 ((__meta)->Reserved2)
300
301 struct rb_page_desc {
302 int cpu;
303 int nr_page_va; /* exclude the meta page */
304 unsigned long meta_va;
305 unsigned long page_va[];
306 };
307
308 struct trace_page_desc {
309 int nr_cpus;
310 char __data[]; /* list of rb_page_desc */
311 };
312
313 static inline
__next_rb_page_desc(struct rb_page_desc * pdesc)314 struct rb_page_desc *__next_rb_page_desc(struct rb_page_desc *pdesc)
315 {
316 size_t len = struct_size(pdesc, page_va, pdesc->nr_page_va);
317
318 return (struct rb_page_desc *)((void *)pdesc + len);
319 }
320
321 static inline
__first_rb_page_desc(struct trace_page_desc * trace_pdesc)322 struct rb_page_desc *__first_rb_page_desc(struct trace_page_desc *trace_pdesc)
323 {
324 return (struct rb_page_desc *)(&trace_pdesc->__data[0]);
325 }
326
327 #define for_each_rb_page_desc(__pdesc, __cpu, __trace_pdesc) \
328 for (__pdesc = __first_rb_page_desc(__trace_pdesc), __cpu = 0; \
329 __cpu < (__trace_pdesc)->nr_cpus; \
330 __cpu++, __pdesc = __next_rb_page_desc(__pdesc))
331
332 struct ring_buffer_writer {
333 struct trace_page_desc *pdesc;
334 int (*get_reader_page)(int cpu);
335 int (*reset)(int cpu);
336 };
337
338 int ring_buffer_poll_writer(struct trace_buffer *buffer, int cpu);
339
340 #define ring_buffer_reader(writer) \
341 ({ \
342 static struct lock_class_key __key; \
343 __ring_buffer_alloc(0, RB_FL_OVERWRITE, &__key, writer);\
344 })
345 #endif /* _LINUX_RING_BUFFER_H */
346