1 /*
2 * Performance events callchain code, extracted from core.c:
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
7 * Copyright � 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 *
9 * For licensing details see kernel-base/COPYING
10 */
11
12 #include <linux/perf_event.h>
13 #include <linux/slab.h>
14 #include "internal.h"
15
16 struct callchain_cpus_entries {
17 struct rcu_head rcu_head;
18 struct perf_callchain_entry *cpu_entries[0];
19 };
20
21 int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
22 int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
23
perf_callchain_entry__sizeof(void)24 static inline size_t perf_callchain_entry__sizeof(void)
25 {
26 return (sizeof(struct perf_callchain_entry) +
27 sizeof(__u64) * (sysctl_perf_event_max_stack +
28 sysctl_perf_event_max_contexts_per_stack));
29 }
30
31 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
32 static atomic_t nr_callchain_events;
33 static DEFINE_MUTEX(callchain_mutex);
34 static struct callchain_cpus_entries *callchain_cpus_entries;
35
36
perf_callchain_kernel(struct perf_callchain_entry_ctx * entry,struct pt_regs * regs)37 __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
38 struct pt_regs *regs)
39 {
40 }
41
perf_callchain_user(struct perf_callchain_entry_ctx * entry,struct pt_regs * regs)42 __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
43 struct pt_regs *regs)
44 {
45 }
46
release_callchain_buffers_rcu(struct rcu_head * head)47 static void release_callchain_buffers_rcu(struct rcu_head *head)
48 {
49 struct callchain_cpus_entries *entries;
50 int cpu;
51
52 entries = container_of(head, struct callchain_cpus_entries, rcu_head);
53
54 for_each_possible_cpu(cpu)
55 kfree(entries->cpu_entries[cpu]);
56
57 kfree(entries);
58 }
59
release_callchain_buffers(void)60 static void release_callchain_buffers(void)
61 {
62 struct callchain_cpus_entries *entries;
63
64 entries = callchain_cpus_entries;
65 RCU_INIT_POINTER(callchain_cpus_entries, NULL);
66 call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
67 }
68
alloc_callchain_buffers(void)69 static int alloc_callchain_buffers(void)
70 {
71 int cpu;
72 int size;
73 struct callchain_cpus_entries *entries;
74
75 /*
76 * We can't use the percpu allocation API for data that can be
77 * accessed from NMI. Use a temporary manual per cpu allocation
78 * until that gets sorted out.
79 */
80 size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
81
82 entries = kzalloc(size, GFP_KERNEL);
83 if (!entries)
84 return -ENOMEM;
85
86 size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
87
88 for_each_possible_cpu(cpu) {
89 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
90 cpu_to_node(cpu));
91 if (!entries->cpu_entries[cpu])
92 goto fail;
93 }
94
95 rcu_assign_pointer(callchain_cpus_entries, entries);
96
97 return 0;
98
99 fail:
100 for_each_possible_cpu(cpu)
101 kfree(entries->cpu_entries[cpu]);
102 kfree(entries);
103
104 return -ENOMEM;
105 }
106
get_callchain_buffers(int event_max_stack)107 int get_callchain_buffers(int event_max_stack)
108 {
109 int err = 0;
110 int count;
111
112 mutex_lock(&callchain_mutex);
113
114 count = atomic_inc_return(&nr_callchain_events);
115 if (WARN_ON_ONCE(count < 1)) {
116 err = -EINVAL;
117 goto exit;
118 }
119
120 /*
121 * If requesting per event more than the global cap,
122 * return a different error to help userspace figure
123 * this out.
124 *
125 * And also do it here so that we have &callchain_mutex held.
126 */
127 if (event_max_stack > sysctl_perf_event_max_stack) {
128 err = -EOVERFLOW;
129 goto exit;
130 }
131
132 if (count > 1) {
133 /* If the allocation failed, give up */
134 if (!callchain_cpus_entries)
135 err = -ENOMEM;
136 goto exit;
137 }
138
139 err = alloc_callchain_buffers();
140 exit:
141 if (err)
142 atomic_dec(&nr_callchain_events);
143
144 mutex_unlock(&callchain_mutex);
145
146 return err;
147 }
148
put_callchain_buffers(void)149 void put_callchain_buffers(void)
150 {
151 if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
152 release_callchain_buffers();
153 mutex_unlock(&callchain_mutex);
154 }
155 }
156
get_callchain_entry(int * rctx)157 static struct perf_callchain_entry *get_callchain_entry(int *rctx)
158 {
159 int cpu;
160 struct callchain_cpus_entries *entries;
161
162 *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
163 if (*rctx == -1)
164 return NULL;
165
166 entries = rcu_dereference(callchain_cpus_entries);
167 if (!entries)
168 return NULL;
169
170 cpu = smp_processor_id();
171
172 return (((void *)entries->cpu_entries[cpu]) +
173 (*rctx * perf_callchain_entry__sizeof()));
174 }
175
176 static void
put_callchain_entry(int rctx)177 put_callchain_entry(int rctx)
178 {
179 put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
180 }
181
182 struct perf_callchain_entry *
perf_callchain(struct perf_event * event,struct pt_regs * regs)183 perf_callchain(struct perf_event *event, struct pt_regs *regs)
184 {
185 bool kernel = !event->attr.exclude_callchain_kernel;
186 bool user = !event->attr.exclude_callchain_user;
187 /* Disallow cross-task user callchains. */
188 bool crosstask = event->ctx->task && event->ctx->task != current;
189 const u32 max_stack = event->attr.sample_max_stack;
190
191 if (!kernel && !user)
192 return NULL;
193
194 return get_perf_callchain(regs, 0, kernel, user, max_stack, crosstask, true);
195 }
196
197 struct perf_callchain_entry *
get_perf_callchain(struct pt_regs * regs,u32 init_nr,bool kernel,bool user,u32 max_stack,bool crosstask,bool add_mark)198 get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
199 u32 max_stack, bool crosstask, bool add_mark)
200 {
201 struct perf_callchain_entry *entry;
202 struct perf_callchain_entry_ctx ctx;
203 int rctx;
204
205 entry = get_callchain_entry(&rctx);
206 if (rctx == -1)
207 return NULL;
208
209 if (!entry)
210 goto exit_put;
211
212 ctx.entry = entry;
213 ctx.max_stack = max_stack;
214 ctx.nr = entry->nr = init_nr;
215 ctx.contexts = 0;
216 ctx.contexts_maxed = false;
217
218 if (kernel && !user_mode(regs)) {
219 if (add_mark)
220 perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
221 perf_callchain_kernel(&ctx, regs);
222 }
223
224 if (user) {
225 if (!user_mode(regs)) {
226 if (current->mm)
227 regs = task_pt_regs(current);
228 else
229 regs = NULL;
230 }
231
232 if (regs) {
233 mm_segment_t fs;
234
235 if (crosstask)
236 goto exit_put;
237
238 if (add_mark)
239 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
240
241 fs = get_fs();
242 set_fs(USER_DS);
243 perf_callchain_user(&ctx, regs);
244 set_fs(fs);
245 }
246 }
247
248 exit_put:
249 put_callchain_entry(rctx);
250
251 return entry;
252 }
253
254 /*
255 * Used for sysctl_perf_event_max_stack and
256 * sysctl_perf_event_max_contexts_per_stack.
257 */
perf_event_max_stack_handler(struct ctl_table * table,int write,void __user * buffer,size_t * lenp,loff_t * ppos)258 int perf_event_max_stack_handler(struct ctl_table *table, int write,
259 void __user *buffer, size_t *lenp, loff_t *ppos)
260 {
261 int *value = table->data;
262 int new_value = *value, ret;
263 struct ctl_table new_table = *table;
264
265 new_table.data = &new_value;
266 ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
267 if (ret || !write)
268 return ret;
269
270 mutex_lock(&callchain_mutex);
271 if (atomic_read(&nr_callchain_events))
272 ret = -EBUSY;
273 else
274 *value = new_value;
275
276 mutex_unlock(&callchain_mutex);
277
278 return ret;
279 }
280