1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_TRACE_RECURSION_H
3 #define _LINUX_TRACE_RECURSION_H
4
5 #include <linux/interrupt.h>
6 #include <linux/sched.h>
7
8 #ifdef CONFIG_TRACING
9
10 /* Only current can touch trace_recursion */
11
12 /*
13 * For function tracing recursion:
14 * The order of these bits are important.
15 *
16 * When function tracing occurs, the following steps are made:
17 * If arch does not support a ftrace feature:
18 * call internal function (uses INTERNAL bits) which calls...
19 * The function callback, which can use the FTRACE bits to
20 * check for recursion.
21 */
22 enum {
23 /* Function recursion bits */
24 TRACE_FTRACE_BIT,
25 TRACE_FTRACE_NMI_BIT,
26 TRACE_FTRACE_IRQ_BIT,
27 TRACE_FTRACE_SIRQ_BIT,
28 TRACE_FTRACE_TRANSITION_BIT,
29
30 /* Internal use recursion bits */
31 TRACE_INTERNAL_BIT,
32 TRACE_INTERNAL_NMI_BIT,
33 TRACE_INTERNAL_IRQ_BIT,
34 TRACE_INTERNAL_SIRQ_BIT,
35 TRACE_INTERNAL_TRANSITION_BIT,
36
37 TRACE_BRANCH_BIT,
38 /*
39 * Abuse of the trace_recursion.
40 * As we need a way to maintain state if we are tracing the function
41 * graph in irq because we want to trace a particular function that
42 * was called in irq context but we have irq tracing off. Since this
43 * can only be modified by current, we can reuse trace_recursion.
44 */
45 TRACE_IRQ_BIT,
46
47 /* Set if the function is in the set_graph_function file */
48 TRACE_GRAPH_BIT,
49
50 /*
51 * In the very unlikely case that an interrupt came in
52 * at a start of graph tracing, and we want to trace
53 * the function in that interrupt, the depth can be greater
54 * than zero, because of the preempted start of a previous
55 * trace. In an even more unlikely case, depth could be 2
56 * if a softirq interrupted the start of graph tracing,
57 * followed by an interrupt preempting a start of graph
58 * tracing in the softirq, and depth can even be 3
59 * if an NMI came in at the start of an interrupt function
60 * that preempted a softirq start of a function that
61 * preempted normal context!!!! Luckily, it can't be
62 * greater than 3, so the next two bits are a mask
63 * of what the depth is when we set TRACE_GRAPH_BIT
64 */
65
66 TRACE_GRAPH_DEPTH_START_BIT,
67 TRACE_GRAPH_DEPTH_END_BIT,
68
69 /*
70 * To implement set_graph_notrace, if this bit is set, we ignore
71 * function graph tracing of called functions, until the return
72 * function is called to clear it.
73 */
74 TRACE_GRAPH_NOTRACE_BIT,
75
76 /* Used to prevent recursion recording from recursing. */
77 TRACE_RECORD_RECURSION_BIT,
78 };
79
80 #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
81 #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
82 #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
83
84 #define trace_recursion_depth() \
85 (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
86 #define trace_recursion_set_depth(depth) \
87 do { \
88 current->trace_recursion &= \
89 ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
90 current->trace_recursion |= \
91 ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
92 } while (0)
93
94 #define TRACE_CONTEXT_BITS 4
95
96 #define TRACE_FTRACE_START TRACE_FTRACE_BIT
97
98 #define TRACE_LIST_START TRACE_INTERNAL_BIT
99
100 #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
101
102 /*
103 * Used for setting context
104 * NMI = 0
105 * IRQ = 1
106 * SOFTIRQ = 2
107 * NORMAL = 3
108 */
109 enum {
110 TRACE_CTX_NMI,
111 TRACE_CTX_IRQ,
112 TRACE_CTX_SOFTIRQ,
113 TRACE_CTX_NORMAL,
114 TRACE_CTX_TRANSITION,
115 };
116
trace_get_context_bit(void)117 static __always_inline int trace_get_context_bit(void)
118 {
119 unsigned char bit = interrupt_context_level();
120
121 return TRACE_CTX_NORMAL - bit;
122 }
123
124 #ifdef CONFIG_FTRACE_RECORD_RECURSION
125 extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
126 # define do_ftrace_record_recursion(ip, pip) \
127 do { \
128 if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
129 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
130 ftrace_record_recursion(ip, pip); \
131 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
132 } \
133 } while (0)
134 #else
135 # define do_ftrace_record_recursion(ip, pip) do { } while (0)
136 #endif
137
trace_test_and_set_recursion(unsigned long ip,unsigned long pip,int start)138 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
139 int start)
140 {
141 unsigned int val = READ_ONCE(current->trace_recursion);
142 int bit;
143
144 bit = trace_get_context_bit() + start;
145 if (unlikely(val & (1 << bit))) {
146 /*
147 * It could be that preempt_count has not been updated during
148 * a switch between contexts. Allow for a single recursion.
149 */
150 bit = TRACE_CTX_TRANSITION + start;
151 if (val & (1 << bit)) {
152 do_ftrace_record_recursion(ip, pip);
153 return -1;
154 }
155 }
156
157 val |= 1 << bit;
158 current->trace_recursion = val;
159 barrier();
160
161 return bit;
162 }
163
trace_clear_recursion(int bit)164 static __always_inline void trace_clear_recursion(int bit)
165 {
166 barrier();
167 trace_recursion_clear(bit);
168 }
169
170 /**
171 * ftrace_test_recursion_trylock - tests for recursion in same context
172 *
173 * Use this for ftrace callbacks. This will detect if the function
174 * tracing recursed in the same context (normal vs interrupt),
175 *
176 * Returns: -1 if a recursion happened.
177 * >= 0 if no recursion
178 */
ftrace_test_recursion_trylock(unsigned long ip,unsigned long parent_ip)179 static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
180 unsigned long parent_ip)
181 {
182 return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
183 }
184
185 /**
186 * ftrace_test_recursion_unlock - called when function callback is complete
187 * @bit: The return of a successful ftrace_test_recursion_trylock()
188 *
189 * This is used at the end of a ftrace callback.
190 */
ftrace_test_recursion_unlock(int bit)191 static __always_inline void ftrace_test_recursion_unlock(int bit)
192 {
193 trace_clear_recursion(bit);
194 }
195
196 #endif /* CONFIG_TRACING */
197 #endif /* _LINUX_TRACE_RECURSION_H */
198