1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * preemptoff and irqoff tracepoints
4 *
5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
6 */
7
8 #include <linux/kallsyms.h>
9 #include <linux/uaccess.h>
10 #include <linux/module.h>
11 #include <linux/ftrace.h>
12 #include <linux/kprobes.h>
13 #include "trace.h"
14
15 #define CREATE_TRACE_POINTS
16 #include <trace/events/preemptirq.h>
17 #undef CREATE_TRACE_POINTS
18 #include <trace/hooks/preemptirq.h>
19
20 /*
21 * Use regular trace points on architectures that implement noinstr
22 * tooling: these calls will only happen with RCU enabled, which can
23 * use a regular tracepoint.
24 *
25 * On older architectures, use the rcuidle tracing methods (which
26 * aren't NMI-safe - so exclude NMI contexts):
27 */
28 #ifdef CONFIG_ARCH_WANTS_NO_INSTR
29 #define trace(point) trace_##point
30 #else
31 #define trace(point) if (!in_nmi()) trace_##point##_rcuidle
32 #endif
33
34 #ifdef CONFIG_TRACE_IRQFLAGS
35 /* Per-cpu variable to prevent redundant calls when IRQs already off */
36 static DEFINE_PER_CPU(int, tracing_irq_cpu);
37
38 /*
39 * Like trace_hardirqs_on() but without the lockdep invocation. This is
40 * used in the low level entry code where the ordering vs. RCU is important
41 * and lockdep uses a staged approach which splits the lockdep hardirq
42 * tracking into a RCU on and a RCU off section.
43 */
trace_hardirqs_on_prepare(void)44 void trace_hardirqs_on_prepare(void)
45 {
46 if (this_cpu_read(tracing_irq_cpu)) {
47 trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1);
48 if (!in_nmi())
49 trace_android_rvh_irqs_enable(CALLER_ADDR0,
50 CALLER_ADDR1);
51 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
52 this_cpu_write(tracing_irq_cpu, 0);
53 }
54 }
55 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
56 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
57
trace_hardirqs_on(void)58 void trace_hardirqs_on(void)
59 {
60 if (this_cpu_read(tracing_irq_cpu)) {
61 trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1);
62 if (!in_nmi())
63 trace_android_rvh_irqs_enable(CALLER_ADDR0,
64 CALLER_ADDR1);
65 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
66 this_cpu_write(tracing_irq_cpu, 0);
67 }
68
69 lockdep_hardirqs_on_prepare();
70 lockdep_hardirqs_on(CALLER_ADDR0);
71 }
72 EXPORT_SYMBOL(trace_hardirqs_on);
73 NOKPROBE_SYMBOL(trace_hardirqs_on);
74
75 /*
76 * Like trace_hardirqs_off() but without the lockdep invocation. This is
77 * used in the low level entry code where the ordering vs. RCU is important
78 * and lockdep uses a staged approach which splits the lockdep hardirq
79 * tracking into a RCU on and a RCU off section.
80 */
trace_hardirqs_off_finish(void)81 void trace_hardirqs_off_finish(void)
82 {
83 if (!this_cpu_read(tracing_irq_cpu)) {
84 this_cpu_write(tracing_irq_cpu, 1);
85 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
86 trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1);
87 if (!in_nmi())
88 trace_android_rvh_irqs_disable(CALLER_ADDR0,
89 CALLER_ADDR1);
90 }
91
92 }
93 EXPORT_SYMBOL(trace_hardirqs_off_finish);
94 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
95
trace_hardirqs_off(void)96 void trace_hardirqs_off(void)
97 {
98 lockdep_hardirqs_off(CALLER_ADDR0);
99
100 if (!this_cpu_read(tracing_irq_cpu)) {
101 this_cpu_write(tracing_irq_cpu, 1);
102 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
103 trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1);
104 if (!in_nmi())
105 trace_android_rvh_irqs_disable(CALLER_ADDR0,
106 CALLER_ADDR1);
107 }
108 }
109 EXPORT_SYMBOL(trace_hardirqs_off);
110 NOKPROBE_SYMBOL(trace_hardirqs_off);
111 #endif /* CONFIG_TRACE_IRQFLAGS */
112
113 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
114
trace_preempt_on(unsigned long a0,unsigned long a1)115 void trace_preempt_on(unsigned long a0, unsigned long a1)
116 {
117 trace(preempt_enable)(a0, a1);
118 if (!in_nmi())
119 trace_android_rvh_preempt_enable(a0, a1);
120 tracer_preempt_on(a0, a1);
121 }
122
trace_preempt_off(unsigned long a0,unsigned long a1)123 void trace_preempt_off(unsigned long a0, unsigned long a1)
124 {
125 trace(preempt_disable)(a0, a1);
126 if (!in_nmi())
127 trace_android_rvh_preempt_disable(a0, a1);
128 tracer_preempt_off(a0, a1);
129 }
130 #endif
131