1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * preemptoff and irqoff tracepoints
4 *
5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
6 */
7
8 #include <linux/kallsyms.h>
9 #include <linux/uaccess.h>
10 #include <linux/module.h>
11 #include <linux/ftrace.h>
12 #include <linux/kprobes.h>
13 #include "trace.h"
14
15 #define CREATE_TRACE_POINTS
16 #include <trace/events/preemptirq.h>
17 #undef CREATE_TRACE_POINTS
18 #include <trace/hooks/preemptirq.h>
19
20 #ifdef CONFIG_TRACE_IRQFLAGS
21 /* Per-cpu variable to prevent redundant calls when IRQs already off */
22 static DEFINE_PER_CPU(int, tracing_irq_cpu);
23
24 /*
25 * Like trace_hardirqs_on() but without the lockdep invocation. This is
26 * used in the low level entry code where the ordering vs. RCU is important
27 * and lockdep uses a staged approach which splits the lockdep hardirq
28 * tracking into a RCU on and a RCU off section.
29 */
trace_hardirqs_on_prepare(void)30 void trace_hardirqs_on_prepare(void)
31 {
32 if (this_cpu_read(tracing_irq_cpu)) {
33 if (!in_nmi()) {
34 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
35 trace_android_rvh_irqs_enable(CALLER_ADDR0,
36 CALLER_ADDR1);
37 }
38 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
39 this_cpu_write(tracing_irq_cpu, 0);
40 }
41 }
42 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
43 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
44
trace_hardirqs_on(void)45 void trace_hardirqs_on(void)
46 {
47 if (this_cpu_read(tracing_irq_cpu)) {
48 if (!in_nmi()) {
49 trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
50 trace_android_rvh_irqs_enable(CALLER_ADDR0,
51 CALLER_ADDR1);
52 }
53 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
54 this_cpu_write(tracing_irq_cpu, 0);
55 }
56
57 lockdep_hardirqs_on_prepare();
58 lockdep_hardirqs_on(CALLER_ADDR0);
59 }
60 EXPORT_SYMBOL(trace_hardirqs_on);
61 NOKPROBE_SYMBOL(trace_hardirqs_on);
62
63 /*
64 * Like trace_hardirqs_off() but without the lockdep invocation. This is
65 * used in the low level entry code where the ordering vs. RCU is important
66 * and lockdep uses a staged approach which splits the lockdep hardirq
67 * tracking into a RCU on and a RCU off section.
68 */
trace_hardirqs_off_finish(void)69 void trace_hardirqs_off_finish(void)
70 {
71 if (!this_cpu_read(tracing_irq_cpu)) {
72 this_cpu_write(tracing_irq_cpu, 1);
73 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
74 if (!in_nmi()) {
75 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
76 trace_android_rvh_irqs_disable(CALLER_ADDR0,
77 CALLER_ADDR1);
78 }
79 }
80
81 }
82 EXPORT_SYMBOL(trace_hardirqs_off_finish);
83 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
84
trace_hardirqs_off(void)85 void trace_hardirqs_off(void)
86 {
87 lockdep_hardirqs_off(CALLER_ADDR0);
88
89 if (!this_cpu_read(tracing_irq_cpu)) {
90 this_cpu_write(tracing_irq_cpu, 1);
91 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
92 if (!in_nmi()) {
93 trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
94 trace_android_rvh_irqs_disable(CALLER_ADDR0,
95 CALLER_ADDR1);
96 }
97 }
98 }
99 EXPORT_SYMBOL(trace_hardirqs_off);
100 NOKPROBE_SYMBOL(trace_hardirqs_off);
101
trace_hardirqs_on_caller(unsigned long caller_addr)102 __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
103 {
104 if (this_cpu_read(tracing_irq_cpu)) {
105 if (!in_nmi()) {
106 trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
107 trace_android_rvh_irqs_enable(CALLER_ADDR0,
108 caller_addr);
109 }
110 tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
111 this_cpu_write(tracing_irq_cpu, 0);
112 }
113
114 lockdep_hardirqs_on_prepare();
115 lockdep_hardirqs_on(caller_addr);
116 }
117 EXPORT_SYMBOL(trace_hardirqs_on_caller);
118 NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
119
trace_hardirqs_off_caller(unsigned long caller_addr)120 __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
121 {
122 lockdep_hardirqs_off(caller_addr);
123
124 if (!this_cpu_read(tracing_irq_cpu)) {
125 this_cpu_write(tracing_irq_cpu, 1);
126 tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
127 if (!in_nmi()) {
128 trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
129 trace_android_rvh_irqs_enable(CALLER_ADDR0,
130 caller_addr);
131 }
132 }
133 }
134 EXPORT_SYMBOL(trace_hardirqs_off_caller);
135 NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
136 #endif /* CONFIG_TRACE_IRQFLAGS */
137
138 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
139
trace_preempt_on(unsigned long a0,unsigned long a1)140 void trace_preempt_on(unsigned long a0, unsigned long a1)
141 {
142 if (!in_nmi()) {
143 trace_preempt_enable_rcuidle(a0, a1);
144 trace_android_rvh_preempt_enable(a0, a1);
145 }
146 tracer_preempt_on(a0, a1);
147 }
148
trace_preempt_off(unsigned long a0,unsigned long a1)149 void trace_preempt_off(unsigned long a0, unsigned long a1)
150 {
151 if (!in_nmi()) {
152 trace_preempt_disable_rcuidle(a0, a1);
153 trace_android_rvh_preempt_disable(a0, a1);
154 }
155 tracer_preempt_off(a0, a1);
156 }
157 #endif
158