1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 SiFive
4 */
5
6 #include <linux/spinlock.h>
7 #include <linux/mm.h>
8 #include <linux/memory.h>
9 #include <linux/uaccess.h>
10 #include <linux/stop_machine.h>
11 #include <asm/kprobes.h>
12 #include <asm/cacheflush.h>
13 #include <asm/fixmap.h>
14 #include <asm/ftrace.h>
15 #include <asm/patch.h>
16 #include <asm/sections.h>
17
18 struct patch_insn {
19 void *addr;
20 u32 insn;
21 atomic_t cpu_count;
22 };
23
24 int riscv_patch_in_stop_machine = false;
25
26 #ifdef CONFIG_MMU
is_kernel_exittext(uintptr_t addr)27 static inline bool is_kernel_exittext(uintptr_t addr)
28 {
29 return system_state < SYSTEM_RUNNING &&
30 addr >= (uintptr_t)__exittext_begin &&
31 addr < (uintptr_t)__exittext_end;
32 }
33
patch_map(void * addr,int fixmap)34 static void *patch_map(void *addr, int fixmap)
35 {
36 uintptr_t uintaddr = (uintptr_t) addr;
37 struct page *page;
38
39 if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr))
40 page = phys_to_page(__pa_symbol(addr));
41 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
42 page = vmalloc_to_page(addr);
43 else
44 return addr;
45
46 BUG_ON(!page);
47
48 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
49 (uintaddr & ~PAGE_MASK));
50 }
51 NOKPROBE_SYMBOL(patch_map);
52
patch_unmap(int fixmap)53 static void patch_unmap(int fixmap)
54 {
55 clear_fixmap(fixmap);
56 }
57 NOKPROBE_SYMBOL(patch_unmap);
58
patch_insn_write(void * addr,const void * insn,size_t len)59 static int patch_insn_write(void *addr, const void *insn, size_t len)
60 {
61 void *waddr = addr;
62 bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
63 int ret;
64
65 /*
66 * Before reaching here, it was expected to lock the text_mutex
67 * already, so we don't need to give another lock here and could
68 * ensure that it was safe between each cores.
69 *
70 * We're currently using stop_machine() for ftrace & kprobes, and while
71 * that ensures text_mutex is held before installing the mappings it
72 * does not ensure text_mutex is held by the calling thread. That's
73 * safe but triggers a lockdep failure, so just elide it for that
74 * specific case.
75 */
76 if (!riscv_patch_in_stop_machine)
77 lockdep_assert_held(&text_mutex);
78
79 if (across_pages)
80 patch_map(addr + len, FIX_TEXT_POKE1);
81
82 waddr = patch_map(addr, FIX_TEXT_POKE0);
83
84 ret = copy_to_kernel_nofault(waddr, insn, len);
85
86 patch_unmap(FIX_TEXT_POKE0);
87
88 if (across_pages)
89 patch_unmap(FIX_TEXT_POKE1);
90
91 return ret;
92 }
93 NOKPROBE_SYMBOL(patch_insn_write);
94 #else
patch_insn_write(void * addr,const void * insn,size_t len)95 static int patch_insn_write(void *addr, const void *insn, size_t len)
96 {
97 return copy_to_kernel_nofault(addr, insn, len);
98 }
99 NOKPROBE_SYMBOL(patch_insn_write);
100 #endif /* CONFIG_MMU */
101
patch_text_nosync(void * addr,const void * insns,size_t len)102 int patch_text_nosync(void *addr, const void *insns, size_t len)
103 {
104 u32 *tp = addr;
105 int ret;
106
107 ret = patch_insn_write(tp, insns, len);
108
109 if (!ret)
110 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
111
112 return ret;
113 }
114 NOKPROBE_SYMBOL(patch_text_nosync);
115
patch_text_cb(void * data)116 static int patch_text_cb(void *data)
117 {
118 struct patch_insn *patch = data;
119 int ret = 0;
120
121 if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
122 ret =
123 patch_text_nosync(patch->addr, &patch->insn,
124 GET_INSN_LENGTH(patch->insn));
125 atomic_inc(&patch->cpu_count);
126 } else {
127 while (atomic_read(&patch->cpu_count) <= num_online_cpus())
128 cpu_relax();
129 smp_mb();
130 }
131
132 return ret;
133 }
134 NOKPROBE_SYMBOL(patch_text_cb);
135
patch_text(void * addr,u32 insn)136 int patch_text(void *addr, u32 insn)
137 {
138 int ret;
139 struct patch_insn patch = {
140 .addr = addr,
141 .insn = insn,
142 .cpu_count = ATOMIC_INIT(0),
143 };
144
145 /*
146 * kprobes takes text_mutex, before calling patch_text(), but as we call
147 * calls stop_machine(), the lockdep assertion in patch_insn_write()
148 * gets confused by the context in which the lock is taken.
149 * Instead, ensure the lock is held before calling stop_machine(), and
150 * set riscv_patch_in_stop_machine to skip the check in
151 * patch_insn_write().
152 */
153 lockdep_assert_held(&text_mutex);
154 riscv_patch_in_stop_machine = true;
155 ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask);
156 riscv_patch_in_stop_machine = false;
157 return ret;
158 }
159 NOKPROBE_SYMBOL(patch_text);
160