1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 SiFive
4 */
5
6 #include <linux/spinlock.h>
7 #include <linux/mm.h>
8 #include <linux/memory.h>
9 #include <linux/uaccess.h>
10 #include <linux/stop_machine.h>
11 #include <asm/kprobes.h>
12 #include <asm/cacheflush.h>
13 #include <asm/fixmap.h>
14 #include <asm/ftrace.h>
15 #include <asm/patch.h>
16
17 struct patch_insn {
18 void *addr;
19 u32 insn;
20 atomic_t cpu_count;
21 };
22
23 int riscv_patch_in_stop_machine = false;
24
25 #ifdef CONFIG_MMU
patch_map(void * addr,int fixmap)26 static void *patch_map(void *addr, int fixmap)
27 {
28 uintptr_t uintaddr = (uintptr_t) addr;
29 struct page *page;
30
31 if (core_kernel_text(uintaddr))
32 page = phys_to_page(__pa_symbol(addr));
33 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
34 page = vmalloc_to_page(addr);
35 else
36 return addr;
37
38 BUG_ON(!page);
39
40 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
41 (uintaddr & ~PAGE_MASK));
42 }
43 NOKPROBE_SYMBOL(patch_map);
44
patch_unmap(int fixmap)45 static void patch_unmap(int fixmap)
46 {
47 clear_fixmap(fixmap);
48 }
49 NOKPROBE_SYMBOL(patch_unmap);
50
patch_insn_write(void * addr,const void * insn,size_t len)51 static int patch_insn_write(void *addr, const void *insn, size_t len)
52 {
53 void *waddr = addr;
54 bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
55 int ret;
56
57 /*
58 * Before reaching here, it was expected to lock the text_mutex
59 * already, so we don't need to give another lock here and could
60 * ensure that it was safe between each cores.
61 *
62 * We're currently using stop_machine() for ftrace & kprobes, and while
63 * that ensures text_mutex is held before installing the mappings it
64 * does not ensure text_mutex is held by the calling thread. That's
65 * safe but triggers a lockdep failure, so just elide it for that
66 * specific case.
67 */
68 if (!riscv_patch_in_stop_machine)
69 lockdep_assert_held(&text_mutex);
70
71 if (across_pages)
72 patch_map(addr + len, FIX_TEXT_POKE1);
73
74 waddr = patch_map(addr, FIX_TEXT_POKE0);
75
76 ret = copy_to_kernel_nofault(waddr, insn, len);
77
78 patch_unmap(FIX_TEXT_POKE0);
79
80 if (across_pages)
81 patch_unmap(FIX_TEXT_POKE1);
82
83 return ret;
84 }
85 NOKPROBE_SYMBOL(patch_insn_write);
86 #else
patch_insn_write(void * addr,const void * insn,size_t len)87 static int patch_insn_write(void *addr, const void *insn, size_t len)
88 {
89 return copy_to_kernel_nofault(addr, insn, len);
90 }
91 NOKPROBE_SYMBOL(patch_insn_write);
92 #endif /* CONFIG_MMU */
93
patch_text_nosync(void * addr,const void * insns,size_t len)94 int patch_text_nosync(void *addr, const void *insns, size_t len)
95 {
96 u32 *tp = addr;
97 int ret;
98
99 ret = patch_insn_write(tp, insns, len);
100
101 if (!ret)
102 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
103
104 return ret;
105 }
106 NOKPROBE_SYMBOL(patch_text_nosync);
107
patch_text_cb(void * data)108 static int patch_text_cb(void *data)
109 {
110 struct patch_insn *patch = data;
111 int ret = 0;
112
113 if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
114 ret =
115 patch_text_nosync(patch->addr, &patch->insn,
116 GET_INSN_LENGTH(patch->insn));
117 atomic_inc(&patch->cpu_count);
118 } else {
119 while (atomic_read(&patch->cpu_count) <= num_online_cpus())
120 cpu_relax();
121 smp_mb();
122 }
123
124 return ret;
125 }
126 NOKPROBE_SYMBOL(patch_text_cb);
127
patch_text(void * addr,u32 insn)128 int patch_text(void *addr, u32 insn)
129 {
130 int ret;
131 struct patch_insn patch = {
132 .addr = addr,
133 .insn = insn,
134 .cpu_count = ATOMIC_INIT(0),
135 };
136
137 /*
138 * kprobes takes text_mutex, before calling patch_text(), but as we call
139 * calls stop_machine(), the lockdep assertion in patch_insn_write()
140 * gets confused by the context in which the lock is taken.
141 * Instead, ensure the lock is held before calling stop_machine(), and
142 * set riscv_patch_in_stop_machine to skip the check in
143 * patch_insn_write().
144 */
145 lockdep_assert_held(&text_mutex);
146 riscv_patch_in_stop_machine = true;
147 ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask);
148 riscv_patch_in_stop_machine = false;
149 return ret;
150 }
151 NOKPROBE_SYMBOL(patch_text);
152