• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020 SiFive
4  */
5 
6 #include <linux/spinlock.h>
7 #include <linux/mm.h>
8 #include <linux/memory.h>
9 #include <linux/uaccess.h>
10 #include <linux/stop_machine.h>
11 #include <asm/kprobes.h>
12 #include <asm/cacheflush.h>
13 #include <asm/fixmap.h>
14 #include <asm/ftrace.h>
15 #include <asm/patch.h>
16 
17 struct patch_insn {
18 	void *addr;
19 	u32 insn;
20 	atomic_t cpu_count;
21 };
22 
23 int riscv_patch_in_stop_machine = false;
24 
25 #ifdef CONFIG_MMU
patch_map(void * addr,int fixmap)26 static void *patch_map(void *addr, int fixmap)
27 {
28 	uintptr_t uintaddr = (uintptr_t) addr;
29 	struct page *page;
30 
31 	if (core_kernel_text(uintaddr))
32 		page = phys_to_page(__pa_symbol(addr));
33 	else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
34 		page = vmalloc_to_page(addr);
35 	else
36 		return addr;
37 
38 	BUG_ON(!page);
39 
40 	return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
41 					 (uintaddr & ~PAGE_MASK));
42 }
43 NOKPROBE_SYMBOL(patch_map);
44 
patch_unmap(int fixmap)45 static void patch_unmap(int fixmap)
46 {
47 	clear_fixmap(fixmap);
48 }
49 NOKPROBE_SYMBOL(patch_unmap);
50 
patch_insn_write(void * addr,const void * insn,size_t len)51 static int patch_insn_write(void *addr, const void *insn, size_t len)
52 {
53 	void *waddr = addr;
54 	bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
55 	int ret;
56 
57 	/*
58 	 * Before reaching here, it was expected to lock the text_mutex
59 	 * already, so we don't need to give another lock here and could
60 	 * ensure that it was safe between each cores.
61 	 *
62 	 * We're currently using stop_machine() for ftrace & kprobes, and while
63 	 * that ensures text_mutex is held before installing the mappings it
64 	 * does not ensure text_mutex is held by the calling thread.  That's
65 	 * safe but triggers a lockdep failure, so just elide it for that
66 	 * specific case.
67 	 */
68 	if (!riscv_patch_in_stop_machine)
69 		lockdep_assert_held(&text_mutex);
70 
71 	if (across_pages)
72 		patch_map(addr + len, FIX_TEXT_POKE1);
73 
74 	waddr = patch_map(addr, FIX_TEXT_POKE0);
75 
76 	ret = copy_to_kernel_nofault(waddr, insn, len);
77 
78 	patch_unmap(FIX_TEXT_POKE0);
79 
80 	if (across_pages)
81 		patch_unmap(FIX_TEXT_POKE1);
82 
83 	return ret;
84 }
85 NOKPROBE_SYMBOL(patch_insn_write);
86 #else
patch_insn_write(void * addr,const void * insn,size_t len)87 static int patch_insn_write(void *addr, const void *insn, size_t len)
88 {
89 	return copy_to_kernel_nofault(addr, insn, len);
90 }
91 NOKPROBE_SYMBOL(patch_insn_write);
92 #endif /* CONFIG_MMU */
93 
patch_text_nosync(void * addr,const void * insns,size_t len)94 int patch_text_nosync(void *addr, const void *insns, size_t len)
95 {
96 	u32 *tp = addr;
97 	int ret;
98 
99 	ret = patch_insn_write(tp, insns, len);
100 
101 	if (!ret)
102 		flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
103 
104 	return ret;
105 }
106 NOKPROBE_SYMBOL(patch_text_nosync);
107 
patch_text_cb(void * data)108 static int patch_text_cb(void *data)
109 {
110 	struct patch_insn *patch = data;
111 	int ret = 0;
112 
113 	if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
114 		ret =
115 		    patch_text_nosync(patch->addr, &patch->insn,
116 					    GET_INSN_LENGTH(patch->insn));
117 		atomic_inc(&patch->cpu_count);
118 	} else {
119 		while (atomic_read(&patch->cpu_count) <= num_online_cpus())
120 			cpu_relax();
121 		smp_mb();
122 	}
123 
124 	return ret;
125 }
126 NOKPROBE_SYMBOL(patch_text_cb);
127 
patch_text(void * addr,u32 insn)128 int patch_text(void *addr, u32 insn)
129 {
130 	int ret;
131 	struct patch_insn patch = {
132 		.addr = addr,
133 		.insn = insn,
134 		.cpu_count = ATOMIC_INIT(0),
135 	};
136 
137 	/*
138 	 * kprobes takes text_mutex, before calling patch_text(), but as we call
139 	 * calls stop_machine(), the lockdep assertion in patch_insn_write()
140 	 * gets confused by the context in which the lock is taken.
141 	 * Instead, ensure the lock is held before calling stop_machine(), and
142 	 * set riscv_patch_in_stop_machine to skip the check in
143 	 * patch_insn_write().
144 	 */
145 	lockdep_assert_held(&text_mutex);
146 	riscv_patch_in_stop_machine = true;
147 	ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask);
148 	riscv_patch_in_stop_machine = false;
149 	return ret;
150 }
151 NOKPROBE_SYMBOL(patch_text);
152