1 /*
2 * Copyright (C) 2013 Huawei Ltd.
3 * Author: Jiang Liu <liuj97@gmail.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17 #include <linux/bitops.h>
18 #include <linux/compiler.h>
19 #include <linux/kernel.h>
20 #include <linux/smp.h>
21 #include <linux/stop_machine.h>
22 #include <linux/uaccess.h>
23 #include <asm/cacheflush.h>
24 #include <asm/insn.h>
25
26 static int aarch64_insn_encoding_class[] = {
27 AARCH64_INSN_CLS_UNKNOWN,
28 AARCH64_INSN_CLS_UNKNOWN,
29 AARCH64_INSN_CLS_UNKNOWN,
30 AARCH64_INSN_CLS_UNKNOWN,
31 AARCH64_INSN_CLS_LDST,
32 AARCH64_INSN_CLS_DP_REG,
33 AARCH64_INSN_CLS_LDST,
34 AARCH64_INSN_CLS_DP_FPSIMD,
35 AARCH64_INSN_CLS_DP_IMM,
36 AARCH64_INSN_CLS_DP_IMM,
37 AARCH64_INSN_CLS_BR_SYS,
38 AARCH64_INSN_CLS_BR_SYS,
39 AARCH64_INSN_CLS_LDST,
40 AARCH64_INSN_CLS_DP_REG,
41 AARCH64_INSN_CLS_LDST,
42 AARCH64_INSN_CLS_DP_FPSIMD,
43 };
44
aarch64_get_insn_class(u32 insn)45 enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
46 {
47 return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
48 }
49
50 /* NOP is an alias of HINT */
aarch64_insn_is_nop(u32 insn)51 bool __kprobes aarch64_insn_is_nop(u32 insn)
52 {
53 if (!aarch64_insn_is_hint(insn))
54 return false;
55
56 switch (insn & 0xFE0) {
57 case AARCH64_INSN_HINT_YIELD:
58 case AARCH64_INSN_HINT_WFE:
59 case AARCH64_INSN_HINT_WFI:
60 case AARCH64_INSN_HINT_SEV:
61 case AARCH64_INSN_HINT_SEVL:
62 return false;
63 default:
64 return true;
65 }
66 }
67
68 /*
69 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
70 * little-endian.
71 */
aarch64_insn_read(void * addr,u32 * insnp)72 int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
73 {
74 int ret;
75 u32 val;
76
77 ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE);
78 if (!ret)
79 *insnp = le32_to_cpu(val);
80
81 return ret;
82 }
83
aarch64_insn_write(void * addr,u32 insn)84 int __kprobes aarch64_insn_write(void *addr, u32 insn)
85 {
86 insn = cpu_to_le32(insn);
87 return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
88 }
89
__aarch64_insn_hotpatch_safe(u32 insn)90 static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
91 {
92 if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
93 return false;
94
95 return aarch64_insn_is_b(insn) ||
96 aarch64_insn_is_bl(insn) ||
97 aarch64_insn_is_svc(insn) ||
98 aarch64_insn_is_hvc(insn) ||
99 aarch64_insn_is_smc(insn) ||
100 aarch64_insn_is_brk(insn) ||
101 aarch64_insn_is_nop(insn);
102 }
103
104 /*
105 * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
106 * Section B2.6.5 "Concurrent modification and execution of instructions":
107 * Concurrent modification and execution of instructions can lead to the
108 * resulting instruction performing any behavior that can be achieved by
109 * executing any sequence of instructions that can be executed from the
110 * same Exception level, except where the instruction before modification
111 * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
112 * or SMC instruction.
113 */
aarch64_insn_hotpatch_safe(u32 old_insn,u32 new_insn)114 bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
115 {
116 return __aarch64_insn_hotpatch_safe(old_insn) &&
117 __aarch64_insn_hotpatch_safe(new_insn);
118 }
119
aarch64_insn_patch_text_nosync(void * addr,u32 insn)120 int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
121 {
122 u32 *tp = addr;
123 int ret;
124
125 /* A64 instructions must be word aligned */
126 if ((uintptr_t)tp & 0x3)
127 return -EINVAL;
128
129 ret = aarch64_insn_write(tp, insn);
130 if (ret == 0)
131 flush_icache_range((uintptr_t)tp,
132 (uintptr_t)tp + AARCH64_INSN_SIZE);
133
134 return ret;
135 }
136
137 struct aarch64_insn_patch {
138 void **text_addrs;
139 u32 *new_insns;
140 int insn_cnt;
141 atomic_t cpu_count;
142 };
143
aarch64_insn_patch_text_cb(void * arg)144 static int __kprobes aarch64_insn_patch_text_cb(void *arg)
145 {
146 int i, ret = 0;
147 struct aarch64_insn_patch *pp = arg;
148
149 /* The first CPU becomes master */
150 if (atomic_inc_return(&pp->cpu_count) == 1) {
151 for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
152 ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
153 pp->new_insns[i]);
154 /*
155 * aarch64_insn_patch_text_nosync() calls flush_icache_range(),
156 * which ends with "dsb; isb" pair guaranteeing global
157 * visibility.
158 */
159 atomic_set(&pp->cpu_count, -1);
160 } else {
161 while (atomic_read(&pp->cpu_count) != -1)
162 cpu_relax();
163 isb();
164 }
165
166 return ret;
167 }
168
aarch64_insn_patch_text_sync(void * addrs[],u32 insns[],int cnt)169 int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
170 {
171 struct aarch64_insn_patch patch = {
172 .text_addrs = addrs,
173 .new_insns = insns,
174 .insn_cnt = cnt,
175 .cpu_count = ATOMIC_INIT(0),
176 };
177
178 if (cnt <= 0)
179 return -EINVAL;
180
181 return stop_machine(aarch64_insn_patch_text_cb, &patch,
182 cpu_online_mask);
183 }
184
aarch64_insn_patch_text(void * addrs[],u32 insns[],int cnt)185 int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
186 {
187 int ret;
188 u32 insn;
189
190 /* Unsafe to patch multiple instructions without synchronizaiton */
191 if (cnt == 1) {
192 ret = aarch64_insn_read(addrs[0], &insn);
193 if (ret)
194 return ret;
195
196 if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
197 /*
198 * ARMv8 architecture doesn't guarantee all CPUs see
199 * the new instruction after returning from function
200 * aarch64_insn_patch_text_nosync(). So send IPIs to
201 * all other CPUs to achieve instruction
202 * synchronization.
203 */
204 ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
205 kick_all_cpus_sync();
206 return ret;
207 }
208 }
209
210 return aarch64_insn_patch_text_sync(addrs, insns, cnt);
211 }
212
aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,u32 insn,u64 imm)213 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
214 u32 insn, u64 imm)
215 {
216 u32 immlo, immhi, lomask, himask, mask;
217 int shift;
218
219 switch (type) {
220 case AARCH64_INSN_IMM_ADR:
221 lomask = 0x3;
222 himask = 0x7ffff;
223 immlo = imm & lomask;
224 imm >>= 2;
225 immhi = imm & himask;
226 imm = (immlo << 24) | (immhi);
227 mask = (lomask << 24) | (himask);
228 shift = 5;
229 break;
230 case AARCH64_INSN_IMM_26:
231 mask = BIT(26) - 1;
232 shift = 0;
233 break;
234 case AARCH64_INSN_IMM_19:
235 mask = BIT(19) - 1;
236 shift = 5;
237 break;
238 case AARCH64_INSN_IMM_16:
239 mask = BIT(16) - 1;
240 shift = 5;
241 break;
242 case AARCH64_INSN_IMM_14:
243 mask = BIT(14) - 1;
244 shift = 5;
245 break;
246 case AARCH64_INSN_IMM_12:
247 mask = BIT(12) - 1;
248 shift = 10;
249 break;
250 case AARCH64_INSN_IMM_9:
251 mask = BIT(9) - 1;
252 shift = 12;
253 break;
254 default:
255 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
256 type);
257 return 0;
258 }
259
260 /* Update the immediate field. */
261 insn &= ~(mask << shift);
262 insn |= (imm & mask) << shift;
263
264 return insn;
265 }
266
aarch64_insn_gen_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_branch_type type)267 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
268 enum aarch64_insn_branch_type type)
269 {
270 u32 insn;
271 long offset;
272
273 /*
274 * PC: A 64-bit Program Counter holding the address of the current
275 * instruction. A64 instructions must be word-aligned.
276 */
277 BUG_ON((pc & 0x3) || (addr & 0x3));
278
279 /*
280 * B/BL support [-128M, 128M) offset
281 * ARM64 virtual address arrangement guarantees all kernel and module
282 * texts are within +/-128M.
283 */
284 offset = ((long)addr - (long)pc);
285 BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
286
287 if (type == AARCH64_INSN_BRANCH_LINK)
288 insn = aarch64_insn_get_bl_value();
289 else
290 insn = aarch64_insn_get_b_value();
291
292 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
293 offset >> 2);
294 }
295
aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)296 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
297 {
298 return aarch64_insn_get_hint_value() | op;
299 }
300
aarch64_insn_gen_nop(void)301 u32 __kprobes aarch64_insn_gen_nop(void)
302 {
303 return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
304 }
305