• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Kernel Probes (KProbes)
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  *
18  * Copyright (C) IBM Corporation, 2002, 2004
19  *
20  * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
21  *		Probes initial implementation ( includes contributions from
22  *		Rusty Russell).
23  * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
24  *		interface to access function arguments.
25  * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
26  *		<prasanna@in.ibm.com> adapted for x86_64 from i386.
27  * 2005-Mar	Roland McGrath <roland@redhat.com>
28  *		Fixed to handle %rip-relative addressing mode correctly.
29  * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
30  *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
31  *		<prasanna@in.ibm.com> added function-return probes.
32  * 2005-May	Rusty Lynch <rusty.lynch@intel.com>
33  * 		Added function return probes functionality
34  * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
35  * 		kprobe-booster and kretprobe-booster for i386.
36  * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
37  * 		and kretprobe-booster for x86-64
38  * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
39  * 		<arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
40  * 		unified x86 kprobes code.
41  */
42 
43 #include <linux/kprobes.h>
44 #include <linux/ptrace.h>
45 #include <linux/string.h>
46 #include <linux/slab.h>
47 #include <linux/hardirq.h>
48 #include <linux/preempt.h>
49 #include <linux/module.h>
50 #include <linux/kdebug.h>
51 
52 #include <asm/cacheflush.h>
53 #include <asm/desc.h>
54 #include <asm/pgtable.h>
55 #include <asm/uaccess.h>
56 #include <asm/alternative.h>
57 
58 void jprobe_return_end(void);
59 
60 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
61 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
62 
63 #ifdef CONFIG_X86_64
64 #define stack_addr(regs) ((unsigned long *)regs->sp)
65 #else
66 /*
67  * "&regs->sp" looks wrong, but it's correct for x86_32.  x86_32 CPUs
68  * don't save the ss and esp registers if the CPU is already in kernel
69  * mode when it traps.  So for kprobes, regs->sp and regs->ss are not
70  * the [nonexistent] saved stack pointer and ss register, but rather
71  * the top 8 bytes of the pre-int3 stack.  So &regs->sp happens to
72  * point to the top of the pre-int3 stack.
73  */
74 #define stack_addr(regs) ((unsigned long *)&regs->sp)
75 #endif
76 
77 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
78 	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
79 	  (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
80 	  (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
81 	  (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
82 	 << (row % 32))
83 	/*
84 	 * Undefined/reserved opcodes, conditional jump, Opcode Extension
85 	 * Groups, and some special opcodes can not boost.
86 	 */
87 static const u32 twobyte_is_boostable[256 / 32] = {
88 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
89 	/*      ----------------------------------------------          */
90 	W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
91 	W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 10 */
92 	W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
93 	W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
94 	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
95 	W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
96 	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
97 	W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
98 	W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
99 	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
100 	W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
101 	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
102 	W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
103 	W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
104 	W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
105 	W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)   /* f0 */
106 	/*      -----------------------------------------------         */
107 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
108 };
109 static const u32 onebyte_has_modrm[256 / 32] = {
110 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
111 	/*      -----------------------------------------------         */
112 	W(0x00, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* 00 */
113 	W(0x10, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) , /* 10 */
114 	W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* 20 */
115 	W(0x30, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) , /* 30 */
116 	W(0x40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 40 */
117 	W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
118 	W(0x60, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0) | /* 60 */
119 	W(0x70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 70 */
120 	W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
121 	W(0x90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 90 */
122 	W(0xa0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* a0 */
123 	W(0xb0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* b0 */
124 	W(0xc0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0) | /* c0 */
125 	W(0xd0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
126 	W(0xe0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* e0 */
127 	W(0xf0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1)   /* f0 */
128 	/*      -----------------------------------------------         */
129 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
130 };
131 static const u32 twobyte_has_modrm[256 / 32] = {
132 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
133 	/*      -----------------------------------------------         */
134 	W(0x00, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1) | /* 0f */
135 	W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0) , /* 1f */
136 	W(0x20, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 2f */
137 	W(0x30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 3f */
138 	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 4f */
139 	W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 5f */
140 	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 6f */
141 	W(0x70, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1) , /* 7f */
142 	W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 8f */
143 	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 9f */
144 	W(0xa0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) | /* af */
145 	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1) , /* bf */
146 	W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0) | /* cf */
147 	W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* df */
148 	W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* ef */
149 	W(0xf0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0)   /* ff */
150 	/*      -----------------------------------------------         */
151 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
152 };
153 #undef W
154 
155 struct kretprobe_blackpoint kretprobe_blacklist[] = {
156 	{"__switch_to", }, /* This function switches only current task, but
157 			      doesn't switch kernel stack.*/
158 	{NULL, NULL}	/* Terminator */
159 };
160 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
161 
162 /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
set_jmp_op(void * from,void * to)163 static void __kprobes set_jmp_op(void *from, void *to)
164 {
165 	struct __arch_jmp_op {
166 		char op;
167 		s32 raddr;
168 	} __attribute__((packed)) * jop;
169 	jop = (struct __arch_jmp_op *)from;
170 	jop->raddr = (s32)((long)(to) - ((long)(from) + 5));
171 	jop->op = RELATIVEJUMP_INSTRUCTION;
172 }
173 
174 /*
175  * Check for the REX prefix which can only exist on X86_64
176  * X86_32 always returns 0
177  */
is_REX_prefix(kprobe_opcode_t * insn)178 static int __kprobes is_REX_prefix(kprobe_opcode_t *insn)
179 {
180 #ifdef CONFIG_X86_64
181 	if ((*insn & 0xf0) == 0x40)
182 		return 1;
183 #endif
184 	return 0;
185 }
186 
187 /*
188  * Returns non-zero if opcode is boostable.
189  * RIP relative instructions are adjusted at copying time in 64 bits mode
190  */
can_boost(kprobe_opcode_t * opcodes)191 static int __kprobes can_boost(kprobe_opcode_t *opcodes)
192 {
193 	kprobe_opcode_t opcode;
194 	kprobe_opcode_t *orig_opcodes = opcodes;
195 
196 	if (search_exception_tables(opcodes))
197 		return 0;	/* Page fault may occur on this address. */
198 
199 retry:
200 	if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
201 		return 0;
202 	opcode = *(opcodes++);
203 
204 	/* 2nd-byte opcode */
205 	if (opcode == 0x0f) {
206 		if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
207 			return 0;
208 		return test_bit(*opcodes,
209 				(unsigned long *)twobyte_is_boostable);
210 	}
211 
212 	switch (opcode & 0xf0) {
213 #ifdef CONFIG_X86_64
214 	case 0x40:
215 		goto retry; /* REX prefix is boostable */
216 #endif
217 	case 0x60:
218 		if (0x63 < opcode && opcode < 0x67)
219 			goto retry; /* prefixes */
220 		/* can't boost Address-size override and bound */
221 		return (opcode != 0x62 && opcode != 0x67);
222 	case 0x70:
223 		return 0; /* can't boost conditional jump */
224 	case 0xc0:
225 		/* can't boost software-interruptions */
226 		return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
227 	case 0xd0:
228 		/* can boost AA* and XLAT */
229 		return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
230 	case 0xe0:
231 		/* can boost in/out and absolute jmps */
232 		return ((opcode & 0x04) || opcode == 0xea);
233 	case 0xf0:
234 		if ((opcode & 0x0c) == 0 && opcode != 0xf1)
235 			goto retry; /* lock/rep(ne) prefix */
236 		/* clear and set flags are boostable */
237 		return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
238 	default:
239 		/* segment override prefixes are boostable */
240 		if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
241 			goto retry; /* prefixes */
242 		/* CS override prefix and call are not boostable */
243 		return (opcode != 0x2e && opcode != 0x9a);
244 	}
245 }
246 
247 /*
248  * Returns non-zero if opcode modifies the interrupt flag.
249  */
is_IF_modifier(kprobe_opcode_t * insn)250 static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
251 {
252 	switch (*insn) {
253 	case 0xfa:		/* cli */
254 	case 0xfb:		/* sti */
255 	case 0xcf:		/* iret/iretd */
256 	case 0x9d:		/* popf/popfd */
257 		return 1;
258 	}
259 
260 	/*
261 	 * on X86_64, 0x40-0x4f are REX prefixes so we need to look
262 	 * at the next byte instead.. but of course not recurse infinitely
263 	 */
264 	if (is_REX_prefix(insn))
265 		return is_IF_modifier(++insn);
266 
267 	return 0;
268 }
269 
270 /*
271  * Adjust the displacement if the instruction uses the %rip-relative
272  * addressing mode.
273  * If it does, Return the address of the 32-bit displacement word.
274  * If not, return null.
275  * Only applicable to 64-bit x86.
276  */
fix_riprel(struct kprobe * p)277 static void __kprobes fix_riprel(struct kprobe *p)
278 {
279 #ifdef CONFIG_X86_64
280 	u8 *insn = p->ainsn.insn;
281 	s64 disp;
282 	int need_modrm;
283 
284 	/* Skip legacy instruction prefixes.  */
285 	while (1) {
286 		switch (*insn) {
287 		case 0x66:
288 		case 0x67:
289 		case 0x2e:
290 		case 0x3e:
291 		case 0x26:
292 		case 0x64:
293 		case 0x65:
294 		case 0x36:
295 		case 0xf0:
296 		case 0xf3:
297 		case 0xf2:
298 			++insn;
299 			continue;
300 		}
301 		break;
302 	}
303 
304 	/* Skip REX instruction prefix.  */
305 	if (is_REX_prefix(insn))
306 		++insn;
307 
308 	if (*insn == 0x0f) {
309 		/* Two-byte opcode.  */
310 		++insn;
311 		need_modrm = test_bit(*insn,
312 				      (unsigned long *)twobyte_has_modrm);
313 	} else
314 		/* One-byte opcode.  */
315 		need_modrm = test_bit(*insn,
316 				      (unsigned long *)onebyte_has_modrm);
317 
318 	if (need_modrm) {
319 		u8 modrm = *++insn;
320 		if ((modrm & 0xc7) == 0x05) {
321 			/* %rip+disp32 addressing mode */
322 			/* Displacement follows ModRM byte.  */
323 			++insn;
324 			/*
325 			 * The copied instruction uses the %rip-relative
326 			 * addressing mode.  Adjust the displacement for the
327 			 * difference between the original location of this
328 			 * instruction and the location of the copy that will
329 			 * actually be run.  The tricky bit here is making sure
330 			 * that the sign extension happens correctly in this
331 			 * calculation, since we need a signed 32-bit result to
332 			 * be sign-extended to 64 bits when it's added to the
333 			 * %rip value and yield the same 64-bit result that the
334 			 * sign-extension of the original signed 32-bit
335 			 * displacement would have given.
336 			 */
337 			disp = (u8 *) p->addr + *((s32 *) insn) -
338 			       (u8 *) p->ainsn.insn;
339 			BUG_ON((s64) (s32) disp != disp); /* Sanity check.  */
340 			*(s32 *)insn = (s32) disp;
341 		}
342 	}
343 #endif
344 }
345 
arch_copy_kprobe(struct kprobe * p)346 static void __kprobes arch_copy_kprobe(struct kprobe *p)
347 {
348 	memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
349 
350 	fix_riprel(p);
351 
352 	if (can_boost(p->addr))
353 		p->ainsn.boostable = 0;
354 	else
355 		p->ainsn.boostable = -1;
356 
357 	p->opcode = *p->addr;
358 }
359 
arch_prepare_kprobe(struct kprobe * p)360 int __kprobes arch_prepare_kprobe(struct kprobe *p)
361 {
362 	/* insn: must be on special executable page on x86. */
363 	p->ainsn.insn = get_insn_slot();
364 	if (!p->ainsn.insn)
365 		return -ENOMEM;
366 	arch_copy_kprobe(p);
367 	return 0;
368 }
369 
arch_arm_kprobe(struct kprobe * p)370 void __kprobes arch_arm_kprobe(struct kprobe *p)
371 {
372 	text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
373 }
374 
arch_disarm_kprobe(struct kprobe * p)375 void __kprobes arch_disarm_kprobe(struct kprobe *p)
376 {
377 	text_poke(p->addr, &p->opcode, 1);
378 }
379 
arch_remove_kprobe(struct kprobe * p)380 void __kprobes arch_remove_kprobe(struct kprobe *p)
381 {
382 	if (p->ainsn.insn) {
383 		free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
384 		p->ainsn.insn = NULL;
385 	}
386 }
387 
save_previous_kprobe(struct kprobe_ctlblk * kcb)388 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
389 {
390 	kcb->prev_kprobe.kp = kprobe_running();
391 	kcb->prev_kprobe.status = kcb->kprobe_status;
392 	kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
393 	kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
394 }
395 
restore_previous_kprobe(struct kprobe_ctlblk * kcb)396 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
397 {
398 	__get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
399 	kcb->kprobe_status = kcb->prev_kprobe.status;
400 	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
401 	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
402 }
403 
set_current_kprobe(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)404 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
405 				struct kprobe_ctlblk *kcb)
406 {
407 	__get_cpu_var(current_kprobe) = p;
408 	kcb->kprobe_saved_flags = kcb->kprobe_old_flags
409 		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
410 	if (is_IF_modifier(p->ainsn.insn))
411 		kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
412 }
413 
clear_btf(void)414 static void __kprobes clear_btf(void)
415 {
416 	if (test_thread_flag(TIF_DEBUGCTLMSR))
417 		update_debugctlmsr(0);
418 }
419 
restore_btf(void)420 static void __kprobes restore_btf(void)
421 {
422 	if (test_thread_flag(TIF_DEBUGCTLMSR))
423 		update_debugctlmsr(current->thread.debugctlmsr);
424 }
425 
prepare_singlestep(struct kprobe * p,struct pt_regs * regs)426 static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
427 {
428 	clear_btf();
429 	regs->flags |= X86_EFLAGS_TF;
430 	regs->flags &= ~X86_EFLAGS_IF;
431 	/* single step inline if the instruction is an int3 */
432 	if (p->opcode == BREAKPOINT_INSTRUCTION)
433 		regs->ip = (unsigned long)p->addr;
434 	else
435 		regs->ip = (unsigned long)p->ainsn.insn;
436 }
437 
arch_prepare_kretprobe(struct kretprobe_instance * ri,struct pt_regs * regs)438 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
439 				      struct pt_regs *regs)
440 {
441 	unsigned long *sara = stack_addr(regs);
442 
443 	ri->ret_addr = (kprobe_opcode_t *) *sara;
444 
445 	/* Replace the return addr with trampoline addr */
446 	*sara = (unsigned long) &kretprobe_trampoline;
447 }
448 
setup_singlestep(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)449 static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
450 				       struct kprobe_ctlblk *kcb)
451 {
452 #if !defined(CONFIG_PREEMPT) || defined(CONFIG_FREEZER)
453 	if (p->ainsn.boostable == 1 && !p->post_handler) {
454 		/* Boost up -- we can execute copied instructions directly */
455 		reset_current_kprobe();
456 		regs->ip = (unsigned long)p->ainsn.insn;
457 		preempt_enable_no_resched();
458 		return;
459 	}
460 #endif
461 	prepare_singlestep(p, regs);
462 	kcb->kprobe_status = KPROBE_HIT_SS;
463 }
464 
465 /*
466  * We have reentered the kprobe_handler(), since another probe was hit while
467  * within the handler. We save the original kprobes variables and just single
468  * step on the instruction of the new probe without calling any user handlers.
469  */
reenter_kprobe(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)470 static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
471 				    struct kprobe_ctlblk *kcb)
472 {
473 	switch (kcb->kprobe_status) {
474 	case KPROBE_HIT_SSDONE:
475 #ifdef CONFIG_X86_64
476 		/* TODO: Provide re-entrancy from post_kprobes_handler() and
477 		 * avoid exception stack corruption while single-stepping on
478 		 * the instruction of the new probe.
479 		 */
480 		arch_disarm_kprobe(p);
481 		regs->ip = (unsigned long)p->addr;
482 		reset_current_kprobe();
483 		preempt_enable_no_resched();
484 		break;
485 #endif
486 	case KPROBE_HIT_ACTIVE:
487 		save_previous_kprobe(kcb);
488 		set_current_kprobe(p, regs, kcb);
489 		kprobes_inc_nmissed_count(p);
490 		prepare_singlestep(p, regs);
491 		kcb->kprobe_status = KPROBE_REENTER;
492 		break;
493 	case KPROBE_HIT_SS:
494 		if (p == kprobe_running()) {
495 			regs->flags &= ~X86_EFLAGS_TF;
496 			regs->flags |= kcb->kprobe_saved_flags;
497 			return 0;
498 		} else {
499 			/* A probe has been hit in the codepath leading up
500 			 * to, or just after, single-stepping of a probed
501 			 * instruction. This entire codepath should strictly
502 			 * reside in .kprobes.text section. Raise a warning
503 			 * to highlight this peculiar case.
504 			 */
505 		}
506 	default:
507 		/* impossible cases */
508 		WARN_ON(1);
509 		return 0;
510 	}
511 
512 	return 1;
513 }
514 
515 /*
516  * Interrupts are disabled on entry as trap3 is an interrupt gate and they
517  * remain disabled thorough out this function.
518  */
kprobe_handler(struct pt_regs * regs)519 static int __kprobes kprobe_handler(struct pt_regs *regs)
520 {
521 	kprobe_opcode_t *addr;
522 	struct kprobe *p;
523 	struct kprobe_ctlblk *kcb;
524 
525 	addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
526 	if (*addr != BREAKPOINT_INSTRUCTION) {
527 		/*
528 		 * The breakpoint instruction was removed right
529 		 * after we hit it.  Another cpu has removed
530 		 * either a probepoint or a debugger breakpoint
531 		 * at this address.  In either case, no further
532 		 * handling of this interrupt is appropriate.
533 		 * Back up over the (now missing) int3 and run
534 		 * the original instruction.
535 		 */
536 		regs->ip = (unsigned long)addr;
537 		return 1;
538 	}
539 
540 	/*
541 	 * We don't want to be preempted for the entire
542 	 * duration of kprobe processing. We conditionally
543 	 * re-enable preemption at the end of this function,
544 	 * and also in reenter_kprobe() and setup_singlestep().
545 	 */
546 	preempt_disable();
547 
548 	kcb = get_kprobe_ctlblk();
549 	p = get_kprobe(addr);
550 
551 	if (p) {
552 		if (kprobe_running()) {
553 			if (reenter_kprobe(p, regs, kcb))
554 				return 1;
555 		} else {
556 			set_current_kprobe(p, regs, kcb);
557 			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
558 
559 			/*
560 			 * If we have no pre-handler or it returned 0, we
561 			 * continue with normal processing.  If we have a
562 			 * pre-handler and it returned non-zero, it prepped
563 			 * for calling the break_handler below on re-entry
564 			 * for jprobe processing, so get out doing nothing
565 			 * more here.
566 			 */
567 			if (!p->pre_handler || !p->pre_handler(p, regs))
568 				setup_singlestep(p, regs, kcb);
569 			return 1;
570 		}
571 	} else if (kprobe_running()) {
572 		p = __get_cpu_var(current_kprobe);
573 		if (p->break_handler && p->break_handler(p, regs)) {
574 			setup_singlestep(p, regs, kcb);
575 			return 1;
576 		}
577 	} /* else: not a kprobe fault; let the kernel handle it */
578 
579 	preempt_enable_no_resched();
580 	return 0;
581 }
582 
583 /*
584  * When a retprobed function returns, this code saves registers and
585  * calls trampoline_handler() runs, which calls the kretprobe's handler.
586  */
kretprobe_trampoline_holder(void)587 static void __used __kprobes kretprobe_trampoline_holder(void)
588 {
589 	asm volatile (
590 			".global kretprobe_trampoline\n"
591 			"kretprobe_trampoline: \n"
592 #ifdef CONFIG_X86_64
593 			/* We don't bother saving the ss register */
594 			"	pushq %rsp\n"
595 			"	pushfq\n"
596 			/*
597 			 * Skip cs, ip, orig_ax.
598 			 * trampoline_handler() will plug in these values
599 			 */
600 			"	subq $24, %rsp\n"
601 			"	pushq %rdi\n"
602 			"	pushq %rsi\n"
603 			"	pushq %rdx\n"
604 			"	pushq %rcx\n"
605 			"	pushq %rax\n"
606 			"	pushq %r8\n"
607 			"	pushq %r9\n"
608 			"	pushq %r10\n"
609 			"	pushq %r11\n"
610 			"	pushq %rbx\n"
611 			"	pushq %rbp\n"
612 			"	pushq %r12\n"
613 			"	pushq %r13\n"
614 			"	pushq %r14\n"
615 			"	pushq %r15\n"
616 			"	movq %rsp, %rdi\n"
617 			"	call trampoline_handler\n"
618 			/* Replace saved sp with true return address. */
619 			"	movq %rax, 152(%rsp)\n"
620 			"	popq %r15\n"
621 			"	popq %r14\n"
622 			"	popq %r13\n"
623 			"	popq %r12\n"
624 			"	popq %rbp\n"
625 			"	popq %rbx\n"
626 			"	popq %r11\n"
627 			"	popq %r10\n"
628 			"	popq %r9\n"
629 			"	popq %r8\n"
630 			"	popq %rax\n"
631 			"	popq %rcx\n"
632 			"	popq %rdx\n"
633 			"	popq %rsi\n"
634 			"	popq %rdi\n"
635 			/* Skip orig_ax, ip, cs */
636 			"	addq $24, %rsp\n"
637 			"	popfq\n"
638 #else
639 			"	pushf\n"
640 			/*
641 			 * Skip cs, ip, orig_ax.
642 			 * trampoline_handler() will plug in these values
643 			 */
644 			"	subl $12, %esp\n"
645 			"	pushl %fs\n"
646 			"	pushl %ds\n"
647 			"	pushl %es\n"
648 			"	pushl %eax\n"
649 			"	pushl %ebp\n"
650 			"	pushl %edi\n"
651 			"	pushl %esi\n"
652 			"	pushl %edx\n"
653 			"	pushl %ecx\n"
654 			"	pushl %ebx\n"
655 			"	movl %esp, %eax\n"
656 			"	call trampoline_handler\n"
657 			/* Move flags to cs */
658 			"	movl 52(%esp), %edx\n"
659 			"	movl %edx, 48(%esp)\n"
660 			/* Replace saved flags with true return address. */
661 			"	movl %eax, 52(%esp)\n"
662 			"	popl %ebx\n"
663 			"	popl %ecx\n"
664 			"	popl %edx\n"
665 			"	popl %esi\n"
666 			"	popl %edi\n"
667 			"	popl %ebp\n"
668 			"	popl %eax\n"
669 			/* Skip ip, orig_ax, es, ds, fs */
670 			"	addl $20, %esp\n"
671 			"	popf\n"
672 #endif
673 			"	ret\n");
674 }
675 
676 /*
677  * Called from kretprobe_trampoline
678  */
trampoline_handler(struct pt_regs * regs)679 static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
680 {
681 	struct kretprobe_instance *ri = NULL;
682 	struct hlist_head *head, empty_rp;
683 	struct hlist_node *node, *tmp;
684 	unsigned long flags, orig_ret_address = 0;
685 	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
686 
687 	INIT_HLIST_HEAD(&empty_rp);
688 	kretprobe_hash_lock(current, &head, &flags);
689 	/* fixup registers */
690 #ifdef CONFIG_X86_64
691 	regs->cs = __KERNEL_CS;
692 #else
693 	regs->cs = __KERNEL_CS | get_kernel_rpl();
694 #endif
695 	regs->ip = trampoline_address;
696 	regs->orig_ax = ~0UL;
697 
698 	/*
699 	 * It is possible to have multiple instances associated with a given
700 	 * task either because multiple functions in the call path have
701 	 * return probes installed on them, and/or more than one
702 	 * return probe was registered for a target function.
703 	 *
704 	 * We can handle this because:
705 	 *     - instances are always pushed into the head of the list
706 	 *     - when multiple return probes are registered for the same
707 	 *	 function, the (chronologically) first instance's ret_addr
708 	 *	 will be the real return address, and all the rest will
709 	 *	 point to kretprobe_trampoline.
710 	 */
711 	hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
712 		if (ri->task != current)
713 			/* another task is sharing our hash bucket */
714 			continue;
715 
716 		if (ri->rp && ri->rp->handler) {
717 			__get_cpu_var(current_kprobe) = &ri->rp->kp;
718 			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
719 			ri->rp->handler(ri, regs);
720 			__get_cpu_var(current_kprobe) = NULL;
721 		}
722 
723 		orig_ret_address = (unsigned long)ri->ret_addr;
724 		recycle_rp_inst(ri, &empty_rp);
725 
726 		if (orig_ret_address != trampoline_address)
727 			/*
728 			 * This is the real return address. Any other
729 			 * instances associated with this task are for
730 			 * other calls deeper on the call stack
731 			 */
732 			break;
733 	}
734 
735 	kretprobe_assert(ri, orig_ret_address, trampoline_address);
736 
737 	kretprobe_hash_unlock(current, &flags);
738 
739 	hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
740 		hlist_del(&ri->hlist);
741 		kfree(ri);
742 	}
743 	return (void *)orig_ret_address;
744 }
745 
746 /*
747  * Called after single-stepping.  p->addr is the address of the
748  * instruction whose first byte has been replaced by the "int 3"
749  * instruction.  To avoid the SMP problems that can occur when we
750  * temporarily put back the original opcode to single-step, we
751  * single-stepped a copy of the instruction.  The address of this
752  * copy is p->ainsn.insn.
753  *
754  * This function prepares to return from the post-single-step
755  * interrupt.  We have to fix up the stack as follows:
756  *
757  * 0) Except in the case of absolute or indirect jump or call instructions,
758  * the new ip is relative to the copied instruction.  We need to make
759  * it relative to the original instruction.
760  *
761  * 1) If the single-stepped instruction was pushfl, then the TF and IF
762  * flags are set in the just-pushed flags, and may need to be cleared.
763  *
764  * 2) If the single-stepped instruction was a call, the return address
765  * that is atop the stack is the address following the copied instruction.
766  * We need to make it the address following the original instruction.
767  *
768  * If this is the first time we've single-stepped the instruction at
769  * this probepoint, and the instruction is boostable, boost it: add a
770  * jump instruction after the copied instruction, that jumps to the next
771  * instruction after the probepoint.
772  */
resume_execution(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)773 static void __kprobes resume_execution(struct kprobe *p,
774 		struct pt_regs *regs, struct kprobe_ctlblk *kcb)
775 {
776 	unsigned long *tos = stack_addr(regs);
777 	unsigned long copy_ip = (unsigned long)p->ainsn.insn;
778 	unsigned long orig_ip = (unsigned long)p->addr;
779 	kprobe_opcode_t *insn = p->ainsn.insn;
780 
781 	/*skip the REX prefix*/
782 	if (is_REX_prefix(insn))
783 		insn++;
784 
785 	regs->flags &= ~X86_EFLAGS_TF;
786 	switch (*insn) {
787 	case 0x9c:	/* pushfl */
788 		*tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
789 		*tos |= kcb->kprobe_old_flags;
790 		break;
791 	case 0xc2:	/* iret/ret/lret */
792 	case 0xc3:
793 	case 0xca:
794 	case 0xcb:
795 	case 0xcf:
796 	case 0xea:	/* jmp absolute -- ip is correct */
797 		/* ip is already adjusted, no more changes required */
798 		p->ainsn.boostable = 1;
799 		goto no_change;
800 	case 0xe8:	/* call relative - Fix return addr */
801 		*tos = orig_ip + (*tos - copy_ip);
802 		break;
803 #ifdef CONFIG_X86_32
804 	case 0x9a:	/* call absolute -- same as call absolute, indirect */
805 		*tos = orig_ip + (*tos - copy_ip);
806 		goto no_change;
807 #endif
808 	case 0xff:
809 		if ((insn[1] & 0x30) == 0x10) {
810 			/*
811 			 * call absolute, indirect
812 			 * Fix return addr; ip is correct.
813 			 * But this is not boostable
814 			 */
815 			*tos = orig_ip + (*tos - copy_ip);
816 			goto no_change;
817 		} else if (((insn[1] & 0x31) == 0x20) ||
818 			   ((insn[1] & 0x31) == 0x21)) {
819 			/*
820 			 * jmp near and far, absolute indirect
821 			 * ip is correct. And this is boostable
822 			 */
823 			p->ainsn.boostable = 1;
824 			goto no_change;
825 		}
826 	default:
827 		break;
828 	}
829 
830 	if (p->ainsn.boostable == 0) {
831 		if ((regs->ip > copy_ip) &&
832 		    (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
833 			/*
834 			 * These instructions can be executed directly if it
835 			 * jumps back to correct address.
836 			 */
837 			set_jmp_op((void *)regs->ip,
838 				   (void *)orig_ip + (regs->ip - copy_ip));
839 			p->ainsn.boostable = 1;
840 		} else {
841 			p->ainsn.boostable = -1;
842 		}
843 	}
844 
845 	regs->ip += orig_ip - copy_ip;
846 
847 no_change:
848 	restore_btf();
849 }
850 
851 /*
852  * Interrupts are disabled on entry as trap1 is an interrupt gate and they
853  * remain disabled thoroughout this function.
854  */
post_kprobe_handler(struct pt_regs * regs)855 static int __kprobes post_kprobe_handler(struct pt_regs *regs)
856 {
857 	struct kprobe *cur = kprobe_running();
858 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
859 
860 	if (!cur)
861 		return 0;
862 
863 	resume_execution(cur, regs, kcb);
864 	regs->flags |= kcb->kprobe_saved_flags;
865 
866 	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
867 		kcb->kprobe_status = KPROBE_HIT_SSDONE;
868 		cur->post_handler(cur, regs, 0);
869 	}
870 
871 	/* Restore back the original saved kprobes variables and continue. */
872 	if (kcb->kprobe_status == KPROBE_REENTER) {
873 		restore_previous_kprobe(kcb);
874 		goto out;
875 	}
876 	reset_current_kprobe();
877 out:
878 	preempt_enable_no_resched();
879 
880 	/*
881 	 * if somebody else is singlestepping across a probe point, flags
882 	 * will have TF set, in which case, continue the remaining processing
883 	 * of do_debug, as if this is not a probe hit.
884 	 */
885 	if (regs->flags & X86_EFLAGS_TF)
886 		return 0;
887 
888 	return 1;
889 }
890 
kprobe_fault_handler(struct pt_regs * regs,int trapnr)891 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
892 {
893 	struct kprobe *cur = kprobe_running();
894 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
895 
896 	switch (kcb->kprobe_status) {
897 	case KPROBE_HIT_SS:
898 	case KPROBE_REENTER:
899 		/*
900 		 * We are here because the instruction being single
901 		 * stepped caused a page fault. We reset the current
902 		 * kprobe and the ip points back to the probe address
903 		 * and allow the page fault handler to continue as a
904 		 * normal page fault.
905 		 */
906 		regs->ip = (unsigned long)cur->addr;
907 		regs->flags |= kcb->kprobe_old_flags;
908 		if (kcb->kprobe_status == KPROBE_REENTER)
909 			restore_previous_kprobe(kcb);
910 		else
911 			reset_current_kprobe();
912 		preempt_enable_no_resched();
913 		break;
914 	case KPROBE_HIT_ACTIVE:
915 	case KPROBE_HIT_SSDONE:
916 		/*
917 		 * We increment the nmissed count for accounting,
918 		 * we can also use npre/npostfault count for accounting
919 		 * these specific fault cases.
920 		 */
921 		kprobes_inc_nmissed_count(cur);
922 
923 		/*
924 		 * We come here because instructions in the pre/post
925 		 * handler caused the page_fault, this could happen
926 		 * if handler tries to access user space by
927 		 * copy_from_user(), get_user() etc. Let the
928 		 * user-specified handler try to fix it first.
929 		 */
930 		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
931 			return 1;
932 
933 		/*
934 		 * In case the user-specified fault handler returned
935 		 * zero, try to fix up.
936 		 */
937 		if (fixup_exception(regs))
938 			return 1;
939 
940 		/*
941 		 * fixup routine could not handle it,
942 		 * Let do_page_fault() fix it.
943 		 */
944 		break;
945 	default:
946 		break;
947 	}
948 	return 0;
949 }
950 
951 /*
952  * Wrapper routine for handling exceptions.
953  */
kprobe_exceptions_notify(struct notifier_block * self,unsigned long val,void * data)954 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
955 				       unsigned long val, void *data)
956 {
957 	struct die_args *args = data;
958 	int ret = NOTIFY_DONE;
959 
960 	if (args->regs && user_mode_vm(args->regs))
961 		return ret;
962 
963 	switch (val) {
964 	case DIE_INT3:
965 		if (kprobe_handler(args->regs))
966 			ret = NOTIFY_STOP;
967 		break;
968 	case DIE_DEBUG:
969 		if (post_kprobe_handler(args->regs))
970 			ret = NOTIFY_STOP;
971 		break;
972 	case DIE_GPF:
973 		/*
974 		 * To be potentially processing a kprobe fault and to
975 		 * trust the result from kprobe_running(), we have
976 		 * be non-preemptible.
977 		 */
978 		if (!preemptible() && kprobe_running() &&
979 		    kprobe_fault_handler(args->regs, args->trapnr))
980 			ret = NOTIFY_STOP;
981 		break;
982 	default:
983 		break;
984 	}
985 	return ret;
986 }
987 
setjmp_pre_handler(struct kprobe * p,struct pt_regs * regs)988 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
989 {
990 	struct jprobe *jp = container_of(p, struct jprobe, kp);
991 	unsigned long addr;
992 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
993 
994 	kcb->jprobe_saved_regs = *regs;
995 	kcb->jprobe_saved_sp = stack_addr(regs);
996 	addr = (unsigned long)(kcb->jprobe_saved_sp);
997 
998 	/*
999 	 * As Linus pointed out, gcc assumes that the callee
1000 	 * owns the argument space and could overwrite it, e.g.
1001 	 * tailcall optimization. So, to be absolutely safe
1002 	 * we also save and restore enough stack bytes to cover
1003 	 * the argument area.
1004 	 */
1005 	memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
1006 	       MIN_STACK_SIZE(addr));
1007 	regs->flags &= ~X86_EFLAGS_IF;
1008 	trace_hardirqs_off();
1009 	regs->ip = (unsigned long)(jp->entry);
1010 	return 1;
1011 }
1012 
jprobe_return(void)1013 void __kprobes jprobe_return(void)
1014 {
1015 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1016 
1017 	asm volatile (
1018 #ifdef CONFIG_X86_64
1019 			"       xchg   %%rbx,%%rsp	\n"
1020 #else
1021 			"       xchgl   %%ebx,%%esp	\n"
1022 #endif
1023 			"       int3			\n"
1024 			"       .globl jprobe_return_end\n"
1025 			"       jprobe_return_end:	\n"
1026 			"       nop			\n"::"b"
1027 			(kcb->jprobe_saved_sp):"memory");
1028 }
1029 
longjmp_break_handler(struct kprobe * p,struct pt_regs * regs)1030 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1031 {
1032 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1033 	u8 *addr = (u8 *) (regs->ip - 1);
1034 	struct jprobe *jp = container_of(p, struct jprobe, kp);
1035 
1036 	if ((addr > (u8 *) jprobe_return) &&
1037 	    (addr < (u8 *) jprobe_return_end)) {
1038 		if (stack_addr(regs) != kcb->jprobe_saved_sp) {
1039 			struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
1040 			printk(KERN_ERR
1041 			       "current sp %p does not match saved sp %p\n",
1042 			       stack_addr(regs), kcb->jprobe_saved_sp);
1043 			printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
1044 			show_registers(saved_regs);
1045 			printk(KERN_ERR "Current registers\n");
1046 			show_registers(regs);
1047 			BUG();
1048 		}
1049 		*regs = kcb->jprobe_saved_regs;
1050 		memcpy((kprobe_opcode_t *)(kcb->jprobe_saved_sp),
1051 		       kcb->jprobes_stack,
1052 		       MIN_STACK_SIZE(kcb->jprobe_saved_sp));
1053 		preempt_enable_no_resched();
1054 		return 1;
1055 	}
1056 	return 0;
1057 }
1058 
arch_init_kprobes(void)1059 int __init arch_init_kprobes(void)
1060 {
1061 	return 0;
1062 }
1063 
arch_trampoline_kprobe(struct kprobe * p)1064 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
1065 {
1066 	return 0;
1067 }
1068