1 /*
2 * Copyright (C) 1995 Linus Torvalds
3 *
4 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000
6 *
7 * X86-64 port
8 * Andi Kleen.
9 *
10 * CPU hotplug support - ashok.raj@intel.com
11 */
12
13 /*
14 * This file handles the architecture-dependent parts of process handling..
15 */
16
17 #include <linux/cpu.h>
18 #include <linux/errno.h>
19 #include <linux/sched.h>
20 #include <linux/sched/task.h>
21 #include <linux/sched/task_stack.h>
22 #include <linux/fs.h>
23 #include <linux/kernel.h>
24 #include <linux/mm.h>
25 #include <linux/elfcore.h>
26 #include <linux/smp.h>
27 #include <linux/slab.h>
28 #include <linux/user.h>
29 #include <linux/interrupt.h>
30 #include <linux/delay.h>
31 #include <linux/export.h>
32 #include <linux/ptrace.h>
33 #include <linux/notifier.h>
34 #include <linux/kprobes.h>
35 #include <linux/kdebug.h>
36 #include <linux/prctl.h>
37 #include <linux/uaccess.h>
38 #include <linux/io.h>
39 #include <linux/ftrace.h>
40 #include <linux/syscalls.h>
41
42 #include <asm/pgtable.h>
43 #include <asm/processor.h>
44 #include <asm/fpu/internal.h>
45 #include <asm/mmu_context.h>
46 #include <asm/prctl.h>
47 #include <asm/desc.h>
48 #include <asm/proto.h>
49 #include <asm/ia32.h>
50 #include <asm/syscalls.h>
51 #include <asm/debugreg.h>
52 #include <asm/switch_to.h>
53 #include <asm/xen/hypervisor.h>
54 #include <asm/vdso.h>
55 #include <asm/intel_rdt_sched.h>
56 #include <asm/unistd.h>
57 #ifdef CONFIG_IA32_EMULATION
58 /* Not included via unistd.h */
59 #include <asm/unistd_32_ia32.h>
60 #endif
61
62 #include "process.h"
63
64 __visible DEFINE_PER_CPU(unsigned long, rsp_scratch);
65
66 /* Prints also some state that isn't saved in the pt_regs */
__show_regs(struct pt_regs * regs,int all)67 void __show_regs(struct pt_regs *regs, int all)
68 {
69 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
70 unsigned long d0, d1, d2, d3, d6, d7;
71 unsigned int fsindex, gsindex;
72 unsigned int ds, cs, es;
73
74 show_iret_regs(regs);
75
76 if (regs->orig_ax != -1)
77 pr_cont(" ORIG_RAX: %016lx\n", regs->orig_ax);
78 else
79 pr_cont("\n");
80
81 printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
82 regs->ax, regs->bx, regs->cx);
83 printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
84 regs->dx, regs->si, regs->di);
85 printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
86 regs->bp, regs->r8, regs->r9);
87 printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
88 regs->r10, regs->r11, regs->r12);
89 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
90 regs->r13, regs->r14, regs->r15);
91
92 if (!all)
93 return;
94
95 asm("movl %%ds,%0" : "=r" (ds));
96 asm("movl %%cs,%0" : "=r" (cs));
97 asm("movl %%es,%0" : "=r" (es));
98 asm("movl %%fs,%0" : "=r" (fsindex));
99 asm("movl %%gs,%0" : "=r" (gsindex));
100
101 rdmsrl(MSR_FS_BASE, fs);
102 rdmsrl(MSR_GS_BASE, gs);
103 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
104
105 cr0 = read_cr0();
106 cr2 = read_cr2();
107 cr3 = __read_cr3();
108 cr4 = __read_cr4();
109
110 printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
111 fs, fsindex, gs, gsindex, shadowgs);
112 printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
113 es, cr0);
114 printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
115 cr4);
116
117 get_debugreg(d0, 0);
118 get_debugreg(d1, 1);
119 get_debugreg(d2, 2);
120 get_debugreg(d3, 3);
121 get_debugreg(d6, 6);
122 get_debugreg(d7, 7);
123
124 /* Only print out debug registers if they are in their non-default state. */
125 if (!((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
126 (d6 == DR6_RESERVED) && (d7 == 0x400))) {
127 printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n",
128 d0, d1, d2);
129 printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n",
130 d3, d6, d7);
131 }
132
133 if (boot_cpu_has(X86_FEATURE_OSPKE))
134 printk(KERN_DEFAULT "PKRU: %08x\n", read_pkru());
135 }
136
release_thread(struct task_struct * dead_task)137 void release_thread(struct task_struct *dead_task)
138 {
139 if (dead_task->mm) {
140 #ifdef CONFIG_MODIFY_LDT_SYSCALL
141 if (dead_task->mm->context.ldt) {
142 pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
143 dead_task->comm,
144 dead_task->mm->context.ldt->entries,
145 dead_task->mm->context.ldt->nr_entries);
146 BUG();
147 }
148 #endif
149 }
150 }
151
152 enum which_selector {
153 FS,
154 GS
155 };
156
157 /*
158 * Saves the FS or GS base for an outgoing thread if FSGSBASE extensions are
159 * not available. The goal is to be reasonably fast on non-FSGSBASE systems.
160 * It's forcibly inlined because it'll generate better code and this function
161 * is hot.
162 */
save_base_legacy(struct task_struct * prev_p,unsigned short selector,enum which_selector which)163 static __always_inline void save_base_legacy(struct task_struct *prev_p,
164 unsigned short selector,
165 enum which_selector which)
166 {
167 if (likely(selector == 0)) {
168 /*
169 * On Intel (without X86_BUG_NULL_SEG), the segment base could
170 * be the pre-existing saved base or it could be zero. On AMD
171 * (with X86_BUG_NULL_SEG), the segment base could be almost
172 * anything.
173 *
174 * This branch is very hot (it's hit twice on almost every
175 * context switch between 64-bit programs), and avoiding
176 * the RDMSR helps a lot, so we just assume that whatever
177 * value is already saved is correct. This matches historical
178 * Linux behavior, so it won't break existing applications.
179 *
180 * To avoid leaking state, on non-X86_BUG_NULL_SEG CPUs, if we
181 * report that the base is zero, it needs to actually be zero:
182 * see the corresponding logic in load_seg_legacy.
183 */
184 } else {
185 /*
186 * If the selector is 1, 2, or 3, then the base is zero on
187 * !X86_BUG_NULL_SEG CPUs and could be anything on
188 * X86_BUG_NULL_SEG CPUs. In the latter case, Linux
189 * has never attempted to preserve the base across context
190 * switches.
191 *
192 * If selector > 3, then it refers to a real segment, and
193 * saving the base isn't necessary.
194 */
195 if (which == FS)
196 prev_p->thread.fsbase = 0;
197 else
198 prev_p->thread.gsbase = 0;
199 }
200 }
201
save_fsgs(struct task_struct * task)202 static __always_inline void save_fsgs(struct task_struct *task)
203 {
204 savesegment(fs, task->thread.fsindex);
205 savesegment(gs, task->thread.gsindex);
206 save_base_legacy(task, task->thread.fsindex, FS);
207 save_base_legacy(task, task->thread.gsindex, GS);
208 }
209
loadseg(enum which_selector which,unsigned short sel)210 static __always_inline void loadseg(enum which_selector which,
211 unsigned short sel)
212 {
213 if (which == FS)
214 loadsegment(fs, sel);
215 else
216 load_gs_index(sel);
217 }
218
load_seg_legacy(unsigned short prev_index,unsigned long prev_base,unsigned short next_index,unsigned long next_base,enum which_selector which)219 static __always_inline void load_seg_legacy(unsigned short prev_index,
220 unsigned long prev_base,
221 unsigned short next_index,
222 unsigned long next_base,
223 enum which_selector which)
224 {
225 if (likely(next_index <= 3)) {
226 /*
227 * The next task is using 64-bit TLS, is not using this
228 * segment at all, or is having fun with arcane CPU features.
229 */
230 if (next_base == 0) {
231 /*
232 * Nasty case: on AMD CPUs, we need to forcibly zero
233 * the base.
234 */
235 if (static_cpu_has_bug(X86_BUG_NULL_SEG)) {
236 loadseg(which, __USER_DS);
237 loadseg(which, next_index);
238 } else {
239 /*
240 * We could try to exhaustively detect cases
241 * under which we can skip the segment load,
242 * but there's really only one case that matters
243 * for performance: if both the previous and
244 * next states are fully zeroed, we can skip
245 * the load.
246 *
247 * (This assumes that prev_base == 0 has no
248 * false positives. This is the case on
249 * Intel-style CPUs.)
250 */
251 if (likely(prev_index | next_index | prev_base))
252 loadseg(which, next_index);
253 }
254 } else {
255 if (prev_index != next_index)
256 loadseg(which, next_index);
257 wrmsrl(which == FS ? MSR_FS_BASE : MSR_KERNEL_GS_BASE,
258 next_base);
259 }
260 } else {
261 /*
262 * The next task is using a real segment. Loading the selector
263 * is sufficient.
264 */
265 loadseg(which, next_index);
266 }
267 }
268
copy_thread_tls(unsigned long clone_flags,unsigned long sp,unsigned long arg,struct task_struct * p,unsigned long tls)269 int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
270 unsigned long arg, struct task_struct *p, unsigned long tls)
271 {
272 int err;
273 struct pt_regs *childregs;
274 struct fork_frame *fork_frame;
275 struct inactive_task_frame *frame;
276 struct task_struct *me = current;
277
278 childregs = task_pt_regs(p);
279 fork_frame = container_of(childregs, struct fork_frame, regs);
280 frame = &fork_frame->frame;
281
282 /*
283 * For a new task use the RESET flags value since there is no before.
284 * All the status flags are zero; DF and all the system flags must also
285 * be 0, specifically IF must be 0 because we context switch to the new
286 * task with interrupts disabled.
287 */
288 frame->flags = X86_EFLAGS_FIXED;
289 frame->bp = 0;
290 frame->ret_addr = (unsigned long) ret_from_fork;
291 p->thread.sp = (unsigned long) fork_frame;
292 p->thread.io_bitmap_ptr = NULL;
293
294 savesegment(gs, p->thread.gsindex);
295 p->thread.gsbase = p->thread.gsindex ? 0 : me->thread.gsbase;
296 savesegment(fs, p->thread.fsindex);
297 p->thread.fsbase = p->thread.fsindex ? 0 : me->thread.fsbase;
298 savesegment(es, p->thread.es);
299 savesegment(ds, p->thread.ds);
300 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
301
302 if (unlikely(p->flags & PF_KTHREAD)) {
303 /* kernel thread */
304 memset(childregs, 0, sizeof(struct pt_regs));
305 frame->bx = sp; /* function */
306 frame->r12 = arg;
307 return 0;
308 }
309 frame->bx = 0;
310 *childregs = *current_pt_regs();
311
312 childregs->ax = 0;
313 if (sp)
314 childregs->sp = sp;
315
316 err = -ENOMEM;
317 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
318 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
319 IO_BITMAP_BYTES, GFP_KERNEL);
320 if (!p->thread.io_bitmap_ptr) {
321 p->thread.io_bitmap_max = 0;
322 return -ENOMEM;
323 }
324 set_tsk_thread_flag(p, TIF_IO_BITMAP);
325 }
326
327 /*
328 * Set a new TLS for the child thread?
329 */
330 if (clone_flags & CLONE_SETTLS) {
331 #ifdef CONFIG_IA32_EMULATION
332 if (in_ia32_syscall())
333 err = do_set_thread_area(p, -1,
334 (struct user_desc __user *)tls, 0);
335 else
336 #endif
337 err = do_arch_prctl_64(p, ARCH_SET_FS, tls);
338 if (err)
339 goto out;
340 }
341 err = 0;
342 out:
343 if (err && p->thread.io_bitmap_ptr) {
344 kfree(p->thread.io_bitmap_ptr);
345 p->thread.io_bitmap_max = 0;
346 }
347
348 return err;
349 }
350
351 static void
start_thread_common(struct pt_regs * regs,unsigned long new_ip,unsigned long new_sp,unsigned int _cs,unsigned int _ss,unsigned int _ds)352 start_thread_common(struct pt_regs *regs, unsigned long new_ip,
353 unsigned long new_sp,
354 unsigned int _cs, unsigned int _ss, unsigned int _ds)
355 {
356 WARN_ON_ONCE(regs != current_pt_regs());
357
358 if (static_cpu_has(X86_BUG_NULL_SEG)) {
359 /* Loading zero below won't clear the base. */
360 loadsegment(fs, __USER_DS);
361 load_gs_index(__USER_DS);
362 }
363
364 loadsegment(fs, 0);
365 loadsegment(es, _ds);
366 loadsegment(ds, _ds);
367 load_gs_index(0);
368
369 regs->ip = new_ip;
370 regs->sp = new_sp;
371 regs->cs = _cs;
372 regs->ss = _ss;
373 regs->flags = X86_EFLAGS_IF;
374 force_iret();
375 }
376
377 void
start_thread(struct pt_regs * regs,unsigned long new_ip,unsigned long new_sp)378 start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
379 {
380 start_thread_common(regs, new_ip, new_sp,
381 __USER_CS, __USER_DS, 0);
382 }
383 EXPORT_SYMBOL_GPL(start_thread);
384
385 #ifdef CONFIG_COMPAT
compat_start_thread(struct pt_regs * regs,u32 new_ip,u32 new_sp)386 void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp)
387 {
388 start_thread_common(regs, new_ip, new_sp,
389 test_thread_flag(TIF_X32)
390 ? __USER_CS : __USER32_CS,
391 __USER_DS, __USER_DS);
392 }
393 #endif
394
395 /*
396 * switch_to(x,y) should switch tasks from x to y.
397 *
398 * This could still be optimized:
399 * - fold all the options into a flag word and test it with a single test.
400 * - could test fs/gs bitsliced
401 *
402 * Kprobes not supported here. Set the probe on schedule instead.
403 * Function graph tracer not supported too.
404 */
405 __visible __notrace_funcgraph struct task_struct *
__switch_to(struct task_struct * prev_p,struct task_struct * next_p)406 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
407 {
408 struct thread_struct *prev = &prev_p->thread;
409 struct thread_struct *next = &next_p->thread;
410 struct fpu *prev_fpu = &prev->fpu;
411 struct fpu *next_fpu = &next->fpu;
412 int cpu = smp_processor_id();
413
414 WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
415 this_cpu_read(irq_count) != -1);
416
417 switch_fpu_prepare(prev_fpu, cpu);
418
419 /* We must save %fs and %gs before load_TLS() because
420 * %fs and %gs may be cleared by load_TLS().
421 *
422 * (e.g. xen_load_tls())
423 */
424 save_fsgs(prev_p);
425
426 /*
427 * Load TLS before restoring any segments so that segment loads
428 * reference the correct GDT entries.
429 */
430 load_TLS(next, cpu);
431
432 /*
433 * Leave lazy mode, flushing any hypercalls made here. This
434 * must be done after loading TLS entries in the GDT but before
435 * loading segments that might reference them, and and it must
436 * be done before fpu__restore(), so the TS bit is up to
437 * date.
438 */
439 arch_end_context_switch(next_p);
440
441 /* Switch DS and ES.
442 *
443 * Reading them only returns the selectors, but writing them (if
444 * nonzero) loads the full descriptor from the GDT or LDT. The
445 * LDT for next is loaded in switch_mm, and the GDT is loaded
446 * above.
447 *
448 * We therefore need to write new values to the segment
449 * registers on every context switch unless both the new and old
450 * values are zero.
451 *
452 * Note that we don't need to do anything for CS and SS, as
453 * those are saved and restored as part of pt_regs.
454 */
455 savesegment(es, prev->es);
456 if (unlikely(next->es | prev->es))
457 loadsegment(es, next->es);
458
459 savesegment(ds, prev->ds);
460 if (unlikely(next->ds | prev->ds))
461 loadsegment(ds, next->ds);
462
463 load_seg_legacy(prev->fsindex, prev->fsbase,
464 next->fsindex, next->fsbase, FS);
465 load_seg_legacy(prev->gsindex, prev->gsbase,
466 next->gsindex, next->gsbase, GS);
467
468 switch_fpu_finish(next_fpu, cpu);
469
470 /*
471 * Switch the PDA and FPU contexts.
472 */
473 this_cpu_write(current_task, next_p);
474 this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));
475
476 /* Reload sp0. */
477 update_sp0(next_p);
478
479 __switch_to_xtra(prev_p, next_p);
480
481 #ifdef CONFIG_XEN_PV
482 /*
483 * On Xen PV, IOPL bits in pt_regs->flags have no effect, and
484 * current_pt_regs()->flags may not match the current task's
485 * intended IOPL. We need to switch it manually.
486 */
487 if (unlikely(static_cpu_has(X86_FEATURE_XENPV) &&
488 prev->iopl != next->iopl))
489 xen_set_iopl_mask(next->iopl);
490 #endif
491
492 if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) {
493 /*
494 * AMD CPUs have a misfeature: SYSRET sets the SS selector but
495 * does not update the cached descriptor. As a result, if we
496 * do SYSRET while SS is NULL, we'll end up in user mode with
497 * SS apparently equal to __USER_DS but actually unusable.
498 *
499 * The straightforward workaround would be to fix it up just
500 * before SYSRET, but that would slow down the system call
501 * fast paths. Instead, we ensure that SS is never NULL in
502 * system call context. We do this by replacing NULL SS
503 * selectors at every context switch. SYSCALL sets up a valid
504 * SS, so the only way to get NULL is to re-enter the kernel
505 * from CPL 3 through an interrupt. Since that can't happen
506 * in the same task as a running syscall, we are guaranteed to
507 * context switch between every interrupt vector entry and a
508 * subsequent SYSRET.
509 *
510 * We read SS first because SS reads are much faster than
511 * writes. Out of caution, we force SS to __KERNEL_DS even if
512 * it previously had a different non-NULL value.
513 */
514 unsigned short ss_sel;
515 savesegment(ss, ss_sel);
516 if (ss_sel != __KERNEL_DS)
517 loadsegment(ss, __KERNEL_DS);
518 }
519
520 /* Load the Intel cache allocation PQR MSR. */
521 intel_rdt_sched_in();
522
523 return prev_p;
524 }
525
set_personality_64bit(void)526 void set_personality_64bit(void)
527 {
528 /* inherit personality from parent */
529
530 /* Make sure to be in 64bit mode */
531 clear_thread_flag(TIF_IA32);
532 clear_thread_flag(TIF_ADDR32);
533 clear_thread_flag(TIF_X32);
534 /* Pretend that this comes from a 64bit execve */
535 task_pt_regs(current)->orig_ax = __NR_execve;
536 current_thread_info()->status &= ~TS_COMPAT;
537
538 /* Ensure the corresponding mm is not marked. */
539 if (current->mm)
540 current->mm->context.ia32_compat = 0;
541
542 /* TBD: overwrites user setup. Should have two bits.
543 But 64bit processes have always behaved this way,
544 so it's not too bad. The main problem is just that
545 32bit childs are affected again. */
546 current->personality &= ~READ_IMPLIES_EXEC;
547 }
548
__set_personality_x32(void)549 static void __set_personality_x32(void)
550 {
551 #ifdef CONFIG_X86_X32
552 clear_thread_flag(TIF_IA32);
553 set_thread_flag(TIF_X32);
554 if (current->mm)
555 current->mm->context.ia32_compat = TIF_X32;
556 current->personality &= ~READ_IMPLIES_EXEC;
557 /*
558 * in_compat_syscall() uses the presence of the x32 syscall bit
559 * flag to determine compat status. The x86 mmap() code relies on
560 * the syscall bitness so set x32 syscall bit right here to make
561 * in_compat_syscall() work during exec().
562 *
563 * Pretend to come from a x32 execve.
564 */
565 task_pt_regs(current)->orig_ax = __NR_x32_execve | __X32_SYSCALL_BIT;
566 current_thread_info()->status &= ~TS_COMPAT;
567 #endif
568 }
569
__set_personality_ia32(void)570 static void __set_personality_ia32(void)
571 {
572 #ifdef CONFIG_IA32_EMULATION
573 set_thread_flag(TIF_IA32);
574 clear_thread_flag(TIF_X32);
575 if (current->mm)
576 current->mm->context.ia32_compat = TIF_IA32;
577 current->personality |= force_personality32;
578 /* Prepare the first "return" to user space */
579 task_pt_regs(current)->orig_ax = __NR_ia32_execve;
580 current_thread_info()->status |= TS_COMPAT;
581 #endif
582 }
583
set_personality_ia32(bool x32)584 void set_personality_ia32(bool x32)
585 {
586 /* Make sure to be in 32bit mode */
587 set_thread_flag(TIF_ADDR32);
588
589 if (x32)
590 __set_personality_x32();
591 else
592 __set_personality_ia32();
593 }
594 EXPORT_SYMBOL_GPL(set_personality_ia32);
595
596 #ifdef CONFIG_CHECKPOINT_RESTORE
prctl_map_vdso(const struct vdso_image * image,unsigned long addr)597 static long prctl_map_vdso(const struct vdso_image *image, unsigned long addr)
598 {
599 int ret;
600
601 ret = map_vdso_once(image, addr);
602 if (ret)
603 return ret;
604
605 return (long)image->size;
606 }
607 #endif
608
do_arch_prctl_64(struct task_struct * task,int option,unsigned long arg2)609 long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
610 {
611 int ret = 0;
612 int doit = task == current;
613 int cpu;
614
615 switch (option) {
616 case ARCH_SET_GS:
617 if (arg2 >= TASK_SIZE_MAX)
618 return -EPERM;
619 cpu = get_cpu();
620 task->thread.gsindex = 0;
621 task->thread.gsbase = arg2;
622 if (doit) {
623 load_gs_index(0);
624 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, arg2);
625 }
626 put_cpu();
627 break;
628 case ARCH_SET_FS:
629 /* Not strictly needed for fs, but do it for symmetry
630 with gs */
631 if (arg2 >= TASK_SIZE_MAX)
632 return -EPERM;
633 cpu = get_cpu();
634 task->thread.fsindex = 0;
635 task->thread.fsbase = arg2;
636 if (doit) {
637 /* set the selector to 0 to not confuse __switch_to */
638 loadsegment(fs, 0);
639 ret = wrmsrl_safe(MSR_FS_BASE, arg2);
640 }
641 put_cpu();
642 break;
643 case ARCH_GET_FS: {
644 unsigned long base;
645
646 if (doit)
647 rdmsrl(MSR_FS_BASE, base);
648 else
649 base = task->thread.fsbase;
650 ret = put_user(base, (unsigned long __user *)arg2);
651 break;
652 }
653 case ARCH_GET_GS: {
654 unsigned long base;
655
656 if (doit)
657 rdmsrl(MSR_KERNEL_GS_BASE, base);
658 else
659 base = task->thread.gsbase;
660 ret = put_user(base, (unsigned long __user *)arg2);
661 break;
662 }
663
664 #ifdef CONFIG_CHECKPOINT_RESTORE
665 # ifdef CONFIG_X86_X32_ABI
666 case ARCH_MAP_VDSO_X32:
667 return prctl_map_vdso(&vdso_image_x32, arg2);
668 # endif
669 # if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
670 case ARCH_MAP_VDSO_32:
671 return prctl_map_vdso(&vdso_image_32, arg2);
672 # endif
673 case ARCH_MAP_VDSO_64:
674 return prctl_map_vdso(&vdso_image_64, arg2);
675 #endif
676
677 default:
678 ret = -EINVAL;
679 break;
680 }
681
682 return ret;
683 }
684
SYSCALL_DEFINE2(arch_prctl,int,option,unsigned long,arg2)685 SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
686 {
687 long ret;
688
689 ret = do_arch_prctl_64(current, option, arg2);
690 if (ret == -EINVAL)
691 ret = do_arch_prctl_common(current, option, arg2);
692
693 return ret;
694 }
695
696 #ifdef CONFIG_IA32_EMULATION
COMPAT_SYSCALL_DEFINE2(arch_prctl,int,option,unsigned long,arg2)697 COMPAT_SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
698 {
699 return do_arch_prctl_common(current, option, arg2);
700 }
701 #endif
702
KSTK_ESP(struct task_struct * task)703 unsigned long KSTK_ESP(struct task_struct *task)
704 {
705 return task_pt_regs(task)->sp;
706 }
707