1/* 2 * 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 */ 5 6/* 7 * entry.S contains the system-call and fault low-level handling routines. 8 * This also contains the timer-interrupt handler, as well as all interrupts 9 * and faults that can result in a task-switch. 10 * 11 * NOTE: This code handles signal-recognition, which happens every time 12 * after a timer-interrupt and after each system call. 13 * 14 * I changed all the .align's to 4 (16 byte alignment), as that's faster 15 * on a 486. 16 * 17 * Stack layout in 'syscall_exit': 18 * ptrace needs to have all regs on the stack. 19 * if the order here is changed, it needs to be 20 * updated in fork.c:copy_process, signal.c:do_signal, 21 * ptrace.c and ptrace.h 22 * 23 * 0(%esp) - %ebx 24 * 4(%esp) - %ecx 25 * 8(%esp) - %edx 26 * C(%esp) - %esi 27 * 10(%esp) - %edi 28 * 14(%esp) - %ebp 29 * 18(%esp) - %eax 30 * 1C(%esp) - %ds 31 * 20(%esp) - %es 32 * 24(%esp) - %fs 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 34 * 2C(%esp) - orig_eax 35 * 30(%esp) - %eip 36 * 34(%esp) - %cs 37 * 38(%esp) - %eflags 38 * 3C(%esp) - %oldesp 39 * 40(%esp) - %oldss 40 * 41 * "current" is in register %ebx during any slow entries. 42 */ 43 44#include <linux/linkage.h> 45#include <linux/err.h> 46#include <asm/thread_info.h> 47#include <asm/irqflags.h> 48#include <asm/errno.h> 49#include <asm/segment.h> 50#include <asm/smp.h> 51#include <asm/page_types.h> 52#include <asm/percpu.h> 53#include <asm/dwarf2.h> 54#include <asm/processor-flags.h> 55#include <asm/ftrace.h> 56#include <asm/irq_vectors.h> 57#include <asm/cpufeature.h> 58#include <asm/alternative-asm.h> 59#include <asm/asm.h> 60#include <asm/smap.h> 61 62/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 63#include <linux/elf-em.h> 64#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 65#define __AUDIT_ARCH_LE 0x40000000 66 67#ifndef CONFIG_AUDITSYSCALL 68#define sysenter_audit syscall_trace_entry 69#define sysexit_audit syscall_exit_work 70#endif 71 72 .section .entry.text, "ax" 73 74/* 75 * We use macros for low-level operations which need to be overridden 76 * for paravirtualization. The following will never clobber any registers: 77 * INTERRUPT_RETURN (aka. "iret") 78 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 79 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 80 * 81 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 82 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 83 * Allowing a register to be clobbered can shrink the paravirt replacement 84 * enough to patch inline, increasing performance. 85 */ 86 87#ifdef CONFIG_PREEMPT 88#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 89#else 90#define preempt_stop(clobbers) 91#define resume_kernel restore_all 92#endif 93 94.macro TRACE_IRQS_IRET 95#ifdef CONFIG_TRACE_IRQFLAGS 96 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 97 jz 1f 98 TRACE_IRQS_ON 991: 100#endif 101.endm 102 103/* 104 * User gs save/restore 105 * 106 * %gs is used for userland TLS and kernel only uses it for stack 107 * canary which is required to be at %gs:20 by gcc. Read the comment 108 * at the top of stackprotector.h for more info. 109 * 110 * Local labels 98 and 99 are used. 111 */ 112#ifdef CONFIG_X86_32_LAZY_GS 113 114 /* unfortunately push/pop can't be no-op */ 115.macro PUSH_GS 116 pushl_cfi $0 117.endm 118.macro POP_GS pop=0 119 addl $(4 + \pop), %esp 120 CFI_ADJUST_CFA_OFFSET -(4 + \pop) 121.endm 122.macro POP_GS_EX 123.endm 124 125 /* all the rest are no-op */ 126.macro PTGS_TO_GS 127.endm 128.macro PTGS_TO_GS_EX 129.endm 130.macro GS_TO_REG reg 131.endm 132.macro REG_TO_PTGS reg 133.endm 134.macro SET_KERNEL_GS reg 135.endm 136 137#else /* CONFIG_X86_32_LAZY_GS */ 138 139.macro PUSH_GS 140 pushl_cfi %gs 141 /*CFI_REL_OFFSET gs, 0*/ 142.endm 143 144.macro POP_GS pop=0 14598: popl_cfi %gs 146 /*CFI_RESTORE gs*/ 147 .if \pop <> 0 148 add $\pop, %esp 149 CFI_ADJUST_CFA_OFFSET -\pop 150 .endif 151.endm 152.macro POP_GS_EX 153.pushsection .fixup, "ax" 15499: movl $0, (%esp) 155 jmp 98b 156.popsection 157 _ASM_EXTABLE(98b,99b) 158.endm 159 160.macro PTGS_TO_GS 16198: mov PT_GS(%esp), %gs 162.endm 163.macro PTGS_TO_GS_EX 164.pushsection .fixup, "ax" 16599: movl $0, PT_GS(%esp) 166 jmp 98b 167.popsection 168 _ASM_EXTABLE(98b,99b) 169.endm 170 171.macro GS_TO_REG reg 172 movl %gs, \reg 173 /*CFI_REGISTER gs, \reg*/ 174.endm 175.macro REG_TO_PTGS reg 176 movl \reg, PT_GS(%esp) 177 /*CFI_REL_OFFSET gs, PT_GS*/ 178.endm 179.macro SET_KERNEL_GS reg 180 movl $(__KERNEL_STACK_CANARY), \reg 181 movl \reg, %gs 182.endm 183 184#endif /* CONFIG_X86_32_LAZY_GS */ 185 186.macro SAVE_ALL 187 cld 188 PUSH_GS 189 pushl_cfi %fs 190 /*CFI_REL_OFFSET fs, 0;*/ 191 pushl_cfi %es 192 /*CFI_REL_OFFSET es, 0;*/ 193 pushl_cfi %ds 194 /*CFI_REL_OFFSET ds, 0;*/ 195 pushl_cfi %eax 196 CFI_REL_OFFSET eax, 0 197 pushl_cfi %ebp 198 CFI_REL_OFFSET ebp, 0 199 pushl_cfi %edi 200 CFI_REL_OFFSET edi, 0 201 pushl_cfi %esi 202 CFI_REL_OFFSET esi, 0 203 pushl_cfi %edx 204 CFI_REL_OFFSET edx, 0 205 pushl_cfi %ecx 206 CFI_REL_OFFSET ecx, 0 207 pushl_cfi %ebx 208 CFI_REL_OFFSET ebx, 0 209 movl $(__USER_DS), %edx 210 movl %edx, %ds 211 movl %edx, %es 212 movl $(__KERNEL_PERCPU), %edx 213 movl %edx, %fs 214 SET_KERNEL_GS %edx 215.endm 216 217.macro RESTORE_INT_REGS 218 popl_cfi %ebx 219 CFI_RESTORE ebx 220 popl_cfi %ecx 221 CFI_RESTORE ecx 222 popl_cfi %edx 223 CFI_RESTORE edx 224 popl_cfi %esi 225 CFI_RESTORE esi 226 popl_cfi %edi 227 CFI_RESTORE edi 228 popl_cfi %ebp 229 CFI_RESTORE ebp 230 popl_cfi %eax 231 CFI_RESTORE eax 232.endm 233 234.macro RESTORE_REGS pop=0 235 RESTORE_INT_REGS 2361: popl_cfi %ds 237 /*CFI_RESTORE ds;*/ 2382: popl_cfi %es 239 /*CFI_RESTORE es;*/ 2403: popl_cfi %fs 241 /*CFI_RESTORE fs;*/ 242 POP_GS \pop 243.pushsection .fixup, "ax" 2444: movl $0, (%esp) 245 jmp 1b 2465: movl $0, (%esp) 247 jmp 2b 2486: movl $0, (%esp) 249 jmp 3b 250.popsection 251 _ASM_EXTABLE(1b,4b) 252 _ASM_EXTABLE(2b,5b) 253 _ASM_EXTABLE(3b,6b) 254 POP_GS_EX 255.endm 256 257.macro RING0_INT_FRAME 258 CFI_STARTPROC simple 259 CFI_SIGNAL_FRAME 260 CFI_DEF_CFA esp, 3*4 261 /*CFI_OFFSET cs, -2*4;*/ 262 CFI_OFFSET eip, -3*4 263.endm 264 265.macro RING0_EC_FRAME 266 CFI_STARTPROC simple 267 CFI_SIGNAL_FRAME 268 CFI_DEF_CFA esp, 4*4 269 /*CFI_OFFSET cs, -2*4;*/ 270 CFI_OFFSET eip, -3*4 271.endm 272 273.macro RING0_PTREGS_FRAME 274 CFI_STARTPROC simple 275 CFI_SIGNAL_FRAME 276 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 277 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 278 CFI_OFFSET eip, PT_EIP-PT_OLDESP 279 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 280 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 281 CFI_OFFSET eax, PT_EAX-PT_OLDESP 282 CFI_OFFSET ebp, PT_EBP-PT_OLDESP 283 CFI_OFFSET edi, PT_EDI-PT_OLDESP 284 CFI_OFFSET esi, PT_ESI-PT_OLDESP 285 CFI_OFFSET edx, PT_EDX-PT_OLDESP 286 CFI_OFFSET ecx, PT_ECX-PT_OLDESP 287 CFI_OFFSET ebx, PT_EBX-PT_OLDESP 288.endm 289 290ENTRY(ret_from_fork) 291 CFI_STARTPROC 292 pushl_cfi %eax 293 call schedule_tail 294 GET_THREAD_INFO(%ebp) 295 popl_cfi %eax 296 pushl_cfi $0x0202 # Reset kernel eflags 297 popfl_cfi 298 jmp syscall_exit 299 CFI_ENDPROC 300END(ret_from_fork) 301 302ENTRY(ret_from_kernel_thread) 303 CFI_STARTPROC 304 pushl_cfi %eax 305 call schedule_tail 306 GET_THREAD_INFO(%ebp) 307 popl_cfi %eax 308 pushl_cfi $0x0202 # Reset kernel eflags 309 popfl_cfi 310 movl PT_EBP(%esp),%eax 311 call *PT_EBX(%esp) 312 movl $0,PT_EAX(%esp) 313 jmp syscall_exit 314 CFI_ENDPROC 315ENDPROC(ret_from_kernel_thread) 316 317/* 318 * Interrupt exit functions should be protected against kprobes 319 */ 320 .pushsection .kprobes.text, "ax" 321/* 322 * Return to user mode is not as complex as all this looks, 323 * but we want the default path for a system call return to 324 * go as quickly as possible which is why some of this is 325 * less clear than it otherwise should be. 326 */ 327 328 # userspace resumption stub bypassing syscall exit tracing 329 ALIGN 330 RING0_PTREGS_FRAME 331ret_from_exception: 332 preempt_stop(CLBR_ANY) 333ret_from_intr: 334 GET_THREAD_INFO(%ebp) 335#ifdef CONFIG_VM86 336 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 337 movb PT_CS(%esp), %al 338 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 339#else 340 /* 341 * We can be coming here from child spawned by kernel_thread(). 342 */ 343 movl PT_CS(%esp), %eax 344 andl $SEGMENT_RPL_MASK, %eax 345#endif 346 cmpl $USER_RPL, %eax 347 jb resume_kernel # not returning to v8086 or userspace 348 349ENTRY(resume_userspace) 350 LOCKDEP_SYS_EXIT 351 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 352 # setting need_resched or sigpending 353 # between sampling and the iret 354 TRACE_IRQS_OFF 355 movl TI_flags(%ebp), %ecx 356 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 357 # int/exception return? 358 jne work_pending 359 jmp restore_all 360END(ret_from_exception) 361 362#ifdef CONFIG_PREEMPT 363ENTRY(resume_kernel) 364 DISABLE_INTERRUPTS(CLBR_ANY) 365 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 366 jnz restore_all 367need_resched: 368 movl TI_flags(%ebp), %ecx # need_resched set ? 369 testb $_TIF_NEED_RESCHED, %cl 370 jz restore_all 371 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 372 jz restore_all 373 call preempt_schedule_irq 374 jmp need_resched 375END(resume_kernel) 376#endif 377 CFI_ENDPROC 378/* 379 * End of kprobes section 380 */ 381 .popsection 382 383/* SYSENTER_RETURN points to after the "sysenter" instruction in 384 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 385 386 # sysenter call handler stub 387ENTRY(ia32_sysenter_target) 388 CFI_STARTPROC simple 389 CFI_SIGNAL_FRAME 390 CFI_DEF_CFA esp, 0 391 CFI_REGISTER esp, ebp 392 movl TSS_sysenter_sp0(%esp),%esp 393sysenter_past_esp: 394 /* 395 * Interrupts are disabled here, but we can't trace it until 396 * enough kernel state to call TRACE_IRQS_OFF can be called - but 397 * we immediately enable interrupts at that point anyway. 398 */ 399 pushl_cfi $__USER_DS 400 /*CFI_REL_OFFSET ss, 0*/ 401 pushl_cfi %ebp 402 CFI_REL_OFFSET esp, 0 403 pushfl_cfi 404 orl $X86_EFLAGS_IF, (%esp) 405 pushl_cfi $__USER_CS 406 /*CFI_REL_OFFSET cs, 0*/ 407 /* 408 * Push current_thread_info()->sysenter_return to the stack. 409 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 410 * pushed above; +8 corresponds to copy_thread's esp0 setting. 411 */ 412 pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp) 413 CFI_REL_OFFSET eip, 0 414 415 pushl_cfi %eax 416 SAVE_ALL 417 ENABLE_INTERRUPTS(CLBR_NONE) 418 419/* 420 * Load the potential sixth argument from user stack. 421 * Careful about security. 422 */ 423 cmpl $__PAGE_OFFSET-3,%ebp 424 jae syscall_fault 425 ASM_STAC 4261: movl (%ebp),%ebp 427 ASM_CLAC 428 movl %ebp,PT_EBP(%esp) 429 _ASM_EXTABLE(1b,syscall_fault) 430 431 GET_THREAD_INFO(%ebp) 432 433 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 434 jnz sysenter_audit 435sysenter_do_call: 436 cmpl $(NR_syscalls), %eax 437 jae syscall_badsys 438 call *sys_call_table(,%eax,4) 439 movl %eax,PT_EAX(%esp) 440 LOCKDEP_SYS_EXIT 441 DISABLE_INTERRUPTS(CLBR_ANY) 442 TRACE_IRQS_OFF 443 movl TI_flags(%ebp), %ecx 444 testl $_TIF_ALLWORK_MASK, %ecx 445 jne sysexit_audit 446sysenter_exit: 447/* if something modifies registers it must also disable sysexit */ 448 movl PT_EIP(%esp), %edx 449 movl PT_OLDESP(%esp), %ecx 450 xorl %ebp,%ebp 451 TRACE_IRQS_ON 4521: mov PT_FS(%esp), %fs 453 PTGS_TO_GS 454 ENABLE_INTERRUPTS_SYSEXIT 455 456#ifdef CONFIG_AUDITSYSCALL 457sysenter_audit: 458 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 459 jnz syscall_trace_entry 460 addl $4,%esp 461 CFI_ADJUST_CFA_OFFSET -4 462 /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 463 /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 464 /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 465 movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 466 movl %eax,%edx /* 2nd arg: syscall number */ 467 movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 468 call __audit_syscall_entry 469 pushl_cfi %ebx 470 movl PT_EAX(%esp),%eax /* reload syscall number */ 471 jmp sysenter_do_call 472 473sysexit_audit: 474 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 475 jne syscall_exit_work 476 TRACE_IRQS_ON 477 ENABLE_INTERRUPTS(CLBR_ANY) 478 movl %eax,%edx /* second arg, syscall return value */ 479 cmpl $-MAX_ERRNO,%eax /* is it an error ? */ 480 setbe %al /* 1 if so, 0 if not */ 481 movzbl %al,%eax /* zero-extend that */ 482 call __audit_syscall_exit 483 DISABLE_INTERRUPTS(CLBR_ANY) 484 TRACE_IRQS_OFF 485 movl TI_flags(%ebp), %ecx 486 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 487 jne syscall_exit_work 488 movl PT_EAX(%esp),%eax /* reload syscall return value */ 489 jmp sysenter_exit 490#endif 491 492 CFI_ENDPROC 493.pushsection .fixup,"ax" 4942: movl $0,PT_FS(%esp) 495 jmp 1b 496.popsection 497 _ASM_EXTABLE(1b,2b) 498 PTGS_TO_GS_EX 499ENDPROC(ia32_sysenter_target) 500 501/* 502 * syscall stub including irq exit should be protected against kprobes 503 */ 504 .pushsection .kprobes.text, "ax" 505 # system call handler stub 506ENTRY(system_call) 507 RING0_INT_FRAME # can't unwind into user space anyway 508 ASM_CLAC 509 pushl_cfi %eax # save orig_eax 510 SAVE_ALL 511 GET_THREAD_INFO(%ebp) 512 # system call tracing in operation / emulation 513 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 514 jnz syscall_trace_entry 515 cmpl $(NR_syscalls), %eax 516 jae syscall_badsys 517syscall_call: 518 call *sys_call_table(,%eax,4) 519 movl %eax,PT_EAX(%esp) # store the return value 520syscall_exit: 521 LOCKDEP_SYS_EXIT 522 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 523 # setting need_resched or sigpending 524 # between sampling and the iret 525 TRACE_IRQS_OFF 526 movl TI_flags(%ebp), %ecx 527 testl $_TIF_ALLWORK_MASK, %ecx # current->work 528 jne syscall_exit_work 529 530restore_all: 531 TRACE_IRQS_IRET 532restore_all_notrace: 533#ifdef CONFIG_X86_ESPFIX32 534 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 535 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 536 # are returning to the kernel. 537 # See comments in process.c:copy_thread() for details. 538 movb PT_OLDSS(%esp), %ah 539 movb PT_CS(%esp), %al 540 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 541 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 542 CFI_REMEMBER_STATE 543 je ldt_ss # returning to user-space with LDT SS 544#endif 545restore_nocheck: 546 RESTORE_REGS 4 # skip orig_eax/error_code 547irq_return: 548 INTERRUPT_RETURN 549.section .fixup,"ax" 550ENTRY(iret_exc) 551 pushl $0 # no error code 552 pushl $do_iret_error 553 jmp error_code 554.previous 555 _ASM_EXTABLE(irq_return,iret_exc) 556 557#ifdef CONFIG_X86_ESPFIX32 558 CFI_RESTORE_STATE 559ldt_ss: 560#ifdef CONFIG_PARAVIRT 561 /* 562 * The kernel can't run on a non-flat stack if paravirt mode 563 * is active. Rather than try to fixup the high bits of 564 * ESP, bypass this code entirely. This may break DOSemu 565 * and/or Wine support in a paravirt VM, although the option 566 * is still available to implement the setting of the high 567 * 16-bits in the INTERRUPT_RETURN paravirt-op. 568 */ 569 cmpl $0, pv_info+PARAVIRT_enabled 570 jne restore_nocheck 571#endif 572 573/* 574 * Setup and switch to ESPFIX stack 575 * 576 * We're returning to userspace with a 16 bit stack. The CPU will not 577 * restore the high word of ESP for us on executing iret... This is an 578 * "official" bug of all the x86-compatible CPUs, which we can work 579 * around to make dosemu and wine happy. We do this by preloading the 580 * high word of ESP with the high word of the userspace ESP while 581 * compensating for the offset by changing to the ESPFIX segment with 582 * a base address that matches for the difference. 583 */ 584#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 585 mov %esp, %edx /* load kernel esp */ 586 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 587 mov %dx, %ax /* eax: new kernel esp */ 588 sub %eax, %edx /* offset (low word is 0) */ 589 shr $16, %edx 590 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 591 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 592 pushl_cfi $__ESPFIX_SS 593 pushl_cfi %eax /* new kernel esp */ 594 /* Disable interrupts, but do not irqtrace this section: we 595 * will soon execute iret and the tracer was already set to 596 * the irqstate after the iret */ 597 DISABLE_INTERRUPTS(CLBR_EAX) 598 lss (%esp), %esp /* switch to espfix segment */ 599 CFI_ADJUST_CFA_OFFSET -8 600 jmp restore_nocheck 601#endif 602 CFI_ENDPROC 603ENDPROC(system_call) 604 605 # perform work that needs to be done immediately before resumption 606 ALIGN 607 RING0_PTREGS_FRAME # can't unwind into user space anyway 608work_pending: 609 testb $_TIF_NEED_RESCHED, %cl 610 jz work_notifysig 611work_resched: 612 call schedule 613 LOCKDEP_SYS_EXIT 614 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 615 # setting need_resched or sigpending 616 # between sampling and the iret 617 TRACE_IRQS_OFF 618 movl TI_flags(%ebp), %ecx 619 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 620 # than syscall tracing? 621 jz restore_all 622 testb $_TIF_NEED_RESCHED, %cl 623 jnz work_resched 624 625work_notifysig: # deal with pending signals and 626 # notify-resume requests 627#ifdef CONFIG_VM86 628 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 629 movl %esp, %eax 630 jne work_notifysig_v86 # returning to kernel-space or 631 # vm86-space 6321: 633#else 634 movl %esp, %eax 635#endif 636 TRACE_IRQS_ON 637 ENABLE_INTERRUPTS(CLBR_NONE) 638 movb PT_CS(%esp), %bl 639 andb $SEGMENT_RPL_MASK, %bl 640 cmpb $USER_RPL, %bl 641 jb resume_kernel 642 xorl %edx, %edx 643 call do_notify_resume 644 jmp resume_userspace 645 646#ifdef CONFIG_VM86 647 ALIGN 648work_notifysig_v86: 649 pushl_cfi %ecx # save ti_flags for do_notify_resume 650 call save_v86_state # %eax contains pt_regs pointer 651 popl_cfi %ecx 652 movl %eax, %esp 653 jmp 1b 654#endif 655END(work_pending) 656 657 # perform syscall exit tracing 658 ALIGN 659syscall_trace_entry: 660 movl $-ENOSYS,PT_EAX(%esp) 661 movl %esp, %eax 662 call syscall_trace_enter 663 /* What it returned is what we'll actually use. */ 664 cmpl $(NR_syscalls), %eax 665 jnae syscall_call 666 jmp syscall_exit 667END(syscall_trace_entry) 668 669 # perform syscall exit tracing 670 ALIGN 671syscall_exit_work: 672 testl $_TIF_WORK_SYSCALL_EXIT, %ecx 673 jz work_pending 674 TRACE_IRQS_ON 675 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 676 # schedule() instead 677 movl %esp, %eax 678 call syscall_trace_leave 679 jmp resume_userspace 680END(syscall_exit_work) 681 CFI_ENDPROC 682 683 RING0_INT_FRAME # can't unwind into user space anyway 684syscall_fault: 685 ASM_CLAC 686 GET_THREAD_INFO(%ebp) 687 movl $-EFAULT,PT_EAX(%esp) 688 jmp resume_userspace 689END(syscall_fault) 690 691syscall_badsys: 692 movl $-ENOSYS,PT_EAX(%esp) 693 jmp resume_userspace 694END(syscall_badsys) 695 CFI_ENDPROC 696/* 697 * End of kprobes section 698 */ 699 .popsection 700 701.macro FIXUP_ESPFIX_STACK 702/* 703 * Switch back for ESPFIX stack to the normal zerobased stack 704 * 705 * We can't call C functions using the ESPFIX stack. This code reads 706 * the high word of the segment base from the GDT and swiches to the 707 * normal stack and adjusts ESP with the matching offset. 708 */ 709#ifdef CONFIG_X86_ESPFIX32 710 /* fixup the stack */ 711 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 712 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 713 shl $16, %eax 714 addl %esp, %eax /* the adjusted stack pointer */ 715 pushl_cfi $__KERNEL_DS 716 pushl_cfi %eax 717 lss (%esp), %esp /* switch to the normal stack segment */ 718 CFI_ADJUST_CFA_OFFSET -8 719#endif 720.endm 721.macro UNWIND_ESPFIX_STACK 722#ifdef CONFIG_X86_ESPFIX32 723 movl %ss, %eax 724 /* see if on espfix stack */ 725 cmpw $__ESPFIX_SS, %ax 726 jne 27f 727 movl $__KERNEL_DS, %eax 728 movl %eax, %ds 729 movl %eax, %es 730 /* switch to normal stack */ 731 FIXUP_ESPFIX_STACK 73227: 733#endif 734.endm 735 736/* 737 * Build the entry stubs and pointer table with some assembler magic. 738 * We pack 7 stubs into a single 32-byte chunk, which will fit in a 739 * single cache line on all modern x86 implementations. 740 */ 741.section .init.rodata,"a" 742ENTRY(interrupt) 743.section .entry.text, "ax" 744 .p2align 5 745 .p2align CONFIG_X86_L1_CACHE_SHIFT 746ENTRY(irq_entries_start) 747 RING0_INT_FRAME 748vector=FIRST_EXTERNAL_VECTOR 749.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 750 .balign 32 751 .rept 7 752 .if vector < NR_VECTORS 753 .if vector <> FIRST_EXTERNAL_VECTOR 754 CFI_ADJUST_CFA_OFFSET -4 755 .endif 7561: pushl_cfi $(~vector+0x80) /* Note: always in signed byte range */ 757 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 758 jmp 2f 759 .endif 760 .previous 761 .long 1b 762 .section .entry.text, "ax" 763vector=vector+1 764 .endif 765 .endr 7662: jmp common_interrupt 767.endr 768END(irq_entries_start) 769 770.previous 771END(interrupt) 772.previous 773 774/* 775 * the CPU automatically disables interrupts when executing an IRQ vector, 776 * so IRQ-flags tracing has to follow that: 777 */ 778 .p2align CONFIG_X86_L1_CACHE_SHIFT 779common_interrupt: 780 ASM_CLAC 781 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 782 SAVE_ALL 783 TRACE_IRQS_OFF 784 movl %esp,%eax 785 call do_IRQ 786 jmp ret_from_intr 787ENDPROC(common_interrupt) 788 CFI_ENDPROC 789 790/* 791 * Irq entries should be protected against kprobes 792 */ 793 .pushsection .kprobes.text, "ax" 794#define BUILD_INTERRUPT3(name, nr, fn) \ 795ENTRY(name) \ 796 RING0_INT_FRAME; \ 797 ASM_CLAC; \ 798 pushl_cfi $~(nr); \ 799 SAVE_ALL; \ 800 TRACE_IRQS_OFF \ 801 movl %esp,%eax; \ 802 call fn; \ 803 jmp ret_from_intr; \ 804 CFI_ENDPROC; \ 805ENDPROC(name) 806 807#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name) 808 809/* The include is where all of the SMP etc. interrupts come from */ 810#include <asm/entry_arch.h> 811 812ENTRY(coprocessor_error) 813 RING0_INT_FRAME 814 ASM_CLAC 815 pushl_cfi $0 816 pushl_cfi $do_coprocessor_error 817 jmp error_code 818 CFI_ENDPROC 819END(coprocessor_error) 820 821ENTRY(simd_coprocessor_error) 822 RING0_INT_FRAME 823 ASM_CLAC 824 pushl_cfi $0 825#ifdef CONFIG_X86_INVD_BUG 826 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 827661: pushl_cfi $do_general_protection 828662: 829.section .altinstructions,"a" 830 altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f 831.previous 832.section .altinstr_replacement,"ax" 833663: pushl $do_simd_coprocessor_error 834664: 835.previous 836#else 837 pushl_cfi $do_simd_coprocessor_error 838#endif 839 jmp error_code 840 CFI_ENDPROC 841END(simd_coprocessor_error) 842 843ENTRY(device_not_available) 844 RING0_INT_FRAME 845 ASM_CLAC 846 pushl_cfi $-1 # mark this as an int 847 pushl_cfi $do_device_not_available 848 jmp error_code 849 CFI_ENDPROC 850END(device_not_available) 851 852#ifdef CONFIG_PARAVIRT 853ENTRY(native_iret) 854 iret 855 _ASM_EXTABLE(native_iret, iret_exc) 856END(native_iret) 857 858ENTRY(native_irq_enable_sysexit) 859 sti 860 sysexit 861END(native_irq_enable_sysexit) 862#endif 863 864ENTRY(overflow) 865 RING0_INT_FRAME 866 ASM_CLAC 867 pushl_cfi $0 868 pushl_cfi $do_overflow 869 jmp error_code 870 CFI_ENDPROC 871END(overflow) 872 873ENTRY(bounds) 874 RING0_INT_FRAME 875 ASM_CLAC 876 pushl_cfi $0 877 pushl_cfi $do_bounds 878 jmp error_code 879 CFI_ENDPROC 880END(bounds) 881 882ENTRY(invalid_op) 883 RING0_INT_FRAME 884 ASM_CLAC 885 pushl_cfi $0 886 pushl_cfi $do_invalid_op 887 jmp error_code 888 CFI_ENDPROC 889END(invalid_op) 890 891ENTRY(coprocessor_segment_overrun) 892 RING0_INT_FRAME 893 ASM_CLAC 894 pushl_cfi $0 895 pushl_cfi $do_coprocessor_segment_overrun 896 jmp error_code 897 CFI_ENDPROC 898END(coprocessor_segment_overrun) 899 900ENTRY(invalid_TSS) 901 RING0_EC_FRAME 902 ASM_CLAC 903 pushl_cfi $do_invalid_TSS 904 jmp error_code 905 CFI_ENDPROC 906END(invalid_TSS) 907 908ENTRY(segment_not_present) 909 RING0_EC_FRAME 910 ASM_CLAC 911 pushl_cfi $do_segment_not_present 912 jmp error_code 913 CFI_ENDPROC 914END(segment_not_present) 915 916ENTRY(stack_segment) 917 RING0_EC_FRAME 918 ASM_CLAC 919 pushl_cfi $do_stack_segment 920 jmp error_code 921 CFI_ENDPROC 922END(stack_segment) 923 924ENTRY(alignment_check) 925 RING0_EC_FRAME 926 ASM_CLAC 927 pushl_cfi $do_alignment_check 928 jmp error_code 929 CFI_ENDPROC 930END(alignment_check) 931 932ENTRY(divide_error) 933 RING0_INT_FRAME 934 ASM_CLAC 935 pushl_cfi $0 # no error code 936 pushl_cfi $do_divide_error 937 jmp error_code 938 CFI_ENDPROC 939END(divide_error) 940 941#ifdef CONFIG_X86_MCE 942ENTRY(machine_check) 943 RING0_INT_FRAME 944 ASM_CLAC 945 pushl_cfi $0 946 pushl_cfi machine_check_vector 947 jmp error_code 948 CFI_ENDPROC 949END(machine_check) 950#endif 951 952ENTRY(spurious_interrupt_bug) 953 RING0_INT_FRAME 954 ASM_CLAC 955 pushl_cfi $0 956 pushl_cfi $do_spurious_interrupt_bug 957 jmp error_code 958 CFI_ENDPROC 959END(spurious_interrupt_bug) 960/* 961 * End of kprobes section 962 */ 963 .popsection 964 965#ifdef CONFIG_XEN 966/* Xen doesn't set %esp to be precisely what the normal sysenter 967 entrypoint expects, so fix it up before using the normal path. */ 968ENTRY(xen_sysenter_target) 969 RING0_INT_FRAME 970 addl $5*4, %esp /* remove xen-provided frame */ 971 CFI_ADJUST_CFA_OFFSET -5*4 972 jmp sysenter_past_esp 973 CFI_ENDPROC 974 975ENTRY(xen_hypervisor_callback) 976 CFI_STARTPROC 977 pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 978 SAVE_ALL 979 TRACE_IRQS_OFF 980 981 /* Check to see if we got the event in the critical 982 region in xen_iret_direct, after we've reenabled 983 events and checked for pending events. This simulates 984 iret instruction's behaviour where it delivers a 985 pending interrupt when enabling interrupts. */ 986 movl PT_EIP(%esp),%eax 987 cmpl $xen_iret_start_crit,%eax 988 jb 1f 989 cmpl $xen_iret_end_crit,%eax 990 jae 1f 991 992 jmp xen_iret_crit_fixup 993 994ENTRY(xen_do_upcall) 9951: mov %esp, %eax 996 call xen_evtchn_do_upcall 997 jmp ret_from_intr 998 CFI_ENDPROC 999ENDPROC(xen_hypervisor_callback) 1000 1001# Hypervisor uses this for application faults while it executes. 1002# We get here for two reasons: 1003# 1. Fault while reloading DS, ES, FS or GS 1004# 2. Fault while executing IRET 1005# Category 1 we fix up by reattempting the load, and zeroing the segment 1006# register if the load fails. 1007# Category 2 we fix up by jumping to do_iret_error. We cannot use the 1008# normal Linux return path in this case because if we use the IRET hypercall 1009# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1010# We distinguish between categories by maintaining a status value in EAX. 1011ENTRY(xen_failsafe_callback) 1012 CFI_STARTPROC 1013 pushl_cfi %eax 1014 movl $1,%eax 10151: mov 4(%esp),%ds 10162: mov 8(%esp),%es 10173: mov 12(%esp),%fs 10184: mov 16(%esp),%gs 1019 /* EAX == 0 => Category 1 (Bad segment) 1020 EAX != 0 => Category 2 (Bad IRET) */ 1021 testl %eax,%eax 1022 popl_cfi %eax 1023 lea 16(%esp),%esp 1024 CFI_ADJUST_CFA_OFFSET -16 1025 jz 5f 1026 jmp iret_exc 10275: pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 1028 SAVE_ALL 1029 jmp ret_from_exception 1030 CFI_ENDPROC 1031 1032.section .fixup,"ax" 10336: xorl %eax,%eax 1034 movl %eax,4(%esp) 1035 jmp 1b 10367: xorl %eax,%eax 1037 movl %eax,8(%esp) 1038 jmp 2b 10398: xorl %eax,%eax 1040 movl %eax,12(%esp) 1041 jmp 3b 10429: xorl %eax,%eax 1043 movl %eax,16(%esp) 1044 jmp 4b 1045.previous 1046 _ASM_EXTABLE(1b,6b) 1047 _ASM_EXTABLE(2b,7b) 1048 _ASM_EXTABLE(3b,8b) 1049 _ASM_EXTABLE(4b,9b) 1050ENDPROC(xen_failsafe_callback) 1051 1052BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1053 xen_evtchn_do_upcall) 1054 1055#endif /* CONFIG_XEN */ 1056 1057#if IS_ENABLED(CONFIG_HYPERV) 1058 1059BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1060 hyperv_vector_handler) 1061 1062#endif /* CONFIG_HYPERV */ 1063 1064#ifdef CONFIG_FUNCTION_TRACER 1065#ifdef CONFIG_DYNAMIC_FTRACE 1066 1067ENTRY(mcount) 1068 ret 1069END(mcount) 1070 1071ENTRY(ftrace_caller) 1072 cmpl $0, function_trace_stop 1073 jne ftrace_stub 1074 1075 pushl %eax 1076 pushl %ecx 1077 pushl %edx 1078 pushl $0 /* Pass NULL as regs pointer */ 1079 movl 4*4(%esp), %eax 1080 movl 0x4(%ebp), %edx 1081 leal function_trace_op, %ecx 1082 subl $MCOUNT_INSN_SIZE, %eax 1083 1084.globl ftrace_call 1085ftrace_call: 1086 call ftrace_stub 1087 1088 addl $4,%esp /* skip NULL pointer */ 1089 popl %edx 1090 popl %ecx 1091 popl %eax 1092ftrace_ret: 1093#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1094.globl ftrace_graph_call 1095ftrace_graph_call: 1096 jmp ftrace_stub 1097#endif 1098 1099.globl ftrace_stub 1100ftrace_stub: 1101 ret 1102END(ftrace_caller) 1103 1104ENTRY(ftrace_regs_caller) 1105 pushf /* push flags before compare (in cs location) */ 1106 cmpl $0, function_trace_stop 1107 jne ftrace_restore_flags 1108 1109 /* 1110 * i386 does not save SS and ESP when coming from kernel. 1111 * Instead, to get sp, ®s->sp is used (see ptrace.h). 1112 * Unfortunately, that means eflags must be at the same location 1113 * as the current return ip is. We move the return ip into the 1114 * ip location, and move flags into the return ip location. 1115 */ 1116 pushl 4(%esp) /* save return ip into ip slot */ 1117 1118 pushl $0 /* Load 0 into orig_ax */ 1119 pushl %gs 1120 pushl %fs 1121 pushl %es 1122 pushl %ds 1123 pushl %eax 1124 pushl %ebp 1125 pushl %edi 1126 pushl %esi 1127 pushl %edx 1128 pushl %ecx 1129 pushl %ebx 1130 1131 movl 13*4(%esp), %eax /* Get the saved flags */ 1132 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 1133 /* clobbering return ip */ 1134 movl $__KERNEL_CS,13*4(%esp) 1135 1136 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 1137 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 1138 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 1139 leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 1140 pushl %esp /* Save pt_regs as 4th parameter */ 1141 1142GLOBAL(ftrace_regs_call) 1143 call ftrace_stub 1144 1145 addl $4, %esp /* Skip pt_regs */ 1146 movl 14*4(%esp), %eax /* Move flags back into cs */ 1147 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 1148 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 1149 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 1150 1151 popl %ebx 1152 popl %ecx 1153 popl %edx 1154 popl %esi 1155 popl %edi 1156 popl %ebp 1157 popl %eax 1158 popl %ds 1159 popl %es 1160 popl %fs 1161 popl %gs 1162 addl $8, %esp /* Skip orig_ax and ip */ 1163 popf /* Pop flags at end (no addl to corrupt flags) */ 1164 jmp ftrace_ret 1165 1166ftrace_restore_flags: 1167 popf 1168 jmp ftrace_stub 1169#else /* ! CONFIG_DYNAMIC_FTRACE */ 1170 1171ENTRY(mcount) 1172 cmpl $0, function_trace_stop 1173 jne ftrace_stub 1174 1175 cmpl $ftrace_stub, ftrace_trace_function 1176 jnz trace 1177#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1178 cmpl $ftrace_stub, ftrace_graph_return 1179 jnz ftrace_graph_caller 1180 1181 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1182 jnz ftrace_graph_caller 1183#endif 1184.globl ftrace_stub 1185ftrace_stub: 1186 ret 1187 1188 /* taken from glibc */ 1189trace: 1190 pushl %eax 1191 pushl %ecx 1192 pushl %edx 1193 movl 0xc(%esp), %eax 1194 movl 0x4(%ebp), %edx 1195 subl $MCOUNT_INSN_SIZE, %eax 1196 1197 call *ftrace_trace_function 1198 1199 popl %edx 1200 popl %ecx 1201 popl %eax 1202 jmp ftrace_stub 1203END(mcount) 1204#endif /* CONFIG_DYNAMIC_FTRACE */ 1205#endif /* CONFIG_FUNCTION_TRACER */ 1206 1207#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1208ENTRY(ftrace_graph_caller) 1209 pushl %eax 1210 pushl %ecx 1211 pushl %edx 1212 movl 0xc(%esp), %edx 1213 lea 0x4(%ebp), %eax 1214 movl (%ebp), %ecx 1215 subl $MCOUNT_INSN_SIZE, %edx 1216 call prepare_ftrace_return 1217 popl %edx 1218 popl %ecx 1219 popl %eax 1220 ret 1221END(ftrace_graph_caller) 1222 1223.globl return_to_handler 1224return_to_handler: 1225 pushl %eax 1226 pushl %edx 1227 movl %ebp, %eax 1228 call ftrace_return_to_handler 1229 movl %eax, %ecx 1230 popl %edx 1231 popl %eax 1232 jmp *%ecx 1233#endif 1234 1235/* 1236 * Some functions should be protected against kprobes 1237 */ 1238 .pushsection .kprobes.text, "ax" 1239 1240ENTRY(page_fault) 1241 RING0_EC_FRAME 1242 ASM_CLAC 1243 pushl_cfi $do_page_fault 1244 ALIGN 1245error_code: 1246 /* the function address is in %gs's slot on the stack */ 1247 pushl_cfi %fs 1248 /*CFI_REL_OFFSET fs, 0*/ 1249 pushl_cfi %es 1250 /*CFI_REL_OFFSET es, 0*/ 1251 pushl_cfi %ds 1252 /*CFI_REL_OFFSET ds, 0*/ 1253 pushl_cfi %eax 1254 CFI_REL_OFFSET eax, 0 1255 pushl_cfi %ebp 1256 CFI_REL_OFFSET ebp, 0 1257 pushl_cfi %edi 1258 CFI_REL_OFFSET edi, 0 1259 pushl_cfi %esi 1260 CFI_REL_OFFSET esi, 0 1261 pushl_cfi %edx 1262 CFI_REL_OFFSET edx, 0 1263 pushl_cfi %ecx 1264 CFI_REL_OFFSET ecx, 0 1265 pushl_cfi %ebx 1266 CFI_REL_OFFSET ebx, 0 1267 cld 1268 movl $(__KERNEL_PERCPU), %ecx 1269 movl %ecx, %fs 1270 UNWIND_ESPFIX_STACK 1271 GS_TO_REG %ecx 1272 movl PT_GS(%esp), %edi # get the function address 1273 movl PT_ORIG_EAX(%esp), %edx # get the error code 1274 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1275 REG_TO_PTGS %ecx 1276 SET_KERNEL_GS %ecx 1277 movl $(__USER_DS), %ecx 1278 movl %ecx, %ds 1279 movl %ecx, %es 1280 TRACE_IRQS_OFF 1281 movl %esp,%eax # pt_regs pointer 1282 call *%edi 1283 jmp ret_from_exception 1284 CFI_ENDPROC 1285END(page_fault) 1286 1287/* 1288 * Debug traps and NMI can happen at the one SYSENTER instruction 1289 * that sets up the real kernel stack. Check here, since we can't 1290 * allow the wrong stack to be used. 1291 * 1292 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1293 * already pushed 3 words if it hits on the sysenter instruction: 1294 * eflags, cs and eip. 1295 * 1296 * We just load the right stack, and push the three (known) values 1297 * by hand onto the new stack - while updating the return eip past 1298 * the instruction that would have done it for sysenter. 1299 */ 1300.macro FIX_STACK offset ok label 1301 cmpw $__KERNEL_CS, 4(%esp) 1302 jne \ok 1303\label: 1304 movl TSS_sysenter_sp0 + \offset(%esp), %esp 1305 CFI_DEF_CFA esp, 0 1306 CFI_UNDEFINED eip 1307 pushfl_cfi 1308 pushl_cfi $__KERNEL_CS 1309 pushl_cfi $sysenter_past_esp 1310 CFI_REL_OFFSET eip, 0 1311.endm 1312 1313ENTRY(debug) 1314 RING0_INT_FRAME 1315 ASM_CLAC 1316 cmpl $ia32_sysenter_target,(%esp) 1317 jne debug_stack_correct 1318 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1319debug_stack_correct: 1320 pushl_cfi $-1 # mark this as an int 1321 SAVE_ALL 1322 TRACE_IRQS_OFF 1323 xorl %edx,%edx # error code 0 1324 movl %esp,%eax # pt_regs pointer 1325 call do_debug 1326 jmp ret_from_exception 1327 CFI_ENDPROC 1328END(debug) 1329 1330/* 1331 * NMI is doubly nasty. It can happen _while_ we're handling 1332 * a debug fault, and the debug fault hasn't yet been able to 1333 * clear up the stack. So we first check whether we got an 1334 * NMI on the sysenter entry path, but after that we need to 1335 * check whether we got an NMI on the debug path where the debug 1336 * fault happened on the sysenter path. 1337 */ 1338ENTRY(nmi) 1339 RING0_INT_FRAME 1340 ASM_CLAC 1341#ifdef CONFIG_X86_ESPFIX32 1342 pushl_cfi %eax 1343 movl %ss, %eax 1344 cmpw $__ESPFIX_SS, %ax 1345 popl_cfi %eax 1346 je nmi_espfix_stack 1347#endif 1348 cmpl $ia32_sysenter_target,(%esp) 1349 je nmi_stack_fixup 1350 pushl_cfi %eax 1351 movl %esp,%eax 1352 /* Do not access memory above the end of our stack page, 1353 * it might not exist. 1354 */ 1355 andl $(THREAD_SIZE-1),%eax 1356 cmpl $(THREAD_SIZE-20),%eax 1357 popl_cfi %eax 1358 jae nmi_stack_correct 1359 cmpl $ia32_sysenter_target,12(%esp) 1360 je nmi_debug_stack_check 1361nmi_stack_correct: 1362 /* We have a RING0_INT_FRAME here */ 1363 pushl_cfi %eax 1364 SAVE_ALL 1365 xorl %edx,%edx # zero error code 1366 movl %esp,%eax # pt_regs pointer 1367 call do_nmi 1368 jmp restore_all_notrace 1369 CFI_ENDPROC 1370 1371nmi_stack_fixup: 1372 RING0_INT_FRAME 1373 FIX_STACK 12, nmi_stack_correct, 1 1374 jmp nmi_stack_correct 1375 1376nmi_debug_stack_check: 1377 /* We have a RING0_INT_FRAME here */ 1378 cmpw $__KERNEL_CS,16(%esp) 1379 jne nmi_stack_correct 1380 cmpl $debug,(%esp) 1381 jb nmi_stack_correct 1382 cmpl $debug_esp_fix_insn,(%esp) 1383 ja nmi_stack_correct 1384 FIX_STACK 24, nmi_stack_correct, 1 1385 jmp nmi_stack_correct 1386 1387#ifdef CONFIG_X86_ESPFIX32 1388nmi_espfix_stack: 1389 /* We have a RING0_INT_FRAME here. 1390 * 1391 * create the pointer to lss back 1392 */ 1393 pushl_cfi %ss 1394 pushl_cfi %esp 1395 addl $4, (%esp) 1396 /* copy the iret frame of 12 bytes */ 1397 .rept 3 1398 pushl_cfi 16(%esp) 1399 .endr 1400 pushl_cfi %eax 1401 SAVE_ALL 1402 FIXUP_ESPFIX_STACK # %eax == %esp 1403 xorl %edx,%edx # zero error code 1404 call do_nmi 1405 RESTORE_REGS 1406 lss 12+4(%esp), %esp # back to espfix stack 1407 CFI_ADJUST_CFA_OFFSET -24 1408 jmp irq_return 1409#endif 1410 CFI_ENDPROC 1411END(nmi) 1412 1413ENTRY(int3) 1414 RING0_INT_FRAME 1415 ASM_CLAC 1416 pushl_cfi $-1 # mark this as an int 1417 SAVE_ALL 1418 TRACE_IRQS_OFF 1419 xorl %edx,%edx # zero error code 1420 movl %esp,%eax # pt_regs pointer 1421 call do_int3 1422 jmp ret_from_exception 1423 CFI_ENDPROC 1424END(int3) 1425 1426ENTRY(general_protection) 1427 RING0_EC_FRAME 1428 pushl_cfi $do_general_protection 1429 jmp error_code 1430 CFI_ENDPROC 1431END(general_protection) 1432 1433#ifdef CONFIG_KVM_GUEST 1434ENTRY(async_page_fault) 1435 RING0_EC_FRAME 1436 ASM_CLAC 1437 pushl_cfi $do_async_page_fault 1438 jmp error_code 1439 CFI_ENDPROC 1440END(async_page_fault) 1441#endif 1442 1443/* 1444 * End of kprobes section 1445 */ 1446 .popsection 1447