2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
12 * entry.S contains the system-call and fault low-level handling routines.
14 * NOTE: This code handles signal-recognition, which happens every time
15 * after an interrupt and after each system call.
17 * Normal syscalls and interrupts don't save a full stack frame, this is
18 * only done for syscall tracing, signals or fork/exec et.al.
20 * A note on terminology:
21 * - top of stack: Architecture defined interrupt frame from SS to RIP
22 * at the top of the kernel process stack.
23 * - partial stack frame: partially saved registers upto R11.
24 * - full stack frame: Like partial stack frame, but all register saved.
27 * - schedule it carefully for the final hardware.
30 #include <linux/linkage.h>
31 #include <asm/segment.h>
32 #include <asm/cache.h>
33 #include <asm/errno.h>
34 #include <asm/dwarf2.h>
35 #include <asm/calling.h>
36 #include <asm/asm-offsets.h>
38 #include <asm/unistd.h>
39 #include <asm/thread_info.h>
40 #include <asm/hw_irq.h>
42 #include <asm/irqflags.h>
46 #ifndef CONFIG_PREEMPT
47 #define retint_kernel retint_restore_args
51 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
52 #ifdef CONFIG_TRACE_IRQFLAGS
53 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
61 * C code is not supposed to know about undefined top of stack. Every time
62 * a C function with an pt_regs argument is called from the SYSCALL based
63 * fast path FIXUP_TOP_OF_STACK is needed.
64 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
68 /* %rsp:at FRAMEEND */
69 .macro FIXUP_TOP_OF_STACK tmp
70 movq %gs:pda_oldrsp,\tmp
72 movq $__USER_DS,SS(%rsp)
73 movq $__USER_CS,CS(%rsp)
75 movq R11(%rsp),\tmp /* get eflags */
76 movq \tmp,EFLAGS(%rsp)
79 .macro RESTORE_TOP_OF_STACK tmp,offset=0
80 movq RSP-\offset(%rsp),\tmp
81 movq \tmp,%gs:pda_oldrsp
82 movq EFLAGS-\offset(%rsp),\tmp
83 movq \tmp,R11-\offset(%rsp)
86 .macro FAKE_STACK_FRAME child_rip
87 /* push in order ss, rsp, eflags, cs, rip */
90 CFI_ADJUST_CFA_OFFSET 8
91 /*CFI_REL_OFFSET ss,0*/
93 CFI_ADJUST_CFA_OFFSET 8
95 pushq $(1<<9) /* eflags - interrupts on */
96 CFI_ADJUST_CFA_OFFSET 8
97 /*CFI_REL_OFFSET rflags,0*/
98 pushq $__KERNEL_CS /* cs */
99 CFI_ADJUST_CFA_OFFSET 8
100 /*CFI_REL_OFFSET cs,0*/
101 pushq \child_rip /* rip */
102 CFI_ADJUST_CFA_OFFSET 8
104 pushq %rax /* orig rax */
105 CFI_ADJUST_CFA_OFFSET 8
108 .macro UNFAKE_STACK_FRAME
110 CFI_ADJUST_CFA_OFFSET -(6*8)
113 .macro CFI_DEFAULT_STACK start=1
118 CFI_DEF_CFA_OFFSET SS+8
120 CFI_REL_OFFSET r15,R15
121 CFI_REL_OFFSET r14,R14
122 CFI_REL_OFFSET r13,R13
123 CFI_REL_OFFSET r12,R12
124 CFI_REL_OFFSET rbp,RBP
125 CFI_REL_OFFSET rbx,RBX
126 CFI_REL_OFFSET r11,R11
127 CFI_REL_OFFSET r10,R10
130 CFI_REL_OFFSET rax,RAX
131 CFI_REL_OFFSET rcx,RCX
132 CFI_REL_OFFSET rdx,RDX
133 CFI_REL_OFFSET rsi,RSI
134 CFI_REL_OFFSET rdi,RDI
135 CFI_REL_OFFSET rip,RIP
136 /*CFI_REL_OFFSET cs,CS*/
137 /*CFI_REL_OFFSET rflags,EFLAGS*/
138 CFI_REL_OFFSET rsp,RSP
139 /*CFI_REL_OFFSET ss,SS*/
142 * A newly forked process directly context switches into this.
148 GET_THREAD_INFO(%rcx)
149 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
153 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
154 je int_ret_from_sys_call
155 testl $_TIF_IA32,threadinfo_flags(%rcx)
156 jnz int_ret_from_sys_call
157 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
158 jmp ret_from_sys_call
161 call syscall_trace_leave
162 GET_THREAD_INFO(%rcx)
168 * System call entry. Upto 6 arguments in registers are supported.
170 * SYSCALL does not save anything on the stack and does not change the
176 * rax system call number
178 * rcx return address for syscall/sysret, C arg3
181 * r10 arg3 (--> moved to rcx for C)
184 * r11 eflags for syscall/sysret, temporary for C
185 * r12-r15,rbp,rbx saved by C code, not touched.
187 * Interrupts are off on entry.
188 * Only called from user space.
190 * XXX if we had a free scratch register we could save the RSP into the stack frame
191 * and report it properly in ps. Unfortunately we haven't.
193 * When user can change the frames always force IRET. That is because
194 * it deals with uncanonical addresses better. SYSRET has trouble
195 * with them due to bugs in both AMD and Intel CPUs.
200 CFI_DEF_CFA rsp,PDA_STACKOFFSET
202 /*CFI_REGISTER rflags,r11*/
204 movq %rsp,%gs:pda_oldrsp
205 movq %gs:pda_kernelstack,%rsp
207 * No need to follow this irqs off/on section - it's straight
212 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
213 movq %rcx,RIP-ARGOFFSET(%rsp)
214 CFI_REL_OFFSET rip,RIP-ARGOFFSET
215 GET_THREAD_INFO(%rcx)
216 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
219 cmpq $__NR_syscall_max,%rax
222 call *sys_call_table(,%rax,8) # XXX: rip relative
223 movq %rax,RAX-ARGOFFSET(%rsp)
225 * Syscall return path ending with SYSRET (fast path)
226 * Has incomplete stack frame and undefined top of stack.
228 .globl ret_from_sys_call
230 movl $_TIF_ALLWORK_MASK,%edi
233 GET_THREAD_INFO(%rcx)
236 movl threadinfo_flags(%rcx),%edx
241 * sysretq will re-enable interrupts:
244 movq RIP-ARGOFFSET(%rsp),%rcx
246 RESTORE_ARGS 0,-ARG_SKIP,1
247 /*CFI_REGISTER rflags,r11*/
248 movq %gs:pda_oldrsp,%rsp
252 /* Handle reschedules */
253 /* edx: work, edi: workmask */
256 bt $TIF_NEED_RESCHED,%edx
261 CFI_ADJUST_CFA_OFFSET 8
264 CFI_ADJUST_CFA_OFFSET -8
267 /* Handle a signal */
271 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
274 /* Really a signal */
275 /* edx: work flags (arg3) */
276 leaq do_notify_resume(%rip),%rax
277 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
278 xorl %esi,%esi # oldset -> arg2
279 call ptregscall_common
280 1: movl $_TIF_NEED_RESCHED,%edi
281 /* Use IRET because user could have changed frame. This
282 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
288 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
289 jmp ret_from_sys_call
291 /* Do syscall tracing */
295 movq $-ENOSYS,RAX(%rsp)
296 FIXUP_TOP_OF_STACK %rdi
298 call syscall_trace_enter
299 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
301 cmpq $__NR_syscall_max,%rax
303 movq %r10,%rcx /* fixup for C */
304 call *sys_call_table(,%rax,8)
305 1: movq %rax,RAX-ARGOFFSET(%rsp)
306 /* Use IRET because user could have changed frame */
307 jmp int_ret_from_sys_call
312 * Syscall return path ending with IRET.
313 * Has correct top of stack, but partial stack frame.
315 ENTRY(int_ret_from_sys_call)
317 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
318 /*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
319 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
320 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
321 /*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
322 CFI_REL_OFFSET rip,RIP-ARGOFFSET
323 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
324 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
325 CFI_REL_OFFSET rax,RAX-ARGOFFSET
326 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
327 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
328 CFI_REL_OFFSET r8,R8-ARGOFFSET
329 CFI_REL_OFFSET r9,R9-ARGOFFSET
330 CFI_REL_OFFSET r10,R10-ARGOFFSET
331 CFI_REL_OFFSET r11,R11-ARGOFFSET
334 testl $3,CS-ARGOFFSET(%rsp)
335 je retint_restore_args
336 movl $_TIF_ALLWORK_MASK,%edi
337 /* edi: mask to check */
339 GET_THREAD_INFO(%rcx)
340 movl threadinfo_flags(%rcx),%edx
343 andl $~TS_COMPAT,threadinfo_status(%rcx)
346 /* Either reschedule or signal or syscall exit tracking needed. */
347 /* First do a reschedule test. */
348 /* edx: work, edi: workmask */
350 bt $TIF_NEED_RESCHED,%edx
355 CFI_ADJUST_CFA_OFFSET 8
358 CFI_ADJUST_CFA_OFFSET -8
363 /* handle signals and tracing -- both require a full stack frame */
368 /* Check for syscall exit trace */
369 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
372 CFI_ADJUST_CFA_OFFSET 8
373 leaq 8(%rsp),%rdi # &ptregs -> arg1
374 call syscall_trace_leave
376 CFI_ADJUST_CFA_OFFSET -8
377 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
383 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
385 movq %rsp,%rdi # &ptregs -> arg1
386 xorl %esi,%esi # oldset -> arg2
387 call do_notify_resume
388 1: movl $_TIF_NEED_RESCHED,%edi
395 END(int_ret_from_sys_call)
398 * Certain special system calls that need to save a complete full stack frame.
401 .macro PTREGSCALL label,func,arg
404 leaq \func(%rip),%rax
405 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
406 jmp ptregscall_common
412 PTREGSCALL stub_clone, sys_clone, %r8
413 PTREGSCALL stub_fork, sys_fork, %rdi
414 PTREGSCALL stub_vfork, sys_vfork, %rdi
415 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
416 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
417 PTREGSCALL stub_iopl, sys_iopl, %rsi
419 ENTRY(ptregscall_common)
421 CFI_ADJUST_CFA_OFFSET -8
422 CFI_REGISTER rip, r11
425 CFI_REGISTER rip, r15
426 FIXUP_TOP_OF_STACK %r11
428 RESTORE_TOP_OF_STACK %r11
430 CFI_REGISTER rip, r11
433 CFI_ADJUST_CFA_OFFSET 8
434 CFI_REL_OFFSET rip, 0
437 END(ptregscall_common)
442 CFI_ADJUST_CFA_OFFSET -8
443 CFI_REGISTER rip, r11
445 FIXUP_TOP_OF_STACK %r11
447 RESTORE_TOP_OF_STACK %r11
450 jmp int_ret_from_sys_call
455 * sigreturn is special because it needs to restore all registers on return.
456 * This cannot be done with SYSRET, so use the IRET return path instead.
458 ENTRY(stub_rt_sigreturn)
461 CFI_ADJUST_CFA_OFFSET -8
464 FIXUP_TOP_OF_STACK %r11
465 call sys_rt_sigreturn
466 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
468 jmp int_ret_from_sys_call
470 END(stub_rt_sigreturn)
473 * initial frame state for interrupts and exceptions
477 CFI_DEF_CFA rsp,SS+8-\ref
478 /*CFI_REL_OFFSET ss,SS-\ref*/
479 CFI_REL_OFFSET rsp,RSP-\ref
480 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
481 /*CFI_REL_OFFSET cs,CS-\ref*/
482 CFI_REL_OFFSET rip,RIP-\ref
485 /* initial frame state for interrupts (and exceptions without error code) */
486 #define INTR_FRAME _frame RIP
487 /* initial frame state for exceptions with error code (and interrupts with
488 vector already pushed) */
489 #define XCPT_FRAME _frame ORIG_RAX
492 * Interrupt entry/exit.
494 * Interrupt entry points save only callee clobbered registers in fast path.
496 * Entry runs with interrupts off.
499 /* 0(%rsp): interrupt number */
500 .macro interrupt func
503 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
505 CFI_ADJUST_CFA_OFFSET 8
506 CFI_REL_OFFSET rbp, 0
508 CFI_DEF_CFA_REGISTER rbp
512 1: incl %gs:pda_irqcount # RED-PEN should check preempt count
513 cmoveq %gs:pda_irqstackptr,%rsp
514 push %rbp # backlink for old unwinder
516 * We entered an interrupt context - irqs are off:
522 ENTRY(common_interrupt)
525 /* 0(%rsp): oldrsp-ARGOFFSET */
529 decl %gs:pda_irqcount
531 CFI_DEF_CFA_REGISTER rsp
532 CFI_ADJUST_CFA_OFFSET -8
534 GET_THREAD_INFO(%rcx)
535 testl $3,CS-ARGOFFSET(%rsp)
538 /* Interrupt came from user space */
540 * Has a correct top of stack, but a partial stack frame
541 * %rcx: thread info. Interrupts off.
543 retint_with_reschedule:
544 movl $_TIF_WORK_MASK,%edi
546 movl threadinfo_flags(%rcx),%edx
552 * The iretq could re-enable interrupts:
562 * The iretq could re-enable interrupts:
570 .section __ex_table,"a"
571 .quad iret_label,bad_iret
574 /* force a signal here? this matches i386 behaviour */
575 /* running with kernel gs */
577 movq $11,%rdi /* SIGSEGV */
583 /* edi: workmask, edx: work */
586 bt $TIF_NEED_RESCHED,%edx
591 CFI_ADJUST_CFA_OFFSET 8
594 CFI_ADJUST_CFA_OFFSET -8
595 GET_THREAD_INFO(%rcx)
601 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
606 movq $-1,ORIG_RAX(%rsp)
607 xorl %esi,%esi # oldset
608 movq %rsp,%rdi # &pt_regs
609 call do_notify_resume
613 movl $_TIF_NEED_RESCHED,%edi
614 GET_THREAD_INFO(%rcx)
617 #ifdef CONFIG_PREEMPT
618 /* Returning to kernel space. Check if we need preemption */
619 /* rcx: threadinfo. interrupts off. */
621 cmpl $0,threadinfo_preempt_count(%rcx)
622 jnz retint_restore_args
623 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
624 jnc retint_restore_args
625 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
626 jnc retint_restore_args
627 call preempt_schedule_irq
632 END(common_interrupt)
637 .macro apicinterrupt num,func
640 CFI_ADJUST_CFA_OFFSET 8
646 ENTRY(thermal_interrupt)
647 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
648 END(thermal_interrupt)
650 ENTRY(threshold_interrupt)
651 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
652 END(threshold_interrupt)
655 ENTRY(reschedule_interrupt)
656 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
657 END(reschedule_interrupt)
659 .macro INVALIDATE_ENTRY num
660 ENTRY(invalidate_interrupt\num)
661 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
662 END(invalidate_interrupt\num)
674 ENTRY(call_function_interrupt)
675 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
676 END(call_function_interrupt)
679 #ifdef CONFIG_X86_LOCAL_APIC
680 ENTRY(apic_timer_interrupt)
681 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
682 END(apic_timer_interrupt)
684 ENTRY(error_interrupt)
685 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
688 ENTRY(spurious_interrupt)
689 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
690 END(spurious_interrupt)
694 * Exception entry points.
698 pushq $0 /* push error code/oldrax */
699 CFI_ADJUST_CFA_OFFSET 8
700 pushq %rax /* push real oldrax to the rdi slot */
701 CFI_ADJUST_CFA_OFFSET 8
707 .macro errorentry sym
710 CFI_ADJUST_CFA_OFFSET 8
716 /* error code is on the stack already */
717 /* handle NMI like exceptions that can happen everywhere */
718 .macro paranoidentry sym, ist=0, irqtrace=1
722 movl $MSR_GS_BASE,%ecx
730 movq %gs:pda_data_offset, %rbp
733 movq ORIG_RAX(%rsp),%rsi
734 movq $-1,ORIG_RAX(%rsp)
736 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
740 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
749 * "Paranoid" exit path from exception stack.
750 * Paranoid because this is used by NMIs and cannot take
751 * any kernel state for granted.
752 * We don't do kernel preemption checks here, because only
753 * NMI should be common and it does not enable IRQs and
754 * cannot get reschedule ticks.
756 * "trace" is 0 for the NMI handler only, because irq-tracing
757 * is fundamentally NMI-unsafe. (we cannot change the soft and
758 * hard flags at once, atomically)
760 .macro paranoidexit trace=1
761 /* ebx: no swapgs flag */
763 testl %ebx,%ebx /* swapgs needed? */
764 jnz paranoid_restore\trace
766 jnz paranoid_userspace\trace
767 paranoid_swapgs\trace:
770 paranoid_restore\trace:
773 paranoid_userspace\trace:
774 GET_THREAD_INFO(%rcx)
775 movl threadinfo_flags(%rcx),%ebx
776 andl $_TIF_WORK_MASK,%ebx
777 jz paranoid_swapgs\trace
778 movq %rsp,%rdi /* &pt_regs */
780 movq %rax,%rsp /* switch stack for scheduling */
781 testl $_TIF_NEED_RESCHED,%ebx
782 jnz paranoid_schedule\trace
783 movl %ebx,%edx /* arg3: thread flags */
788 xorl %esi,%esi /* arg2: oldset */
789 movq %rsp,%rdi /* arg1: &pt_regs */
790 call do_notify_resume
795 jmp paranoid_userspace\trace
796 paranoid_schedule\trace:
806 jmp paranoid_userspace\trace
811 * Exception entry point. This expects an error code/orig_rax on the stack
812 * and the exception handler in %rax.
816 /* rdi slot contains rax, oldrax contains error code */
819 CFI_ADJUST_CFA_OFFSET (14*8)
821 CFI_REL_OFFSET rsi,RSI
822 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
824 CFI_REL_OFFSET rdx,RDX
826 CFI_REL_OFFSET rcx,RCX
827 movq %rsi,10*8(%rsp) /* store rax */
828 CFI_REL_OFFSET rax,RAX
834 CFI_REL_OFFSET r10,R10
836 CFI_REL_OFFSET r11,R11
838 CFI_REL_OFFSET rbx,RBX
840 CFI_REL_OFFSET rbp,RBP
842 CFI_REL_OFFSET r12,R12
844 CFI_REL_OFFSET r13,R13
846 CFI_REL_OFFSET r14,R14
848 CFI_REL_OFFSET r15,R15
857 movq ORIG_RAX(%rsp),%rsi /* get error code */
858 movq $-1,ORIG_RAX(%rsp)
860 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
866 GET_THREAD_INFO(%rcx)
869 movl threadinfo_flags(%rcx),%edx
870 movl $_TIF_WORK_MASK,%edi
874 * The iret might restore flags:
884 /* There are two places in the kernel that can potentially fault with
885 usergs. Handle them here. The exception handlers after
886 iret run with kernel gs again, so don't set the user space flag.
887 B stepping K8s sometimes report an truncated RIP for IRET
888 exceptions returning to compat mode. Check for these here too. */
889 leaq iret_label(%rip),%rbp
892 movl %ebp,%ebp /* zero extend */
895 cmpq $gs_change,RIP(%rsp)
900 /* Reload gs selector with exception handling */
901 /* edi: new selector */
905 CFI_ADJUST_CFA_OFFSET 8
910 2: mfence /* workaround */
913 CFI_ADJUST_CFA_OFFSET -8
916 ENDPROC(load_gs_index)
918 .section __ex_table,"a"
920 .quad gs_change,bad_gs
923 /* running with kernelgs */
925 swapgs /* switch back to user gs */
932 * Create a kernel thread.
934 * C extern interface:
935 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
937 * asm input arguments:
938 * rdi: fn, rsi: arg, rdx: flags
942 FAKE_STACK_FRAME $child_rip
945 # rdi: flags, rsi: usp, rdx: will be &pt_regs
947 orq kernel_thread_flags(%rip),%rdi
960 * It isn't worth to check for reschedule here,
961 * so internally to the x86_64 port you can rely on kernel_thread()
962 * not to reschedule the child before returning, this avoids the need
963 * of hacks for example to fork off the per-CPU idle tasks.
964 * [Hopefully no generic code relies on the reschedule -AK]
970 ENDPROC(kernel_thread)
973 pushq $0 # fake return address
976 * Here we are in the child and the registers are set as they were
977 * at kernel_thread() invocation in the parent.
989 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
991 * C extern interface:
992 * extern long execve(char *name, char **argv, char **envp)
994 * asm input arguments:
995 * rdi: name, rsi: argv, rdx: envp
997 * We want to fallback into:
998 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
1000 * do_sys_execve asm fallback arguments:
1001 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
1008 movq %rax, RAX(%rsp)
1011 je int_ret_from_sys_call
1018 KPROBE_ENTRY(page_fault)
1019 errorentry do_page_fault
1023 ENTRY(coprocessor_error)
1024 zeroentry do_coprocessor_error
1025 END(coprocessor_error)
1027 ENTRY(simd_coprocessor_error)
1028 zeroentry do_simd_coprocessor_error
1029 END(simd_coprocessor_error)
1031 ENTRY(device_not_available)
1032 zeroentry math_state_restore
1033 END(device_not_available)
1035 /* runs on exception stack */
1039 CFI_ADJUST_CFA_OFFSET 8
1040 paranoidentry do_debug, DEBUG_STACK
1045 /* runs on exception stack */
1049 CFI_ADJUST_CFA_OFFSET 8
1050 paranoidentry do_nmi, 0, 0
1051 #ifdef CONFIG_TRACE_IRQFLAGS
1063 CFI_ADJUST_CFA_OFFSET 8
1064 paranoidentry do_int3, DEBUG_STACK
1071 zeroentry do_overflow
1079 zeroentry do_invalid_op
1082 ENTRY(coprocessor_segment_overrun)
1083 zeroentry do_coprocessor_segment_overrun
1084 END(coprocessor_segment_overrun)
1087 zeroentry do_reserved
1090 /* runs on exception stack */
1093 paranoidentry do_double_fault
1099 errorentry do_invalid_TSS
1102 ENTRY(segment_not_present)
1103 errorentry do_segment_not_present
1104 END(segment_not_present)
1106 /* runs on exception stack */
1107 ENTRY(stack_segment)
1109 paranoidentry do_stack_segment
1114 KPROBE_ENTRY(general_protection)
1115 errorentry do_general_protection
1116 END(general_protection)
1119 ENTRY(alignment_check)
1120 errorentry do_alignment_check
1121 END(alignment_check)
1124 zeroentry do_divide_error
1127 ENTRY(spurious_interrupt_bug)
1128 zeroentry do_spurious_interrupt_bug
1129 END(spurious_interrupt_bug)
1131 #ifdef CONFIG_X86_MCE
1132 /* runs on exception stack */
1133 ENTRY(machine_check)
1136 CFI_ADJUST_CFA_OFFSET 8
1137 paranoidentry do_machine_check
1143 /* Call softirq on interrupt stack. Interrupts are off. */
1147 CFI_ADJUST_CFA_OFFSET 8
1148 CFI_REL_OFFSET rbp,0
1150 CFI_DEF_CFA_REGISTER rbp
1151 incl %gs:pda_irqcount
1152 cmove %gs:pda_irqstackptr,%rsp
1153 push %rbp # backlink for old unwinder
1156 CFI_DEF_CFA_REGISTER rsp
1157 CFI_ADJUST_CFA_OFFSET -8
1158 decl %gs:pda_irqcount
1161 ENDPROC(call_softirq)
1163 #ifdef CONFIG_STACK_UNWIND
1164 ENTRY(arch_unwind_init_running)
1166 movq %r15, R15(%rdi)
1167 movq %r14, R14(%rdi)
1169 movq %r13, R13(%rdi)
1170 movq %r12, R12(%rdi)
1172 movq %rbp, RBP(%rdi)
1173 movq %rbx, RBX(%rdi)
1175 movq %rax, R11(%rdi)
1176 movq %rax, R10(%rdi)
1179 movq %rax, RAX(%rdi)
1180 movq %rax, RCX(%rdi)
1181 movq %rax, RDX(%rdi)
1182 movq %rax, RSI(%rdi)
1183 movq %rax, RDI(%rdi)
1184 movq %rax, ORIG_RAX(%rdi)
1185 movq %rcx, RIP(%rdi)
1187 movq $__KERNEL_CS, CS(%rdi)
1188 movq %rax, EFLAGS(%rdi)
1189 movq %rcx, RSP(%rdi)
1190 movq $__KERNEL_DS, SS(%rdi)
1193 ENDPROC(arch_unwind_init_running)