2 * Compatibility mode system call entry point for x86-64.
4 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
7 #include <asm/dwarf2.h>
8 #include <asm/calling.h>
9 #include <asm/asm-offsets.h>
10 #include <asm/current.h>
11 #include <asm/errno.h>
12 #include <asm/ia32_unistd.h>
13 #include <asm/thread_info.h>
14 #include <asm/segment.h>
15 #include <asm/irqflags.h>
16 #include <linux/linkage.h>
18 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
19 #include <linux/elf-em.h>
20 #define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
21 #define __AUDIT_ARCH_LE 0x40000000
23 #ifndef CONFIG_AUDITSYSCALL
24 #define sysexit_audit ia32_ret_from_sys_call
25 #define sysretl_audit ia32_ret_from_sys_call
28 .section .entry.text, "ax"
30 .macro IA32_ARG_FIXUP noebp=0
38 movl %edx,%edx /* zero extension */
42 .macro CLEAR_RREGS offset=0, _r9=rax
44 movq %rax,\offset+R11(%rsp)
45 movq %rax,\offset+R10(%rsp)
46 movq %\_r9,\offset+R9(%rsp)
47 movq %rax,\offset+R8(%rsp)
51 * Reload arg registers from stack in case ptrace changed them.
52 * We don't reload %eax because syscall_trace_enter() returned
53 * the %rax value we should see. Instead, we just truncate that
54 * value to 32 bits again as we did on entry from user mode.
55 * If it's a new value set by user_regset during entry tracing,
56 * this matches the normal truncation of the user-mode value.
57 * If it's -1 to make us punt the syscall, then (u32)-1 is still
58 * an appropriately invalid value.
60 .macro LOAD_ARGS32 offset, _r9=0
62 movl \offset+16(%rsp),%r9d
64 movl \offset+40(%rsp),%ecx
65 movl \offset+48(%rsp),%edx
66 movl \offset+56(%rsp),%esi
67 movl \offset+64(%rsp),%edi
68 movl %eax,%eax /* zero extension */
71 .macro CFI_STARTPROC32 simple
83 #ifdef CONFIG_PARAVIRT
84 ENTRY(native_usergs_sysret32)
87 ENDPROC(native_usergs_sysret32)
89 ENTRY(native_irq_enable_sysexit)
93 ENDPROC(native_irq_enable_sysexit)
97 * 32bit SYSENTER instruction entry.
100 * %eax System call number.
111 * This is purely a fast path. For anything complicated we use the int 0x80
112 * path below. Set up a complete hardware stack frame to share code
113 * with the int 0x80 path.
115 ENTRY(ia32_sysenter_target)
116 CFI_STARTPROC32 simple
121 movq PER_CPU_VAR(kernel_stack), %rsp
122 addq $(KERNEL_STACK_OFFSET),%rsp
124 * No need to follow this irqs on/off section: the syscall
125 * disabled irqs, here we enable it straight after entry:
127 ENABLE_INTERRUPTS(CLBR_NONE)
128 movl %ebp,%ebp /* zero extension */
129 pushq_cfi $__USER32_DS
130 /*CFI_REL_OFFSET ss,0*/
134 /*CFI_REL_OFFSET rflags,0*/
135 movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
137 pushq_cfi $__USER32_CS
138 /*CFI_REL_OFFSET cs,0*/
145 /* no need to do an access_ok check here because rbp has been
146 32bit zero extended */
148 .section __ex_table,"a"
151 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
152 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
154 jnz sysenter_tracesys
155 cmpq $(IA32_NR_syscalls-1),%rax
160 call *ia32_sys_call_table(,%rax,8)
161 movq %rax,RAX-ARGOFFSET(%rsp)
162 DISABLE_INTERRUPTS(CLBR_NONE)
164 testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
166 sysexit_from_sys_call:
167 andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
168 /* clear IF, that popfq doesn't enable interrupts early */
169 andl $~0x200,EFLAGS-R11(%rsp)
170 movl RIP-R11(%rsp),%edx /* User %eip */
172 RESTORE_ARGS 0,24,0,0,0,0
178 /*CFI_RESTORE rflags*/
179 popq_cfi %rcx /* User %esp */
182 ENABLE_INTERRUPTS_SYSEXIT32
184 #ifdef CONFIG_AUDITSYSCALL
185 .macro auditsys_entry_common
186 movl %esi,%r9d /* 6th arg: 4th syscall arg */
187 movl %edx,%r8d /* 5th arg: 3rd syscall arg */
188 /* (already in %ecx) 4th arg: 2nd syscall arg */
189 movl %ebx,%edx /* 3rd arg: 1st syscall arg */
190 movl %eax,%esi /* 2nd arg: syscall number */
191 movl $AUDIT_ARCH_I386,%edi /* 1st arg: audit arch */
192 call audit_syscall_entry
193 movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall number */
194 cmpq $(IA32_NR_syscalls-1),%rax
196 movl %ebx,%edi /* reload 1st syscall arg */
197 movl RCX-ARGOFFSET(%rsp),%esi /* reload 2nd syscall arg */
198 movl RDX-ARGOFFSET(%rsp),%edx /* reload 3rd syscall arg */
199 movl RSI-ARGOFFSET(%rsp),%ecx /* reload 4th syscall arg */
200 movl RDI-ARGOFFSET(%rsp),%r8d /* reload 5th syscall arg */
203 .macro auditsys_exit exit
204 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
205 jnz ia32_ret_from_sys_call
208 movl %eax,%esi /* second arg, syscall return value */
209 cmpl $0,%eax /* is it < 0? */
210 setl %al /* 1 if so, 0 if not */
211 movzbl %al,%edi /* zero-extend that into %edi */
212 inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
213 call audit_syscall_exit
214 movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall return value */
215 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
218 testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
220 CLEAR_RREGS -ARGOFFSET
226 auditsys_entry_common
227 movl %ebp,%r9d /* reload 6th syscall arg */
228 jmp sysenter_dispatch
231 auditsys_exit sysexit_from_sys_call
235 #ifdef CONFIG_AUDITSYSCALL
236 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
241 movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
242 movq %rsp,%rdi /* &pt_regs -> arg1 */
243 call syscall_trace_enter
244 LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */
246 cmpq $(IA32_NR_syscalls-1),%rax
247 ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
250 ENDPROC(ia32_sysenter_target)
253 * 32bit SYSCALL instruction entry.
256 * %eax System call number.
262 * %ebp Arg2 [note: not saved in the stack frame, should not be touched]
268 * This is purely a fast path. For anything complicated we use the int 0x80
269 * path below. Set up a complete hardware stack frame to share code
270 * with the int 0x80 path.
272 ENTRY(ia32_cstar_target)
273 CFI_STARTPROC32 simple
275 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
277 /*CFI_REGISTER rflags,r11*/
281 movq PER_CPU_VAR(kernel_stack),%rsp
283 * No need to follow this irqs on/off section: the syscall
284 * disabled irqs and here we enable it straight after entry:
286 ENABLE_INTERRUPTS(CLBR_NONE)
288 movl %eax,%eax /* zero extension */
289 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
290 movq %rcx,RIP-ARGOFFSET(%rsp)
291 CFI_REL_OFFSET rip,RIP-ARGOFFSET
292 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
294 movq $__USER32_CS,CS-ARGOFFSET(%rsp)
295 movq $__USER32_DS,SS-ARGOFFSET(%rsp)
296 movq %r11,EFLAGS-ARGOFFSET(%rsp)
297 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
298 movq %r8,RSP-ARGOFFSET(%rsp)
299 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
300 /* no need to do an access_ok check here because r8 has been
301 32bit zero extended */
302 /* hardware stack frame is complete now */
304 .section __ex_table,"a"
307 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
308 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
311 cmpq $IA32_NR_syscalls-1,%rax
316 call *ia32_sys_call_table(,%rax,8)
317 movq %rax,RAX-ARGOFFSET(%rsp)
318 DISABLE_INTERRUPTS(CLBR_NONE)
320 testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
322 sysretl_from_sys_call:
323 andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
324 RESTORE_ARGS 0,-ARG_SKIP,0,0,0
325 movl RIP-ARGOFFSET(%rsp),%ecx
327 movl EFLAGS-ARGOFFSET(%rsp),%r11d
328 /*CFI_REGISTER rflags,r11*/
333 movl RSP-ARGOFFSET(%rsp),%esp
337 #ifdef CONFIG_AUDITSYSCALL
340 movl %r9d,R9-ARGOFFSET(%rsp) /* register to be clobbered by call */
341 auditsys_entry_common
342 movl R9-ARGOFFSET(%rsp),%r9d /* reload 6th syscall arg */
346 auditsys_exit sysretl_from_sys_call
350 #ifdef CONFIG_AUDITSYSCALL
351 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
357 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
358 movq %rsp,%rdi /* &pt_regs -> arg1 */
359 call syscall_trace_enter
360 LOAD_ARGS32 ARGOFFSET, 1 /* reload args from stack in case ptrace changed it */
363 cmpq $(IA32_NR_syscalls-1),%rax
364 ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
366 END(ia32_cstar_target)
374 * Emulated IA32 system calls via int 0x80.
377 * %eax System call number.
383 * %ebp Arg6 [note: not saved in the stack frame, should not be touched]
386 * Uses the same stack frame as the x86-64 version.
387 * All registers except %eax must be saved (but ptrace may violate that)
388 * Arguments are zero extended. For system calls that want sign extension and
389 * take long arguments a wrapper is needed. Most calls can just be called
391 * Assumes it is only called from user space and entered with interrupts off.
395 CFI_STARTPROC32 simple
397 CFI_DEF_CFA rsp,SS+8-RIP
398 /*CFI_REL_OFFSET ss,SS-RIP*/
399 CFI_REL_OFFSET rsp,RSP-RIP
400 /*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
401 /*CFI_REL_OFFSET cs,CS-RIP*/
402 CFI_REL_OFFSET rip,RIP-RIP
403 PARAVIRT_ADJUST_EXCEPTION_FRAME
406 * No need to follow this irqs on/off section: the syscall
407 * disabled irqs and here we enable it straight after entry:
409 ENABLE_INTERRUPTS(CLBR_NONE)
413 /* note the registers are not zero extended to the sf.
414 this could be a problem. */
416 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
417 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
419 cmpq $(IA32_NR_syscalls-1),%rax
423 call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
425 movq %rax,RAX-ARGOFFSET(%rsp)
426 ia32_ret_from_sys_call:
427 CLEAR_RREGS -ARGOFFSET
428 jmp int_ret_from_sys_call
433 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
434 movq %rsp,%rdi /* &pt_regs -> arg1 */
435 call syscall_trace_enter
436 LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */
438 cmpq $(IA32_NR_syscalls-1),%rax
439 ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
444 movq $0,ORIG_RAX-ARGOFFSET(%rsp)
450 .macro PTREGSCALL label, func, arg
453 leaq \func(%rip),%rax
454 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
455 jmp ia32_ptregs_common
460 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
461 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
462 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx
463 PTREGSCALL stub32_execve, sys32_execve, %rcx
464 PTREGSCALL stub32_fork, sys_fork, %rdi
465 PTREGSCALL stub32_clone, sys32_clone, %rdx
466 PTREGSCALL stub32_vfork, sys_vfork, %rdi
467 PTREGSCALL stub32_iopl, sys_iopl, %rsi
473 CFI_STARTPROC32 simple
475 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
476 CFI_REL_OFFSET rax,RAX-ARGOFFSET
477 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
478 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
479 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
480 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
481 CFI_REL_OFFSET rip,RIP-ARGOFFSET
482 /* CFI_REL_OFFSET cs,CS-ARGOFFSET*/
483 /* CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
484 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
485 /* CFI_REL_OFFSET ss,SS-ARGOFFSET*/
489 jmp ia32_sysret /* misbalances the return cache */
491 END(ia32_ptregs_common)