ARM64: ipipe: intercept syscalls

diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h
index a0baa9af54878b5ea4dc95f21701eca55e65c4f0..90e64fd5e50be7cb359f161df60530f8673745b0 100644 (file)
--- a/arch/arm64/include/asm/unistd.h
+++ b/arch/arm64/include/asm/unistd.h
@@ -53,3 +53,5 @@
 #endif
 
 #define NR_syscalls (__NR_syscalls)
+
+#define __ARM_ipipe_syscall    0x10000000

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index b8d001d430cd5dad08e064c0e3b9577a2706c299..ea207e98060cb7a5bac525cf181530fcf46aceda 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -789,7 +789,7 @@ el0_sys:
        mov     x0, x25
        mov     x1, sp
        bl      do_sysinstr
-       b       ret_to_user
+       b       ret_from_exception
 el0_dbg:
        /*
         * Debug exception handling
@@ -864,9 +864,7 @@ ret_fast_syscall_trace:
 work_pending:
        mov     x0, sp                          // 'regs'
        bl      do_notify_resume
-#ifdef CONFIG_TRACE_IRQFLAGS
-       bl      trace_hardirqs_on               // enabled while in userspace
-#endif
+       TRACE_IRQSON
        ldr     x1, [tsk, #TSK_TI_FLAGS]        // re-check for single-step
        b       finish_ret_to_user
 /*
@@ -894,6 +892,57 @@ el0_svc:
 el0_svc_naked:                                 // compat entry point
        stp     x0, xscno, [sp, #S_ORIG_X0]     // save the original x0 and syscall number
        enable_dbg_and_irq
+
+#ifdef CONFIG_IPIPE
+       ldr     x16, [tsk, #TSK_TI_IPIPE]
+       tst     wscno, __ARM_ipipe_syscall
+       b.eq    fastcall_bypass
+       tst     x16, #_TIP_HEAD
+       b.eq    fastcall_bypass
+       mov     x0, sp
+       bl      ipipe_fastcall_hook
+       cmp     w0, #0
+       b.lt    no_fastcall
+       ldr     x16, [tsk, #TSK_TI_IPIPE]
+       tst     x16, #_TIP_HEAD
+       b.ne    fastcall_exit
+       bl      __ipipe_root_sync
+fastcall_tail:
+       ldr     x0, [sp, #S_X0]
+       b       ret_fast_syscall
+fastcall_exit:
+       tst     x16, #_TIP_MAYDAY
+       b.eq    fastcall_notail
+       mov     x0, sp
+       bl      __ipipe_call_mayday
+fastcall_notail:
+       ldr     x0, [sp, #S_X0]
+       disable_irq
+       ldr     x1, [tsk, #TSK_TI_FLAGS]
+       enable_step_tsk x1, x2
+       kernel_exit 0
+no_fastcall:
+       ldr     x16, [tsk, #TSK_TI_IPIPE]
+fastcall_bypass:
+       tst     x16, #_TIP_NOTIFY
+       b.ne    syscall_pipeline
+       tst     wscno, __ARM_ipipe_syscall
+       b.eq    regular_syscall
+syscall_pipeline:
+       mov     x0, sp
+       bl      __ipipe_notify_syscall
+       ldr     x16, [tsk, #TSK_TI_IPIPE]
+       tst     x16, #_TIP_HEAD
+       b.ne    fastcall_notail
+       cmp     w0, #0
+       b.ne    fastcall_tail
+regular_syscall:
+       ldp     x0, x1, [sp, #S_X0]
+       ldp     x2, x3, [sp, #S_X2]
+       ldp     x4, x5, [sp, #S_X4]
+       ldp     x6, x7, [sp, #S_X6]
+#endif /* CONFIG_IPIPE */
+
        ct_user_exit 1
 
        ldr     x16, [tsk, #TSK_TI_FLAGS]       // check for syscall hooks