arm64/sve: Rework SVE access trap to convert state in registers

When we enable SVE usage in userspace after taking a SVE access trap we
need to ensure that the portions of the register state that are not
shared with the FPSIMD registers are zeroed. Currently we do this by
forcing the FPSIMD registers to be saved to the task struct and converting
them there. This is wasteful in the common case where the task state is
loaded into the registers and we will immediately return to userspace
since we can initialise the SVE state directly in registers instead of
accessing multiple copies of the register state in memory.

Instead in that common case do the conversion in the registers and
update the task metadata so that we can return to userspace without
spilling the register state to memory unless there is some other reason
to do so.

Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20210312190313.24598-1-broonie@kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index bec5f14b622ae2fd50e471803418ca6c91c624f0..ebb263b2d3b19ebec07f68e4fdadb2e684106076 100644 (file)
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -73,6 +73,7 @@ extern void sve_flush_live(void);
 extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state,
                                       unsigned long vq_minus_1);
 extern unsigned int sve_get_vl(void);
+extern void sve_set_vq(unsigned long vq_minus_1);
 
 struct arm64_cpu_capabilities;
 extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);

diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 2ca395c25448f45f29e55ca67992d3ebd491aeb4..3ecec60d3295832b8c7a5917767c638e3866d92e 100644 (file)
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -48,6 +48,11 @@ SYM_FUNC_START(sve_get_vl)
        ret
 SYM_FUNC_END(sve_get_vl)
 
+SYM_FUNC_START(sve_set_vq)
+       sve_load_vq x0, x1, x2
+       ret
+SYM_FUNC_END(sve_set_vq)
+
 /*
  * Load SVE state from FPSIMD state.
  *

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 062b21f30f9422aa03d6cf0eb8cc56e2ba8032d9..0f58e45bd3d1610efd722339becf2b2053b91d03 100644 (file)
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -926,9 +926,8 @@ void fpsimd_release_task(struct task_struct *dead_task)
  * Trapped SVE access
  *
  * Storage is allocated for the full SVE state, the current FPSIMD
- * register contents are migrated across, and TIF_SVE is set so that
- * the SVE access trap will be disabled the next time this task
- * reaches ret_to_user.
+ * register contents are migrated across, and the access trap is
+ * disabled.
  *
  * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
  * would have disabled the SVE access trap for userspace during
@@ -946,15 +945,24 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs)
 
        get_cpu_fpsimd_context();
 
-       fpsimd_save();
-
-       /* Force ret_to_user to reload the registers: */
-       fpsimd_flush_task_state(current);
-
-       fpsimd_to_sve(current);
        if (test_and_set_thread_flag(TIF_SVE))
                WARN_ON(1); /* SVE access shouldn't have trapped */
 
+       /*
+        * Convert the FPSIMD state to SVE, zeroing all the state that
+        * is not shared with FPSIMD. If (as is likely) the current
+        * state is live in the registers then do this there and
+        * update our metadata for the current task including
+        * disabling the trap, otherwise update our in-memory copy.
+        */
+       if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
+               sve_set_vq(sve_vq_from_vl(current->thread.sve_vl) - 1);
+               sve_flush_live();
+               fpsimd_bind_task_to_cpu();
+       } else {
+               fpsimd_to_sve(current);
+       }
+
        put_cpu_fpsimd_context();
 }