RISC-V: Add hwprobe vDSO function and data

Add a vDSO function __vdso_riscv_hwprobe, which can sit in front of the
riscv_hwprobe syscall and answer common queries. We stash a copy of
static answers for the "all CPUs" case in the vDSO data page. This data
is private to the vDSO, so we can decide later to change what's stored
there or under what conditions we defer to the syscall. Currently all
data can be discovered at boot, so the vDSO function answers all queries
when the cpumask is set to the "all CPUs" hint.

There's also a boolean in the data that lets the vDSO function know that
all CPUs are the same. In that case, the vDSO will also answer queries
for arbitrary CPU masks in addition to the "all CPUs" hint.

Signed-off-by: Evan Green <evan@rivosinc.com>
Link: https://lore.kernel.org/r/20230407231103.2622178-7-evan@rivosinc.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index c5e42cc376048dbc9d960a6275b40c983acb7618..a20e105efa4ed824451647e56bf835ae2e94d1f8 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -33,6 +33,7 @@ config RISCV
        select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL
        select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
        select ARCH_HAS_UBSAN_SANITIZE_ALL
+       select ARCH_HAS_VDSO_DATA
        select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
        select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT
        select ARCH_STACKWALK

diff --git a/arch/riscv/include/asm/vdso/data.h b/arch/riscv/include/asm/vdso/data.h
new file mode 100644 (file)
index 0000000..dc2f76f
--- /dev/null
+++ b/arch/riscv/include/asm/vdso/data.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __RISCV_ASM_VDSO_DATA_H
+#define __RISCV_ASM_VDSO_DATA_H
+
+#include <linux/types.h>
+#include <vdso/datapage.h>
+#include <asm/hwprobe.h>
+
+struct arch_vdso_data {
+       /* Stash static answers to the hwprobe queries when all CPUs are selected. */
+       __u64 all_cpu_hwprobe_values[RISCV_HWPROBE_MAX_KEY + 1];
+
+       /* Boolean indicating all CPUs have the same static hwprobe values. */
+       __u8 homogeneous_cpus;
+};
+
+#endif /* __RISCV_ASM_VDSO_DATA_H */

diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h
index 77d9c2f721c464603216d18700958cce032b8b54..ba3283cf7accaa93a38512d2c17eda0eefde0612 100644 (file)
--- a/arch/riscv/include/asm/vdso/gettimeofday.h
+++ b/arch/riscv/include/asm/vdso/gettimeofday.h
@@ -9,6 +9,12 @@
 #include <asm/csr.h>
 #include <uapi/linux/time.h>
 
+/*
+ * 32-bit land is lacking generic time vsyscalls as well as the legacy 32-bit
+ * time syscalls like gettimeofday. Skip these definitions since on 32-bit.
+ */
+#ifdef CONFIG_GENERIC_TIME_VSYSCALL
+
 #define VDSO_HAS_CLOCK_GETRES  1
 
 static __always_inline
@@ -60,6 +66,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
        return ret;
 }
 
+#endif /* CONFIG_GENERIC_TIME_VSYSCALL */
+
 static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
                                                 const struct vdso_data *vd)
 {

diff --git a/arch/riscv/kernel/compat_vdso/Makefile b/arch/riscv/kernel/compat_vdso/Makefile
index 260daf3236d3a8dd08ac046410170c59f5d09b29..91a3431ae1fe9206b7d6c3ce83edf9012ffa332b 100644 (file)
--- a/arch/riscv/kernel/compat_vdso/Makefile
+++ b/arch/riscv/kernel/compat_vdso/Makefile
@@ -22,7 +22,7 @@ targets := $(obj-compat_vdso) compat_vdso.so compat_vdso.so.dbg compat_vdso.lds
 obj-compat_vdso := $(addprefix $(obj)/, $(obj-compat_vdso))
 
 obj-y += compat_vdso.o
-CPPFLAGS_compat_vdso.lds += -P -C -U$(ARCH)
+CPPFLAGS_compat_vdso.lds += -P -C -DCOMPAT_VDSO -U$(ARCH)
 
 # Disable profiling and instrumentation for VDSO code
 GCOV_PROFILE := n

diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
index 55389e7595f60d9a6f0c934cdbdeb9a88954e530..849b4170629dc566e0322bbf0068ee019a3de1ea 100644 (file)
--- a/arch/riscv/kernel/sys_riscv.c
+++ b/arch/riscv/kernel/sys_riscv.c
@@ -14,6 +14,7 @@
 #include <asm/uaccess.h>
 #include <asm/unistd.h>
 #include <asm-generic/mman-common.h>
+#include <vdso/vsyscall.h>
 
 static long riscv_sys_mmap(unsigned long addr, unsigned long len,
                           unsigned long prot, unsigned long flags,
@@ -243,6 +244,50 @@ static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
        return 0;
 }
 
+#ifdef CONFIG_MMU
+
+static int __init init_hwprobe_vdso_data(void)
+{
+       struct vdso_data *vd = __arch_get_k_vdso_data();
+       struct arch_vdso_data *avd = &vd->arch_data;
+       u64 id_bitsmash = 0;
+       struct riscv_hwprobe pair;
+       int key;
+
+       /*
+        * Initialize vDSO data with the answers for the "all CPUs" case, to
+        * save a syscall in the common case.
+        */
+       for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
+               pair.key = key;
+               hwprobe_one_pair(&pair, cpu_online_mask);
+
+               WARN_ON_ONCE(pair.key < 0);
+
+               avd->all_cpu_hwprobe_values[key] = pair.value;
+               /*
+                * Smash together the vendor, arch, and impl IDs to see if
+                * they're all 0 or any negative.
+                */
+               if (key <= RISCV_HWPROBE_KEY_MIMPID)
+                       id_bitsmash |= pair.value;
+       }
+
+       /*
+        * If the arch, vendor, and implementation ID are all the same across
+        * all harts, then assume all CPUs are the same, and allow the vDSO to
+        * answer queries for arbitrary masks. However if all values are 0 (not
+        * populated) or any value returns -1 (varies across CPUs), then the
+        * vDSO should defer to the kernel for exotic cpu masks.
+        */
+       avd->homogeneous_cpus = (id_bitsmash > 0);
+       return 0;
+}
+
+arch_initcall_sync(init_hwprobe_vdso_data);
+
+#endif /* CONFIG_MMU */
+
 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
                size_t, pair_count, size_t, cpu_count, unsigned long __user *,
                cpus, unsigned int, flags)

diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c
index cc2d1e8c8736478c89d8d3d49d8b3219bf1f4df1..9a68e7eaae4d47e573c7c96289ddbf325164721d 100644 (file)
--- a/arch/riscv/kernel/vdso.c
+++ b/arch/riscv/kernel/vdso.c
@@ -14,13 +14,7 @@
 #include <asm/page.h>
 #include <asm/vdso.h>
 #include <linux/time_namespace.h>
-
-#ifdef CONFIG_GENERIC_TIME_VSYSCALL
 #include <vdso/datapage.h>
-#else
-struct vdso_data {
-};
-#endif
 
 enum vvar_pages {
        VVAR_DATA_PAGE_OFFSET,

diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index 06e6b27f3bcc938ba336f32e51de459fd6c15ad2..022258426050db974128bd320fd0562da84e8c75 100644 (file)
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -12,6 +12,8 @@ vdso-syms += vgettimeofday
 endif
 vdso-syms += getcpu
 vdso-syms += flush_icache
+vdso-syms += hwprobe
+vdso-syms += sys_hwprobe
 
 # Files to link into the vdso
 obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o
@@ -23,6 +25,8 @@ ifneq ($(c-gettimeofday-y),)
   CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y)
 endif
 
+CFLAGS_hwprobe.o += -fPIC
+
 # Build rules
 targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds
 obj-vdso := $(addprefix $(obj)/, $(obj-vdso))

diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c
new file mode 100644 (file)
index 0000000..d40bec6
--- /dev/null
+++ b/arch/riscv/kernel/vdso/hwprobe.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2023 Rivos, Inc
+ */
+
+#include <linux/types.h>
+#include <vdso/datapage.h>
+#include <vdso/helpers.h>
+
+extern int riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
+                        size_t cpu_count, unsigned long *cpus,
+                        unsigned int flags);
+
+/* Add a prototype to avoid -Wmissing-prototypes warning. */
+int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
+                        size_t cpu_count, unsigned long *cpus,
+                        unsigned int flags);
+
+int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
+                        size_t cpu_count, unsigned long *cpus,
+                        unsigned int flags)
+{
+       const struct vdso_data *vd = __arch_get_vdso_data();
+       const struct arch_vdso_data *avd = &vd->arch_data;
+       bool all_cpus = !cpu_count && !cpus;
+       struct riscv_hwprobe *p = pairs;
+       struct riscv_hwprobe *end = pairs + pair_count;
+
+       /*
+        * Defer to the syscall for exotic requests. The vdso has answers
+        * stashed away only for the "all cpus" case. If all CPUs are
+        * homogeneous, then this function can handle requests for arbitrary
+        * masks.
+        */
+       if ((flags != 0) || (!all_cpus && !avd->homogeneous_cpus))
+               return riscv_hwprobe(pairs, pair_count, cpu_count, cpus, flags);
+
+       /* This is something we can handle, fill out the pairs. */
+       while (p < end) {
+               if (p->key <= RISCV_HWPROBE_MAX_KEY) {
+                       p->value = avd->all_cpu_hwprobe_values[p->key];
+
+               } else {
+                       p->key = -1;
+                       p->value = 0;
+               }
+
+               p++;
+       }
+
+       return 0;
+}

diff --git a/arch/riscv/kernel/vdso/sys_hwprobe.S b/arch/riscv/kernel/vdso/sys_hwprobe.S
new file mode 100644 (file)
index 0000000..4e70414
--- /dev/null
+++ b/arch/riscv/kernel/vdso/sys_hwprobe.S
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2023 Rivos, Inc */
+
+#include <linux/linkage.h>
+#include <asm/unistd.h>
+
+.text
+ENTRY(riscv_hwprobe)
+       .cfi_startproc
+       li a7, __NR_riscv_hwprobe
+       ecall
+       ret
+
+       .cfi_endproc
+ENDPROC(riscv_hwprobe)

diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S
index 4a060663329024feffbb82d7c5d663b1099f36ce..82ce64900f3d7e7af48a211b8e24aeec0952d504 100644 (file)
--- a/arch/riscv/kernel/vdso/vdso.lds.S
+++ b/arch/riscv/kernel/vdso/vdso.lds.S
@@ -82,6 +82,9 @@ VERSION
 #endif
                __vdso_getcpu;
                __vdso_flush_icache;
+#ifndef COMPAT_VDSO
+               __vdso_riscv_hwprobe;
+#endif
        local: *;
        };
 }