extern u64 read_zcr_features(void);
-extern int __ro_after_init sve_max_vl;
-extern int __ro_after_init sve_max_virtualisable_vl;
-extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
-
/*
* Helpers to translate bit indices in sve_vq_map to VQ values (and
* vice versa). This allows find_next_bit() to be used to find the
return SVE_VQ_MAX - bit;
}
-/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
-static inline bool sve_vq_available(unsigned int vq)
-{
- return test_bit(__vq_to_bit(vq), sve_vq_map);
-}
+
+struct vl_info {
+ enum vec_type type;
+ const char *name; /* For display purposes */
+
+ /* Minimum supported vector length across all CPUs */
+ int min_vl;
+
+ /* Maximum supported vector length across all CPUs */
+ int max_vl;
+ int max_virtualisable_vl;
+
+ /*
+ * Set of available vector lengths,
+ * where length vq encoded as bit __vq_to_bit(vq):
+ */
+ DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
+
+ /* Set of vector lengths present on at least one cpu: */
+ DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
+};
#ifdef CONFIG_ARM64_SVE
* Probing and setup functions.
* Calls to these functions must be serialised with one another.
*/
-extern void __init sve_init_vq_map(void);
-extern void sve_update_vq_map(void);
-extern int sve_verify_vq_map(void);
+enum vec_type;
+
+extern void __init vec_init_vq_map(enum vec_type type);
+extern void vec_update_vq_map(enum vec_type type);
+extern int vec_verify_vq_map(enum vec_type type);
extern void __init sve_setup(void);
+extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
+
+static inline void write_vl(enum vec_type type, u64 val)
+{
+ u64 tmp;
+
+ switch (type) {
+#ifdef CONFIG_ARM64_SVE
+ case ARM64_VEC_SVE:
+ tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
+ write_sysreg_s(tmp | val, SYS_ZCR_EL1);
+ break;
+#endif
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+}
+
+static inline int vec_max_vl(enum vec_type type)
+{
+ return vl_info[type].max_vl;
+}
+
+static inline int vec_max_virtualisable_vl(enum vec_type type)
+{
+ return vl_info[type].max_virtualisable_vl;
+}
+
+static inline int sve_max_vl(void)
+{
+ return vec_max_vl(ARM64_VEC_SVE);
+}
+
+static inline int sve_max_virtualisable_vl(void)
+{
+ return vec_max_virtualisable_vl(ARM64_VEC_SVE);
+}
+
+/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
+static inline bool vq_available(enum vec_type type, unsigned int vq)
+{
+ return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
+}
+
+static inline bool sve_vq_available(unsigned int vq)
+{
+ return vq_available(ARM64_VEC_SVE, vq);
+}
+
#else /* ! CONFIG_ARM64_SVE */
static inline void sve_alloc(struct task_struct *task) { }
return -EINVAL;
}
+static inline int sve_max_vl(void)
+{
+ return -EINVAL;
+}
+
+static inline bool sve_vq_available(unsigned int vq) { return false; }
+
static inline void sve_user_disable(void) { BUILD_BUG(); }
static inline void sve_user_enable(void) { BUILD_BUG(); }
#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
-static inline void sve_init_vq_map(void) { }
-static inline void sve_update_vq_map(void) { }
-static inline int sve_verify_vq_map(void) { return 0; }
+static inline void vec_init_vq_map(enum vec_type t) { }
+static inline void vec_update_vq_map(enum vec_type t) { }
+static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
static inline void sve_setup(void) { }
#endif /* ! CONFIG_ARM64_SVE */
static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
-/* Default VL for tasks that don't set it explicitly: */
-static int __sve_default_vl = -1;
+__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = {
+#ifdef CONFIG_ARM64_SVE
+ [ARM64_VEC_SVE] = {
+ .type = ARM64_VEC_SVE,
+ .name = "SVE",
+ .min_vl = SVE_VL_MIN,
+ .max_vl = SVE_VL_MIN,
+ .max_virtualisable_vl = SVE_VL_MIN,
+ },
+#endif
+};
+
+struct vl_config {
+ int __default_vl; /* Default VL for tasks */
+};
+
+static struct vl_config vl_config[ARM64_VEC_MAX];
+
+static int get_default_vl(enum vec_type type)
+{
+ return READ_ONCE(vl_config[type].__default_vl);
+}
static int get_sve_default_vl(void)
{
- return READ_ONCE(__sve_default_vl);
+ return get_default_vl(ARM64_VEC_SVE);
}
#ifdef CONFIG_ARM64_SVE
-static void set_sve_default_vl(int val)
+static void set_default_vl(enum vec_type type, int val)
{
- WRITE_ONCE(__sve_default_vl, val);
+ WRITE_ONCE(vl_config[type].__default_vl, val);
}
-/* Maximum supported vector length across all CPUs (initially poisoned) */
-int __ro_after_init sve_max_vl = SVE_VL_MIN;
-int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
-
-/*
- * Set of available vector lengths,
- * where length vq encoded as bit __vq_to_bit(vq):
- */
-__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
-/* Set of vector lengths present on at least one cpu: */
-static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
+static void set_sve_default_vl(int val)
+{
+ set_default_vl(ARM64_VEC_SVE, val);
+}
static void __percpu *efi_sve_state;
#else /* ! CONFIG_ARM64_SVE */
/* Dummy declaration for code that will be optimised out: */
-extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
-extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
extern void __percpu *efi_sve_state;
#endif /* ! CONFIG_ARM64_SVE */
* If things go wrong there's a bug somewhere, but try to fall back to a
* safe choice.
*/
-static unsigned int find_supported_sve_vector_length(unsigned int vl)
+static unsigned int find_supported_vector_length(enum vec_type type,
+ unsigned int vl)
{
+ struct vl_info *info = &vl_info[type];
int bit;
- int max_vl = sve_max_vl;
+ int max_vl = info->max_vl;
if (WARN_ON(!sve_vl_valid(vl)))
- vl = SVE_VL_MIN;
+ vl = info->min_vl;
if (WARN_ON(!sve_vl_valid(max_vl)))
- max_vl = SVE_VL_MIN;
+ max_vl = info->min_vl;
if (vl > max_vl)
vl = max_vl;
- bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
+ bit = find_next_bit(info->vq_map, SVE_VQ_MAX,
__vq_to_bit(sve_vq_from_vl(vl)));
return sve_vl_from_vq(__bit_to_vq(bit));
}
static int sve_proc_do_default_vl(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
+ struct vl_info *info = &vl_info[ARM64_VEC_SVE];
int ret;
int vl = get_sve_default_vl();
struct ctl_table tmp_table = {
/* Writing -1 has the special meaning "set to max": */
if (vl == -1)
- vl = sve_max_vl;
+ vl = info->max_vl;
if (!sve_vl_valid(vl))
return -EINVAL;
- set_sve_default_vl(find_supported_sve_vector_length(vl));
+ set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, vl));
return 0;
}
if (vl > SVE_VL_ARCH_MAX)
vl = SVE_VL_ARCH_MAX;
- vl = find_supported_sve_vector_length(vl);
+ vl = find_supported_vector_length(ARM64_VEC_SVE, vl);
if (flags & (PR_SVE_VL_INHERIT |
PR_SVE_SET_VL_ONEXEC))
return sve_prctl_status(0);
}
-static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
+static void vec_probe_vqs(struct vl_info *info,
+ DECLARE_BITMAP(map, SVE_VQ_MAX))
{
unsigned int vq, vl;
- unsigned long zcr;
bitmap_zero(map, SVE_VQ_MAX);
- zcr = ZCR_ELx_LEN_MASK;
- zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
-
for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
- write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
+ write_vl(info->type, vq - 1); /* self-syncing */
vl = sve_get_vl();
vq = sve_vq_from_vl(vl); /* skip intervening lengths */
set_bit(__vq_to_bit(vq), map);
* Initialise the set of known supported VQs for the boot CPU.
* This is called during kernel boot, before secondary CPUs are brought up.
*/
-void __init sve_init_vq_map(void)
+void __init vec_init_vq_map(enum vec_type type)
{
- sve_probe_vqs(sve_vq_map);
- bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
+ struct vl_info *info = &vl_info[type];
+ vec_probe_vqs(info, info->vq_map);
+ bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX);
}
/*
* those not supported by the current CPU.
* This function is called during the bring-up of early secondary CPUs only.
*/
-void sve_update_vq_map(void)
+void vec_update_vq_map(enum vec_type type)
{
+ struct vl_info *info = &vl_info[type];
DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
- sve_probe_vqs(tmp_map);
- bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
- bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
+ vec_probe_vqs(info, tmp_map);
+ bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX);
+ bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map,
+ SVE_VQ_MAX);
}
/*
* Check whether the current CPU supports all VQs in the committed set.
* This function is called during the bring-up of late secondary CPUs only.
*/
-int sve_verify_vq_map(void)
+int vec_verify_vq_map(enum vec_type type)
{
+ struct vl_info *info = &vl_info[type];
DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
unsigned long b;
- sve_probe_vqs(tmp_map);
+ vec_probe_vqs(info, tmp_map);
bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
- if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
- pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
- smp_processor_id());
+ if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) {
+ pr_warn("%s: cpu%d: Required vector length(s) missing\n",
+ info->name, smp_processor_id());
return -EINVAL;
}
/* Recover the set of supported VQs: */
bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
/* Find VQs supported that are not globally supported: */
- bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
+ bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX);
/* Find the lowest such VQ, if any: */
b = find_last_bit(tmp_map, SVE_VQ_MAX);
* Mismatches above sve_max_virtualisable_vl are fine, since
* no guest is allowed to configure ZCR_EL2.LEN to exceed this:
*/
- if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
- pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
- smp_processor_id());
+ if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) {
+ pr_warn("%s: cpu%d: Unsupported vector length(s) present\n",
+ info->name, smp_processor_id());
return -EINVAL;
}
static void __init sve_efi_setup(void)
{
+ struct vl_info *info = &vl_info[ARM64_VEC_SVE];
+
if (!IS_ENABLED(CONFIG_EFI))
return;
* This is evidence of a crippled system and we are returning void,
* so no attempt is made to handle this situation here.
*/
- if (!sve_vl_valid(sve_max_vl))
+ if (!sve_vl_valid(info->max_vl))
goto fail;
efi_sve_state = __alloc_percpu(
- SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
+ SVE_SIG_REGS_SIZE(sve_vq_from_vl(info->max_vl)), SVE_VQ_BYTES);
if (!efi_sve_state)
goto fail;
void __init sve_setup(void)
{
+ struct vl_info *info = &vl_info[ARM64_VEC_SVE];
u64 zcr;
DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
unsigned long b;
* so sve_vq_map must have at least SVE_VQ_MIN set.
* If something went wrong, at least try to patch it up:
*/
- if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
- set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
+ if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map)))
+ set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map);
zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
- sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
+ info->max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
/*
* Sanity-check that the max VL we determined through CPU features
* corresponds properly to sve_vq_map. If not, do our best:
*/
- if (WARN_ON(sve_max_vl != find_supported_sve_vector_length(sve_max_vl)))
- sve_max_vl = find_supported_sve_vector_length(sve_max_vl);
+ if (WARN_ON(info->max_vl != find_supported_vector_length(ARM64_VEC_SVE,
+ info->max_vl)))
+ info->max_vl = find_supported_vector_length(ARM64_VEC_SVE,
+ info->max_vl);
/*
* For the default VL, pick the maximum supported value <= 64.
* VL == 64 is guaranteed not to grow the signal frame.
*/
- set_sve_default_vl(find_supported_sve_vector_length(64));
+ set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64));
- bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
+ bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map,
SVE_VQ_MAX);
b = find_last_bit(tmp_map, SVE_VQ_MAX);
if (b >= SVE_VQ_MAX)
/* No non-virtualisable VLs found */
- sve_max_virtualisable_vl = SVE_VQ_MAX;
+ info->max_virtualisable_vl = SVE_VQ_MAX;
else if (WARN_ON(b == SVE_VQ_MAX - 1))
/* No virtualisable VLs? This is architecturally forbidden. */
- sve_max_virtualisable_vl = SVE_VQ_MIN;
+ info->max_virtualisable_vl = SVE_VQ_MIN;
else /* b + 1 < SVE_VQ_MAX */
- sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
+ info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
- if (sve_max_virtualisable_vl > sve_max_vl)
- sve_max_virtualisable_vl = sve_max_vl;
+ if (info->max_virtualisable_vl > info->max_vl)
+ info->max_virtualisable_vl = info->max_vl;
- pr_info("SVE: maximum available vector length %u bytes per vector\n",
- sve_max_vl);
- pr_info("SVE: default vector length %u bytes per vector\n",
- get_sve_default_vl());
+ pr_info("%s: maximum available vector length %u bytes per vector\n",
+ info->name, info->max_vl);
+ pr_info("%s: default vector length %u bytes per vector\n",
+ info->name, get_sve_default_vl());
/* KVM decides whether to support mismatched systems. Just warn here: */
- if (sve_max_virtualisable_vl < sve_max_vl)
- pr_warn("SVE: unvirtualisable vector lengths present\n");
+ if (sve_max_virtualisable_vl() < sve_max_vl())
+ pr_warn("%s: unvirtualisable vector lengths present\n",
+ info->name);
sve_efi_setup();
}
if (WARN_ON(!sve_vl_valid(vl)))
vl = SVE_VL_MIN;
- supported_vl = find_supported_sve_vector_length(vl);
+ supported_vl = find_supported_vector_length(ARM64_VEC_SVE, vl);
if (WARN_ON(supported_vl != vl))
vl = supported_vl;
__this_cpu_write(efi_sve_state_used, true);
- sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
+ sve_save_state(sve_state + sve_ffr_offset(sve_max_vl()),
&this_cpu_ptr(&efi_fpsimd_state)->fpsr,
true);
} else {
likely(__this_cpu_read(efi_sve_state_used))) {
char const *sve_state = this_cpu_ptr(efi_sve_state);
- sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
+ sve_load_state(sve_state + sve_ffr_offset(sve_max_vl()),
&this_cpu_ptr(&efi_fpsimd_state)->fpsr,
true,
sve_vq_from_vl(sve_get_vl()) - 1);