1 // SPDX-License-Identifier: GPL-2.0-only
3 * SBI initialilization and all extension implementation.
5 * Copyright (c) 2020 Western Digital Corporation or its affiliates.
8 #include <linux/bits.h>
9 #include <linux/init.h>
11 #include <linux/reboot.h>
15 /* default SBI version is 0.1 */
16 unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT;
17 EXPORT_SYMBOL(sbi_spec_version);
19 static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init;
20 static void (*__sbi_send_ipi)(unsigned int cpu) __ro_after_init;
21 static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask,
22 unsigned long start, unsigned long size,
23 unsigned long arg4, unsigned long arg5) __ro_after_init;
25 struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
26 unsigned long arg1, unsigned long arg2,
27 unsigned long arg3, unsigned long arg4,
32 register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0);
33 register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1);
34 register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2);
35 register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3);
36 register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4);
37 register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5);
38 register uintptr_t a6 asm ("a6") = (uintptr_t)(fid);
39 register uintptr_t a7 asm ("a7") = (uintptr_t)(ext);
41 : "+r" (a0), "+r" (a1)
42 : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7)
49 EXPORT_SYMBOL(sbi_ecall);
51 int sbi_err_map_linux_errno(int err)
58 case SBI_ERR_INVALID_PARAM:
60 case SBI_ERR_INVALID_ADDRESS:
62 case SBI_ERR_NOT_SUPPORTED:
68 EXPORT_SYMBOL(sbi_err_map_linux_errno);
70 #ifdef CONFIG_RISCV_SBI_V01
71 static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mask)
73 unsigned long cpuid, hartid;
74 unsigned long hmask = 0;
77 * There is no maximum hartid concept in RISC-V and NR_CPUS must not be
78 * associated with hartid. As SBI v0.1 is only kept for backward compatibility
79 * and will be removed in the future, there is no point in supporting hartid
80 * greater than BITS_PER_LONG (32 for RV32 and 64 for RV64). Ideally, SBI v0.2
81 * should be used for platforms with hartid greater than BITS_PER_LONG.
83 for_each_cpu(cpuid, cpu_mask) {
84 hartid = cpuid_to_hartid_map(cpuid);
85 if (hartid >= BITS_PER_LONG) {
86 pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n");
96 * sbi_console_putchar() - Writes given character to the console device.
97 * @ch: The data to be written to the console.
101 void sbi_console_putchar(int ch)
103 sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0);
105 EXPORT_SYMBOL(sbi_console_putchar);
108 * sbi_console_getchar() - Reads a byte from console device.
110 * Returns the value read from console.
112 int sbi_console_getchar(void)
116 ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0);
120 EXPORT_SYMBOL(sbi_console_getchar);
123 * sbi_shutdown() - Remove all the harts from executing supervisor code.
127 void sbi_shutdown(void)
129 sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0);
131 EXPORT_SYMBOL(sbi_shutdown);
134 * __sbi_set_timer_v01() - Program the timer for next timer event.
135 * @stime_value: The value after which next timer event should fire.
139 static void __sbi_set_timer_v01(uint64_t stime_value)
141 #if __riscv_xlen == 32
142 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value,
143 stime_value >> 32, 0, 0, 0, 0);
145 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0);
149 static void __sbi_send_ipi_v01(unsigned int cpu)
151 unsigned long hart_mask =
152 __sbi_v01_cpumask_to_hartmask(cpumask_of(cpu));
153 sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask),
157 static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask,
158 unsigned long start, unsigned long size,
159 unsigned long arg4, unsigned long arg5)
162 unsigned long hart_mask;
164 if (!cpu_mask || cpumask_empty(cpu_mask))
165 cpu_mask = cpu_online_mask;
166 hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask);
168 /* v0.2 function IDs are equivalent to v0.1 extension IDs */
170 case SBI_EXT_RFENCE_REMOTE_FENCE_I:
171 sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0,
172 (unsigned long)&hart_mask, 0, 0, 0, 0, 0);
174 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
175 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0,
176 (unsigned long)&hart_mask, start, size,
179 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
180 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0,
181 (unsigned long)&hart_mask, start, size,
185 pr_err("SBI call [%d]not supported in SBI v0.1\n", fid);
192 static void sbi_set_power_off(void)
194 pm_power_off = sbi_shutdown;
197 static void __sbi_set_timer_v01(uint64_t stime_value)
199 pr_warn("Timer extension is not available in SBI v%lu.%lu\n",
200 sbi_major_version(), sbi_minor_version());
203 static void __sbi_send_ipi_v01(unsigned int cpu)
205 pr_warn("IPI extension is not available in SBI v%lu.%lu\n",
206 sbi_major_version(), sbi_minor_version());
209 static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask,
210 unsigned long start, unsigned long size,
211 unsigned long arg4, unsigned long arg5)
213 pr_warn("remote fence extension is not available in SBI v%lu.%lu\n",
214 sbi_major_version(), sbi_minor_version());
219 static void sbi_set_power_off(void) {}
220 #endif /* CONFIG_RISCV_SBI_V01 */
222 static void __sbi_set_timer_v02(uint64_t stime_value)
224 #if __riscv_xlen == 32
225 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value,
226 stime_value >> 32, 0, 0, 0, 0);
228 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0,
233 static void __sbi_send_ipi_v02(unsigned int cpu)
236 struct sbiret ret = {0};
238 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI,
239 1UL, cpuid_to_hartid_map(cpu), 0, 0, 0, 0);
241 result = sbi_err_map_linux_errno(ret.error);
242 pr_err("%s: hbase = [%lu] failed (error [%d])\n",
243 __func__, cpuid_to_hartid_map(cpu), result);
247 static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask,
248 unsigned long hbase, unsigned long start,
249 unsigned long size, unsigned long arg4,
252 struct sbiret ret = {0};
253 int ext = SBI_EXT_RFENCE;
257 case SBI_EXT_RFENCE_REMOTE_FENCE_I:
258 ret = sbi_ecall(ext, fid, hmask, hbase, 0, 0, 0, 0);
260 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
261 ret = sbi_ecall(ext, fid, hmask, hbase, start,
264 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
265 ret = sbi_ecall(ext, fid, hmask, hbase, start,
269 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
270 ret = sbi_ecall(ext, fid, hmask, hbase, start,
273 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
274 ret = sbi_ecall(ext, fid, hmask, hbase, start,
277 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
278 ret = sbi_ecall(ext, fid, hmask, hbase, start,
281 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
282 ret = sbi_ecall(ext, fid, hmask, hbase, start,
286 pr_err("unknown function ID [%lu] for SBI extension [%d]\n",
292 result = sbi_err_map_linux_errno(ret.error);
293 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n",
294 __func__, hbase, hmask, result);
300 static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask,
301 unsigned long start, unsigned long size,
302 unsigned long arg4, unsigned long arg5)
304 unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0;
307 if (!cpu_mask || cpumask_empty(cpu_mask))
308 cpu_mask = cpu_online_mask;
310 for_each_cpu(cpuid, cpu_mask) {
311 hartid = cpuid_to_hartid_map(cpuid);
313 if (hartid + BITS_PER_LONG <= htop ||
314 hbase + BITS_PER_LONG <= hartid) {
315 result = __sbi_rfence_v02_call(fid, hmask,
316 hbase, start, size, arg4, arg5);
320 } else if (hartid < hbase) {
321 /* shift the mask to fit lower hartid */
322 hmask <<= hbase - hartid;
329 } else if (hartid > htop) {
332 hmask |= BIT(hartid - hbase);
336 result = __sbi_rfence_v02_call(fid, hmask, hbase,
337 start, size, arg4, arg5);
346 * sbi_set_timer() - Program the timer for next timer event.
347 * @stime_value: The value after which next timer event should fire.
351 void sbi_set_timer(uint64_t stime_value)
353 __sbi_set_timer(stime_value);
357 * sbi_send_ipi() - Send an IPI to any hart.
358 * @cpu: Logical id of the target CPU.
360 void sbi_send_ipi(unsigned int cpu)
364 EXPORT_SYMBOL(sbi_send_ipi);
367 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts.
368 * @cpu_mask: A cpu mask containing all the target harts.
370 * Return: 0 on success, appropriate linux error code otherwise.
372 int sbi_remote_fence_i(const struct cpumask *cpu_mask)
374 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I,
375 cpu_mask, 0, 0, 0, 0);
377 EXPORT_SYMBOL(sbi_remote_fence_i);
380 * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote
381 * harts for the specified virtual address range.
382 * @cpu_mask: A cpu mask containing all the target harts.
383 * @start: Start of the virtual address
384 * @size: Total size of the virtual address range.
386 * Return: 0 on success, appropriate linux error code otherwise.
388 int sbi_remote_sfence_vma(const struct cpumask *cpu_mask,
392 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA,
393 cpu_mask, start, size, 0, 0);
395 EXPORT_SYMBOL(sbi_remote_sfence_vma);
398 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given
399 * remote harts for a virtual address range belonging to a specific ASID.
401 * @cpu_mask: A cpu mask containing all the target harts.
402 * @start: Start of the virtual address
403 * @size: Total size of the virtual address range.
404 * @asid: The value of address space identifier (ASID).
406 * Return: 0 on success, appropriate linux error code otherwise.
408 int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask,
413 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID,
414 cpu_mask, start, size, asid, 0);
416 EXPORT_SYMBOL(sbi_remote_sfence_vma_asid);
419 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote
420 * harts for the specified guest physical address range.
421 * @cpu_mask: A cpu mask containing all the target harts.
422 * @start: Start of the guest physical address
423 * @size: Total size of the guest physical address range.
427 int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask,
431 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA,
432 cpu_mask, start, size, 0, 0);
434 EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma);
437 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given
438 * remote harts for a guest physical address range belonging to a specific VMID.
440 * @cpu_mask: A cpu mask containing all the target harts.
441 * @start: Start of the guest physical address
442 * @size: Total size of the guest physical address range.
443 * @vmid: The value of guest ID (VMID).
445 * Return: 0 if success, Error otherwise.
447 int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask,
452 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID,
453 cpu_mask, start, size, vmid, 0);
455 EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid);
458 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote
459 * harts for the current guest virtual address range.
460 * @cpu_mask: A cpu mask containing all the target harts.
461 * @start: Start of the current guest virtual address
462 * @size: Total size of the current guest virtual address range.
466 int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask,
470 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA,
471 cpu_mask, start, size, 0, 0);
473 EXPORT_SYMBOL(sbi_remote_hfence_vvma);
476 * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given
477 * remote harts for current guest virtual address range belonging to a specific
480 * @cpu_mask: A cpu mask containing all the target harts.
481 * @start: Start of the current guest virtual address
482 * @size: Total size of the current guest virtual address range.
483 * @asid: The value of address space identifier (ASID).
487 int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask,
492 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID,
493 cpu_mask, start, size, asid, 0);
495 EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid);
497 static void sbi_srst_reset(unsigned long type, unsigned long reason)
499 sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason,
501 pr_warn("%s: type=0x%lx reason=0x%lx failed\n",
502 __func__, type, reason);
505 static int sbi_srst_reboot(struct notifier_block *this,
506 unsigned long mode, void *cmd)
508 sbi_srst_reset((mode == REBOOT_WARM || mode == REBOOT_SOFT) ?
509 SBI_SRST_RESET_TYPE_WARM_REBOOT :
510 SBI_SRST_RESET_TYPE_COLD_REBOOT,
511 SBI_SRST_RESET_REASON_NONE);
515 static struct notifier_block sbi_srst_reboot_nb;
517 static void sbi_srst_power_off(void)
519 sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN,
520 SBI_SRST_RESET_REASON_NONE);
524 * sbi_probe_extension() - Check if an SBI extension ID is supported or not.
525 * @extid: The extension ID to be probed.
527 * Return: Extension specific nonzero value f yes, -ENOTSUPP otherwise.
529 int sbi_probe_extension(int extid)
533 ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid,
541 EXPORT_SYMBOL(sbi_probe_extension);
543 static long __sbi_base_ecall(int fid)
547 ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0);
551 return sbi_err_map_linux_errno(ret.error);
554 static inline long sbi_get_spec_version(void)
556 return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION);
559 static inline long sbi_get_firmware_id(void)
561 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID);
564 static inline long sbi_get_firmware_version(void)
566 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION);
569 long sbi_get_mvendorid(void)
571 return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID);
573 EXPORT_SYMBOL_GPL(sbi_get_mvendorid);
575 long sbi_get_marchid(void)
577 return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID);
579 EXPORT_SYMBOL_GPL(sbi_get_marchid);
581 long sbi_get_mimpid(void)
583 return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID);
585 EXPORT_SYMBOL_GPL(sbi_get_mimpid);
587 void __init sbi_init(void)
592 ret = sbi_get_spec_version();
594 sbi_spec_version = ret;
596 pr_info("SBI specification v%lu.%lu detected\n",
597 sbi_major_version(), sbi_minor_version());
599 if (!sbi_spec_is_0_1()) {
600 pr_info("SBI implementation ID=0x%lx Version=0x%lx\n",
601 sbi_get_firmware_id(), sbi_get_firmware_version());
602 if (sbi_probe_extension(SBI_EXT_TIME) > 0) {
603 __sbi_set_timer = __sbi_set_timer_v02;
604 pr_info("SBI TIME extension detected\n");
606 __sbi_set_timer = __sbi_set_timer_v01;
608 if (sbi_probe_extension(SBI_EXT_IPI) > 0) {
609 __sbi_send_ipi = __sbi_send_ipi_v02;
610 pr_info("SBI IPI extension detected\n");
612 __sbi_send_ipi = __sbi_send_ipi_v01;
614 if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) {
615 __sbi_rfence = __sbi_rfence_v02;
616 pr_info("SBI RFENCE extension detected\n");
618 __sbi_rfence = __sbi_rfence_v01;
620 if ((sbi_spec_version >= sbi_mk_version(0, 3)) &&
621 (sbi_probe_extension(SBI_EXT_SRST) > 0)) {
622 pr_info("SBI SRST extension detected\n");
623 pm_power_off = sbi_srst_power_off;
624 sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot;
625 sbi_srst_reboot_nb.priority = 192;
626 register_restart_handler(&sbi_srst_reboot_nb);
629 __sbi_set_timer = __sbi_set_timer_v01;
630 __sbi_send_ipi = __sbi_send_ipi_v01;
631 __sbi_rfence = __sbi_rfence_v01;