riscv: mm: modify pte format for Svnapot

Add one alternative to enable/disable svnapot support, enable this static
key when "svnapot" is in the "riscv,isa" field of fdt and SVNAPOT compile
option is set. It will influence the behavior of has_svnapot. All code
dependent on svnapot should make sure that has_svnapot return true firstly.

Modify PTE definition for Svnapot, and creates some functions in pgtable.h
to mark a PTE as napot and check if it is a Svnapot PTE. Until now, only
64KB napot size is supported in spec, so some macros has only 64KB version.

Signed-off-by: Qinglin Pan <panqinglin00@gmail.com>
Reviewed-by: Andrew Jones <ajones@ventanamicro.com>
Link: https://lore.kernel.org/r/20230209131647.17245-2-panqinglin00@gmail.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 7c814fb..abbb7b9 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -397,6 +397,25 @@ config RISCV_ISA_C
 
          If you don't know what to do here, say Y.
 
+config RISCV_ISA_SVNAPOT
+       bool "SVNAPOT extension support"
+       depends on 64BIT && MMU
+       default y
+       select RISCV_ALTERNATIVE
+       help
+         Allow kernel to detect the SVNAPOT ISA-extension dynamically at boot
+         time and enable its usage.
+
+         The SVNAPOT extension is used to mark contiguous PTEs as a range
+         of contiguous virtual-to-physical translations for a naturally
+         aligned power-of-2 (NAPOT) granularity larger than the base 4KB page
+         size. When HUGETLBFS is also selected this option unconditionally
+         allocates some memory for each NAPOT page size supported by the kernel.
+         When optimizing for low memory consumption and for platforms without
+         the SVNAPOT extension, it may be better to say N here.
+
+         If you don't know what to do here, say Y.
+
 config RISCV_ISA_SVPBMT
        bool "SVPBMT extension support"
        depends on 64BIT && MMU

diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index ee9c80f..6e368d3 100644 (file)
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -43,10 +43,11 @@
 #define RISCV_ISA_EXT_SSCOFPMF         26
 #define RISCV_ISA_EXT_SSTC             27
 #define RISCV_ISA_EXT_SVINVAL          28
-#define RISCV_ISA_EXT_SVPBMT           29
-#define RISCV_ISA_EXT_ZBB              30
-#define RISCV_ISA_EXT_ZICBOM           31
-#define RISCV_ISA_EXT_ZIHINTPAUSE      32
+#define RISCV_ISA_EXT_SVNAPOT          29
+#define RISCV_ISA_EXT_SVPBMT           30
+#define RISCV_ISA_EXT_ZBB              31
+#define RISCV_ISA_EXT_ZICBOM           32
+#define RISCV_ISA_EXT_ZIHINTPAUSE      33
 
 #ifndef __ASSEMBLY__
 

diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 9f432c1..24a3dd2 100644 (file)
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -16,11 +16,6 @@
 #define PAGE_SIZE      (_AC(1, UL) << PAGE_SHIFT)
 #define PAGE_MASK      (~(PAGE_SIZE - 1))
 
-#ifdef CONFIG_64BIT
-#define HUGE_MAX_HSTATE                2
-#else
-#define HUGE_MAX_HSTATE                1
-#endif
 #define HPAGE_SHIFT            PMD_SHIFT
 #define HPAGE_SIZE             (_AC(1, UL) << HPAGE_SHIFT)
 #define HPAGE_MASK              (~(HPAGE_SIZE - 1))

diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h
index 42a042c..7a50972 100644 (file)
--- a/arch/riscv/include/asm/pgtable-64.h
+++ b/arch/riscv/include/asm/pgtable-64.h
@@ -79,6 +79,40 @@ typedef struct {
 #define _PAGE_PFN_MASK  GENMASK(53, 10)
 
 /*
+ * [63] Svnapot definitions:
+ * 0 Svnapot disabled
+ * 1 Svnapot enabled
+ */
+#define _PAGE_NAPOT_SHIFT      63
+#define _PAGE_NAPOT            BIT(_PAGE_NAPOT_SHIFT)
+/*
+ * Only 64KB (order 4) napot ptes supported.
+ */
+#define NAPOT_CONT_ORDER_BASE 4
+enum napot_cont_order {
+       NAPOT_CONT64KB_ORDER = NAPOT_CONT_ORDER_BASE,
+       NAPOT_ORDER_MAX,
+};
+
+#define for_each_napot_order(order)                                            \
+       for (order = NAPOT_CONT_ORDER_BASE; order < NAPOT_ORDER_MAX; order++)
+#define for_each_napot_order_rev(order)                                                \
+       for (order = NAPOT_ORDER_MAX - 1;                                       \
+            order >= NAPOT_CONT_ORDER_BASE; order--)
+#define napot_cont_order(val)  (__builtin_ctzl((val.pte >> _PAGE_PFN_SHIFT) << 1))
+
+#define napot_cont_shift(order)        ((order) + PAGE_SHIFT)
+#define napot_cont_size(order) BIT(napot_cont_shift(order))
+#define napot_cont_mask(order) (~(napot_cont_size(order) - 1UL))
+#define napot_pte_num(order)   BIT(order)
+
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+#define HUGE_MAX_HSTATE                (2 + (NAPOT_ORDER_MAX - NAPOT_CONT_ORDER_BASE))
+#else
+#define HUGE_MAX_HSTATE                2
+#endif
+
+/*
  * [62:61] Svpbmt Memory Type definitions:
  *
  *  00 - PMA    Normal Cacheable, No change to implied PMA memory type

diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 2a88362..76502bc 100644 (file)
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -264,10 +264,47 @@ static inline pte_t pud_pte(pud_t pud)
        return __pte(pud_val(pud));
 }
 
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+
+static __always_inline bool has_svnapot(void)
+{
+       return riscv_has_extension_likely(RISCV_ISA_EXT_SVNAPOT);
+}
+
+static inline unsigned long pte_napot(pte_t pte)
+{
+       return pte_val(pte) & _PAGE_NAPOT;
+}
+
+static inline pte_t pte_mknapot(pte_t pte, unsigned int order)
+{
+       int pos = order - 1 + _PAGE_PFN_SHIFT;
+       unsigned long napot_bit = BIT(pos);
+       unsigned long napot_mask = ~GENMASK(pos, _PAGE_PFN_SHIFT);
+
+       return __pte((pte_val(pte) & napot_mask) | napot_bit | _PAGE_NAPOT);
+}
+
+#else
+
+static __always_inline bool has_svnapot(void) { return false; }
+
+static inline unsigned long pte_napot(pte_t pte)
+{
+       return 0;
+}
+
+#endif /* CONFIG_RISCV_ISA_SVNAPOT */
+
 /* Yields the page frame number (PFN) of a page table entry */
 static inline unsigned long pte_pfn(pte_t pte)
 {
-       return __page_val_to_pfn(pte_val(pte));
+       unsigned long res  = __page_val_to_pfn(pte_val(pte));
+
+       if (has_svnapot() && pte_napot(pte))
+               res = res & (res - 1UL);
+
+       return res;
 }
 
 #define pte_page(x)     pfn_to_page(pte_pfn(x))

diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
index 420228e..5670909 100644 (file)
--- a/arch/riscv/kernel/cpu.c
+++ b/arch/riscv/kernel/cpu.c
@@ -191,6 +191,7 @@ static struct riscv_isa_ext_data isa_ext_arr[] = {
        __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
        __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
        __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
+       __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
        __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
        __RISCV_ISA_EXT_DATA("", RISCV_ISA_EXT_MAX),
 };

diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index 21fb567..271e391 100644 (file)
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -223,6 +223,7 @@ void __init riscv_fill_hwcap(void)
                                SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF);
                                SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC);
                                SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL);
+                               SET_ISA_EXT_MAP("svnapot", RISCV_ISA_EXT_SVNAPOT);
                                SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT);
                                SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB);
                                SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM);