arch/riscv/mm/tlbflush.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/mm.h>
   4 #include <linux/smp.h>
   5 #include <linux/sched.h>
   6 #include <asm/sbi.h>
   7 #include <asm/mmu_context.h>
   8
   9 static inline void local_flush_tlb_all_asid(unsigned long asid)
  10 {
  11         if (asid != FLUSH_TLB_NO_ASID)
  12                 __asm__ __volatile__ ("sfence.vma x0, %0"
  13                                 :
  14                                 : "r" (asid)
  15                                 : "memory");
  16         else
  17                 local_flush_tlb_all();
  18 }
  19
  20 static inline void local_flush_tlb_page_asid(unsigned long addr,
  21                 unsigned long asid)
  22 {
  23         if (asid != FLUSH_TLB_NO_ASID)
  24                 __asm__ __volatile__ ("sfence.vma %0, %1"
  25                                 :
  26                                 : "r" (addr), "r" (asid)
  27                                 : "memory");
  28         else
  29                 local_flush_tlb_page(addr);
  30 }
  31
  32 /*
  33  * Flush entire TLB if number of entries to be flushed is greater
  34  * than the threshold below.
  35  */
  36 static unsigned long tlb_flush_all_threshold __read_mostly = 64;
  37
  38 static void local_flush_tlb_range_threshold_asid(unsigned long start,
  39                                                  unsigned long size,
  40                                                  unsigned long stride,
  41                                                  unsigned long asid)
  42 {
  43         unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride);
  44         int i;
  45
  46         if (nr_ptes_in_range > tlb_flush_all_threshold) {
  47                 local_flush_tlb_all_asid(asid);
  48                 return;
  49         }
  50
  51         for (i = 0; i < nr_ptes_in_range; ++i) {
  52                 local_flush_tlb_page_asid(start, asid);
  53                 start += stride;
  54         }
  55 }
  56
  57 static inline void local_flush_tlb_range_asid(unsigned long start,
  58                 unsigned long size, unsigned long stride, unsigned long asid)
  59 {
  60         if (size <= stride)
  61                 local_flush_tlb_page_asid(start, asid);
  62         else if (size == FLUSH_TLB_MAX_SIZE)
  63                 local_flush_tlb_all_asid(asid);
  64         else
  65                 local_flush_tlb_range_threshold_asid(start, size, stride, asid);
  66 }
  67
  68 /* Flush a range of kernel pages without broadcasting */
  69 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
  70 {
  71         local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
  72 }
  73
  74 static void __ipi_flush_tlb_all(void *info)
  75 {
  76         local_flush_tlb_all();
  77 }
  78
  79 void flush_tlb_all(void)
  80 {
  81         if (riscv_use_ipi_for_rfence())
  82                 on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
  83         else
  84                 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID);
  85 }
  86
  87 struct flush_tlb_range_data {
  88         unsigned long asid;
  89         unsigned long start;
  90         unsigned long size;
  91         unsigned long stride;
  92 };
  93
  94 static void __ipi_flush_tlb_range_asid(void *info)
  95 {
  96         struct flush_tlb_range_data *d = info;
  97
  98         local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
  99 }
 100
 101 static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
 102                               unsigned long size, unsigned long stride)
 103 {
 104         struct flush_tlb_range_data ftd;
 105         const struct cpumask *cmask;
 106         unsigned long asid = FLUSH_TLB_NO_ASID;
 107         bool broadcast;
 108
 109         if (mm) {
 110                 unsigned int cpuid;
 111
 112                 cmask = mm_cpumask(mm);
 113                 if (cpumask_empty(cmask))
 114                         return;
 115
 116                 cpuid = get_cpu();
 117                 /* check if the tlbflush needs to be sent to other CPUs */
 118                 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
 119
 120                 if (static_branch_unlikely(&use_asid_allocator))
 121                         asid = atomic_long_read(&mm->context.id) & asid_mask;
 122         } else {
 123                 cmask = cpu_online_mask;
 124                 broadcast = true;
 125         }
 126
 127         if (broadcast) {
 128                 if (riscv_use_ipi_for_rfence()) {
 129                         ftd.asid = asid;
 130                         ftd.start = start;
 131                         ftd.size = size;
 132                         ftd.stride = stride;
 133                         on_each_cpu_mask(cmask,
 134                                          __ipi_flush_tlb_range_asid,
 135                                          &ftd, 1);
 136                 } else
 137                         sbi_remote_sfence_vma_asid(cmask,
 138                                                    start, size, asid);
 139         } else {
 140                 local_flush_tlb_range_asid(start, size, stride, asid);
 141         }
 142
 143         if (mm)
 144                 put_cpu();
 145 }
 146
 147 void flush_tlb_mm(struct mm_struct *mm)
 148 {
 149         __flush_tlb_range(mm, 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
 150 }
 151
 152 void flush_tlb_mm_range(struct mm_struct *mm,
 153                         unsigned long start, unsigned long end,
 154                         unsigned int page_size)
 155 {
 156         __flush_tlb_range(mm, start, end - start, page_size);
 157 }
 158
 159 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 160 {
 161         __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
 162 }
 163
 164 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 165                      unsigned long end)
 166 {
 167         __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE);
 168 }
 169
 170 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 171 {
 172         __flush_tlb_range(NULL, start, end - start, PAGE_SIZE);
 173 }
 174
 175 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 176 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 177                         unsigned long end)
 178 {
 179         __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE);
 180 }
 181 #endif