arch/riscv/mm/tlbflush.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/mm.h>
   4 #include <linux/smp.h>
   5 #include <linux/sched.h>
   6 #include <asm/sbi.h>
   7 #include <asm/mmu_context.h>
   8
   9 static inline void local_flush_tlb_all_asid(unsigned long asid)
  10 {
  11         __asm__ __volatile__ ("sfence.vma x0, %0"
  12                         :
  13                         : "r" (asid)
  14                         : "memory");
  15 }
  16
  17 static inline void local_flush_tlb_page_asid(unsigned long addr,
  18                 unsigned long asid)
  19 {
  20         __asm__ __volatile__ ("sfence.vma %0, %1"
  21                         :
  22                         : "r" (addr), "r" (asid)
  23                         : "memory");
  24 }
  25
  26 static inline void local_flush_tlb_range(unsigned long start,
  27                 unsigned long size, unsigned long stride)
  28 {
  29         if (size <= stride)
  30                 local_flush_tlb_page(start);
  31         else
  32                 local_flush_tlb_all();
  33 }
  34
  35 static inline void local_flush_tlb_range_asid(unsigned long start,
  36                 unsigned long size, unsigned long stride, unsigned long asid)
  37 {
  38         if (size <= stride)
  39                 local_flush_tlb_page_asid(start, asid);
  40         else
  41                 local_flush_tlb_all_asid(asid);
  42 }
  43
  44 static void __ipi_flush_tlb_all(void *info)
  45 {
  46         local_flush_tlb_all();
  47 }
  48
  49 void flush_tlb_all(void)
  50 {
  51         if (riscv_use_ipi_for_rfence())
  52                 on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
  53         else
  54                 sbi_remote_sfence_vma(NULL, 0, -1);
  55 }
  56
  57 struct flush_tlb_range_data {
  58         unsigned long asid;
  59         unsigned long start;
  60         unsigned long size;
  61         unsigned long stride;
  62 };
  63
  64 static void __ipi_flush_tlb_range_asid(void *info)
  65 {
  66         struct flush_tlb_range_data *d = info;
  67
  68         local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
  69 }
  70
  71 static void __ipi_flush_tlb_range(void *info)
  72 {
  73         struct flush_tlb_range_data *d = info;
  74
  75         local_flush_tlb_range(d->start, d->size, d->stride);
  76 }
  77
  78 static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
  79                               unsigned long size, unsigned long stride)
  80 {
  81         struct flush_tlb_range_data ftd;
  82         struct cpumask *cmask = mm_cpumask(mm);
  83         unsigned int cpuid;
  84         bool broadcast;
  85
  86         if (cpumask_empty(cmask))
  87                 return;
  88
  89         cpuid = get_cpu();
  90         /* check if the tlbflush needs to be sent to other CPUs */
  91         broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
  92         if (static_branch_unlikely(&use_asid_allocator)) {
  93                 unsigned long asid = atomic_long_read(&mm->context.id) & asid_mask;
  94
  95                 if (broadcast) {
  96                         if (riscv_use_ipi_for_rfence()) {
  97                                 ftd.asid = asid;
  98                                 ftd.start = start;
  99                                 ftd.size = size;
 100                                 ftd.stride = stride;
 101                                 on_each_cpu_mask(cmask,
 102                                                  __ipi_flush_tlb_range_asid,
 103                                                  &ftd, 1);
 104                         } else
 105                                 sbi_remote_sfence_vma_asid(cmask,
 106                                                            start, size, asid);
 107                 } else {
 108                         local_flush_tlb_range_asid(start, size, stride, asid);
 109                 }
 110         } else {
 111                 if (broadcast) {
 112                         if (riscv_use_ipi_for_rfence()) {
 113                                 ftd.asid = 0;
 114                                 ftd.start = start;
 115                                 ftd.size = size;
 116                                 ftd.stride = stride;
 117                                 on_each_cpu_mask(cmask,
 118                                                  __ipi_flush_tlb_range,
 119                                                  &ftd, 1);
 120                         } else
 121                                 sbi_remote_sfence_vma(cmask, start, size);
 122                 } else {
 123                         local_flush_tlb_range(start, size, stride);
 124                 }
 125         }
 126
 127         put_cpu();
 128 }
 129
 130 void flush_tlb_mm(struct mm_struct *mm)
 131 {
 132         __flush_tlb_range(mm, 0, -1, PAGE_SIZE);
 133 }
 134
 135 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 136 {
 137         __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
 138 }
 139
 140 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 141                      unsigned long end)
 142 {
 143         __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE);
 144 }
 145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 146 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 147                         unsigned long end)
 148 {
 149         __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE);
 150 }
 151 #endif