Our TLB invalidation routines may require a barrier before the
maintenance (in order to ensure pending page table writes are visible to
the hardware walker) and barriers afterwards (in order to ensure
completion of the maintenance and visibility in the instruction stream).
Whilst this is expensive, the cost can be reduced somewhat by reducing
the scope of the barrier instructions:
- The barrier before only needs to apply to stores (pte writes)
- Local ops are required only to affect the non-shareable domain
- Global ops are required only to affect the inner-shareable domain
This patch makes these changes for the TLB flushing code.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
__local_flush_tlb_all();
tlb_op(TLB_V7_UIS_FULL, "c8, c7, 0", zero);
if (tlb_flag(TLB_BARRIER)) {
__local_flush_tlb_all();
tlb_op(TLB_V7_UIS_FULL, "c8, c7, 0", zero);
if (tlb_flag(TLB_BARRIER)) {
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
__local_flush_tlb_all();
tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
if (tlb_flag(TLB_BARRIER)) {
__local_flush_tlb_all();
tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
if (tlb_flag(TLB_BARRIER)) {
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
__local_flush_tlb_mm(mm);
tlb_op(TLB_V7_UIS_ASID, "c8, c7, 2", asid);
if (tlb_flag(TLB_BARRIER))
__local_flush_tlb_mm(mm);
tlb_op(TLB_V7_UIS_ASID, "c8, c7, 2", asid);
if (tlb_flag(TLB_BARRIER))
}
static inline void __flush_tlb_mm(struct mm_struct *mm)
}
static inline void __flush_tlb_mm(struct mm_struct *mm)
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
const unsigned int __tlb_flag = __cpu_tlb_flags;
if (tlb_flag(TLB_WB))
__local_flush_tlb_mm(mm);
#ifdef CONFIG_ARM_ERRATA_720789
__local_flush_tlb_mm(mm);
#ifdef CONFIG_ARM_ERRATA_720789
#endif
if (tlb_flag(TLB_BARRIER))
#endif
if (tlb_flag(TLB_BARRIER))
uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
if (tlb_flag(TLB_WB))
uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
if (tlb_flag(TLB_WB))
__local_flush_tlb_page(vma, uaddr);
tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", uaddr);
if (tlb_flag(TLB_BARRIER))
__local_flush_tlb_page(vma, uaddr);
tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", uaddr);
if (tlb_flag(TLB_BARRIER))
uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
if (tlb_flag(TLB_WB))
uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
if (tlb_flag(TLB_WB))
__local_flush_tlb_page(vma, uaddr);
#ifdef CONFIG_ARM_ERRATA_720789
__local_flush_tlb_page(vma, uaddr);
#ifdef CONFIG_ARM_ERRATA_720789
#endif
if (tlb_flag(TLB_BARRIER))
#endif
if (tlb_flag(TLB_BARRIER))
}
static inline void __local_flush_tlb_kernel_page(unsigned long kaddr)
}
static inline void __local_flush_tlb_kernel_page(unsigned long kaddr)
kaddr &= PAGE_MASK;
if (tlb_flag(TLB_WB))
kaddr &= PAGE_MASK;
if (tlb_flag(TLB_WB))
__local_flush_tlb_kernel_page(kaddr);
tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", kaddr);
if (tlb_flag(TLB_BARRIER)) {
__local_flush_tlb_kernel_page(kaddr);
tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", kaddr);
if (tlb_flag(TLB_BARRIER)) {
kaddr &= PAGE_MASK;
if (tlb_flag(TLB_WB))
kaddr &= PAGE_MASK;
if (tlb_flag(TLB_WB))
__local_flush_tlb_kernel_page(kaddr);
tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr);
if (tlb_flag(TLB_BARRIER)) {
__local_flush_tlb_kernel_page(kaddr);
tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr);
if (tlb_flag(TLB_BARRIER)) {
* Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
*/
asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
* Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
*/
asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
}
#else
static inline int erratum_a15_798181(void)
}
#else
static inline int erratum_a15_798181(void)
tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd);
if (tlb_flag(TLB_WB))
tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd);
if (tlb_flag(TLB_WB))
}
static inline void clean_pmd_entry(void *pmd)
}
static inline void clean_pmd_entry(void *pmd)