2 #include <linux/highmem.h>
3 #include <linux/kernel.h>
4 #include <linux/mmdebug.h>
5 #include <linux/mm_types.h>
6 #include <linux/pagemap.h>
7 #include <linux/rcupdate.h>
9 #include <linux/swap.h>
11 #include <asm/pgalloc.h>
14 #ifndef CONFIG_MMU_GATHER_NO_GATHER
16 static bool tlb_next_batch(struct mmu_gather *tlb)
18 struct mmu_gather_batch *batch;
22 tlb->active = batch->next;
26 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
29 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
36 batch->max = MAX_GATHER_BATCH;
38 tlb->active->next = batch;
44 static void tlb_batch_pages_flush(struct mmu_gather *tlb)
46 struct mmu_gather_batch *batch;
48 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
49 free_pages_and_swap_cache(batch->pages, batch->nr);
52 tlb->active = &tlb->local;
55 static void tlb_batch_list_free(struct mmu_gather *tlb)
57 struct mmu_gather_batch *batch, *next;
59 for (batch = tlb->local.next; batch; batch = next) {
61 free_pages((unsigned long)batch, 0);
63 tlb->local.next = NULL;
66 bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
68 struct mmu_gather_batch *batch;
72 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
73 VM_WARN_ON(tlb->page_size != page_size);
78 * Add the page and check if we are full. If so
81 batch->pages[batch->nr++] = page;
82 if (batch->nr == batch->max) {
83 if (!tlb_next_batch(tlb))
87 VM_BUG_ON_PAGE(batch->nr > batch->max, page);
92 #endif /* MMU_GATHER_NO_GATHER */
94 #ifdef CONFIG_MMU_GATHER_TABLE_FREE
96 static void __tlb_remove_table_free(struct mmu_table_batch *batch)
100 for (i = 0; i < batch->nr; i++)
101 __tlb_remove_table(batch->tables[i]);
103 free_page((unsigned long)batch);
106 #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
109 * Semi RCU freeing of the page directories.
111 * This is needed by some architectures to implement software pagetable walkers.
113 * gup_fast() and other software pagetable walkers do a lockless page-table
114 * walk and therefore needs some synchronization with the freeing of the page
115 * directories. The chosen means to accomplish that is by disabling IRQs over
118 * Architectures that use IPIs to flush TLBs will then automagically DTRT,
119 * since we unlink the page, flush TLBs, free the page. Since the disabling of
120 * IRQs delays the completion of the TLB flush we can never observe an already
123 * Architectures that do not have this (PPC) need to delay the freeing by some
124 * other means, this is that means.
126 * What we do is batch the freed directory pages (tables) and RCU free them.
127 * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
128 * holds off grace periods.
130 * However, in order to batch these pages we need to allocate storage, this
131 * allocation is deep inside the MM code and can thus easily fail on memory
132 * pressure. To guarantee progress we fall back to single table freeing, see
133 * the implementation of tlb_remove_table_one().
137 static void tlb_remove_table_smp_sync(void *arg)
139 /* Simply deliver the interrupt */
142 void tlb_remove_table_sync_one(void)
145 * This isn't an RCU grace period and hence the page-tables cannot be
146 * assumed to be actually RCU-freed.
148 * It is however sufficient for software page-table walkers that rely on
151 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
154 static void tlb_remove_table_rcu(struct rcu_head *head)
156 __tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
159 static void tlb_remove_table_free(struct mmu_table_batch *batch)
161 call_rcu(&batch->rcu, tlb_remove_table_rcu);
164 #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
166 static void tlb_remove_table_free(struct mmu_table_batch *batch)
168 __tlb_remove_table_free(batch);
171 #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
174 * If we want tlb_remove_table() to imply TLB invalidates.
176 static inline void tlb_table_invalidate(struct mmu_gather *tlb)
178 if (tlb_needs_table_invalidate()) {
180 * Invalidate page-table caches used by hardware walkers. Then
181 * we still need to RCU-sched wait while freeing the pages
182 * because software walkers can still be in-flight.
184 tlb_flush_mmu_tlbonly(tlb);
188 static void tlb_remove_table_one(void *table)
190 tlb_remove_table_sync_one();
191 __tlb_remove_table(table);
194 static void tlb_table_flush(struct mmu_gather *tlb)
196 struct mmu_table_batch **batch = &tlb->batch;
199 tlb_table_invalidate(tlb);
200 tlb_remove_table_free(*batch);
205 void tlb_remove_table(struct mmu_gather *tlb, void *table)
207 struct mmu_table_batch **batch = &tlb->batch;
209 if (*batch == NULL) {
210 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
211 if (*batch == NULL) {
212 tlb_table_invalidate(tlb);
213 tlb_remove_table_one(table);
219 (*batch)->tables[(*batch)->nr++] = table;
220 if ((*batch)->nr == MAX_TABLE_BATCH)
221 tlb_table_flush(tlb);
224 static inline void tlb_table_init(struct mmu_gather *tlb)
229 #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
231 static inline void tlb_table_flush(struct mmu_gather *tlb) { }
232 static inline void tlb_table_init(struct mmu_gather *tlb) { }
234 #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
236 static void tlb_flush_mmu_free(struct mmu_gather *tlb)
238 tlb_table_flush(tlb);
239 #ifndef CONFIG_MMU_GATHER_NO_GATHER
240 tlb_batch_pages_flush(tlb);
244 void tlb_flush_mmu(struct mmu_gather *tlb)
246 tlb_flush_mmu_tlbonly(tlb);
247 tlb_flush_mmu_free(tlb);
250 static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
254 tlb->fullmm = fullmm;
256 #ifndef CONFIG_MMU_GATHER_NO_GATHER
257 tlb->need_flush_all = 0;
258 tlb->local.next = NULL;
260 tlb->local.max = ARRAY_SIZE(tlb->__pages);
261 tlb->active = &tlb->local;
262 tlb->batch_count = 0;
266 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
270 __tlb_reset_range(tlb);
271 inc_tlb_flush_pending(tlb->mm);
275 * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
276 * @tlb: the mmu_gather structure to initialize
277 * @mm: the mm_struct of the target address space
279 * Called to initialize an (on-stack) mmu_gather structure for page-table
280 * tear-down from @mm.
282 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm)
284 __tlb_gather_mmu(tlb, mm, false);
288 * tlb_gather_mmu_fullmm - initialize an mmu_gather structure for page-table tear-down
289 * @tlb: the mmu_gather structure to initialize
290 * @mm: the mm_struct of the target address space
292 * In this case, @mm is without users and we're going to destroy the
293 * full address space (exit/execve).
295 * Called to initialize an (on-stack) mmu_gather structure for page-table
296 * tear-down from @mm.
298 void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
300 __tlb_gather_mmu(tlb, mm, true);
304 * tlb_finish_mmu - finish an mmu_gather structure
305 * @tlb: the mmu_gather structure to finish
307 * Called at the end of the shootdown operation to free up any resources that
310 void tlb_finish_mmu(struct mmu_gather *tlb)
313 * If there are parallel threads are doing PTE changes on same range
314 * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
315 * flush by batching, one thread may end up seeing inconsistent PTEs
316 * and result in having stale TLB entries. So flush TLB forcefully
317 * if we detect parallel PTE batching threads.
319 * However, some syscalls, e.g. munmap(), may free page tables, this
320 * needs force flush everything in the given range. Otherwise this
321 * may result in having stale TLB entries for some architectures,
322 * e.g. aarch64, that could specify flush what level TLB.
324 if (mm_tlb_flush_nested(tlb->mm)) {
326 * The aarch64 yields better performance with fullmm by
327 * avoiding multiple CPUs spamming TLBI messages at the
330 * On x86 non-fullmm doesn't yield significant difference
334 __tlb_reset_range(tlb);
335 tlb->freed_tables = 1;
340 #ifndef CONFIG_MMU_GATHER_NO_GATHER
341 tlb_batch_list_free(tlb);
343 dec_tlb_flush_pending(tlb->mm);