return true;
}
+/*
+ * Aside from avoiding lock contention, compaction also periodically checks
+ * need_resched() and either schedules in sync compaction or aborts async
+ * compaction. This is similar to what compact_checklock_irqsave() does, but
+ * is used where no lock is concerned.
+ *
+ * Returns false when no scheduling was needed, or sync compaction scheduled.
+ * Returns true when async compaction should abort.
+ */
+static inline bool compact_should_abort(struct compact_control *cc)
+{
+ /* async compaction aborts if contended */
+ if (need_resched()) {
+ if (cc->mode == MIGRATE_ASYNC) {
+ cc->contended = true;
+ return true;
+ }
+
+ cond_resched();
+ }
+
+ return false;
+}
+
/* Returns true if the page is within a block suitable for migration to */
static bool suitable_migration_target(struct page *page)
{
return 0;
}
- if (cond_resched()) {
- /* Async terminates prematurely on need_resched() */
- if (cc->mode == MIGRATE_ASYNC)
- return 0;
- }
+ if (compact_should_abort(cc))
+ return 0;
/* Time to isolate some pages for migration */
for (; low_pfn < end_pfn; low_pfn++) {
unsigned long block_start_pfn; /* start of current pageblock */
unsigned long block_end_pfn; /* end of current pageblock */
unsigned long low_pfn; /* lowest pfn scanner is able to scan */
- unsigned long next_free_pfn; /* start pfn for scaning at next round */
int nr_freepages = cc->nr_freepages;
struct list_head *freelist = &cc->freepages;
low_pfn = ALIGN(cc->migrate_pfn + 1, pageblock_nr_pages);
/*
- * If no pages are isolated, the block_start_pfn < low_pfn check
- * will kick in.
- */
- next_free_pfn = 0;
-
- /*
* Isolate free pages until enough are available to migrate the
* pages on cc->migratepages. We stop searching if the migrate
* and free page scanners meet or enough free pages are isolated.
/*
* This can iterate a massively long zone without finding any
* suitable migration targets, so periodically check if we need
- * to schedule.
+ * to schedule, or even abort async compaction.
*/
- cond_resched();
+ if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages))
+ && compact_should_abort(cc))
+ break;
if (!pfn_valid(block_start_pfn))
continue;
continue;
/* Found a block suitable for isolating free pages from */
+ cc->free_pfn = block_start_pfn;
isolated = isolate_freepages_block(cc, block_start_pfn,
block_end_pfn, freelist, false);
nr_freepages += isolated;
/*
- * Record the highest PFN we isolated pages from. When next
- * looking for free pages, the search will restart here as
- * page migration may have returned some pages to the allocator
+ * Set a flag that we successfully isolated in this pageblock.
+ * In the next loop iteration, zone->compact_cached_free_pfn
+ * will not be updated and thus it will effectively contain the
+ * highest pageblock we isolated pages from.
*/
- if (isolated && next_free_pfn == 0) {
+ if (isolated)
cc->finished_update_free = true;
- next_free_pfn = block_start_pfn;
- }
+
+ /*
+ * isolate_freepages_block() might have aborted due to async
+ * compaction being contended
+ */
+ if (cc->contended)
+ break;
}
/* split_free_page does not map the pages */
* so that compact_finished() may detect this
*/
if (block_start_pfn < low_pfn)
- next_free_pfn = cc->migrate_pfn;
+ cc->free_pfn = cc->migrate_pfn;
- cc->free_pfn = next_free_pfn;
cc->nr_freepages = nr_freepages;
}
struct compact_control *cc = (struct compact_control *)data;
struct page *freepage;
- /* Isolate free pages if necessary */
+ /*
+ * Isolate free pages if necessary, and if we are not aborting due to
+ * contention.
+ */
if (list_empty(&cc->freepages)) {
- isolate_freepages(cc->zone, cc);
+ if (!cc->contended)
+ isolate_freepages(cc->zone, cc);
if (list_empty(&cc->freepages))
return NULL;
unsigned int order;
unsigned long watermark;
- if (fatal_signal_pending(current))
+ if (cc->contended || fatal_signal_pending(current))
return COMPACT_PARTIAL;
/* Compaction run completes if the migrate and free scanner meet */