mm, compaction: embed migration mode in compact_control

We're going to want to manipulate the migration mode for compaction in the
page allocator, and currently compact_control's sync field is only a bool.

Currently, we only do MIGRATE_ASYNC or MIGRATE_SYNC_LIGHT compaction
depending on the value of this bool.  Convert the bool to enum
migrate_mode and pass the migration mode in directly.  Later, we'll want
to avoid MIGRATE_SYNC_LIGHT for thp allocations in the pagefault patch to
avoid unnecessary latency.

This also alters compaction triggered from sysfs, either for the entire
system or for a node, to force MIGRATE_SYNC.

[akpm@linux-foundation.org: fix build]
[iamjoonsoo.kim@lge.com: use MIGRATE_SYNC in alloc_contig_range()]
Signed-off-by: David Rientjes <rientjes@google.com>
Suggested-by: Mel Gorman <mgorman@suse.de>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Greg Thelen <gthelen@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/compaction.h b/include/linux/compaction.h
index 7e1c76e..01e3132 100644 (file)
--- a/include/linux/compaction.h
+++ b/include/linux/compaction.h
@@ -22,7 +22,7 @@ extern int sysctl_extfrag_handler(struct ctl_table *table, int write,
 extern int fragmentation_index(struct zone *zone, unsigned int order);
 extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
                        int order, gfp_t gfp_mask, nodemask_t *mask,
-                       bool sync, bool *contended);
+                       enum migrate_mode mode, bool *contended);
 extern void compact_pgdat(pg_data_t *pgdat, int order);
 extern void reset_isolation_suitable(pg_data_t *pgdat);
 extern unsigned long compaction_suitable(struct zone *zone, int order);
@@ -91,7 +91,7 @@ static inline bool compaction_restarting(struct zone *zone, int order)
 #else
 static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
                        int order, gfp_t gfp_mask, nodemask_t *nodemask,
-                       bool sync, bool *contended)
+                       enum migrate_mode mode, bool *contended)
 {
        return COMPACT_CONTINUE;
 }

diff --git a/mm/compaction.c b/mm/compaction.c
index 70c0f8c..217a6ad 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -161,7 +161,8 @@ static void update_pageblock_skip(struct compact_control *cc,
                        return;
                if (pfn > zone->compact_cached_migrate_pfn[0])
                        zone->compact_cached_migrate_pfn[0] = pfn;
-               if (cc->sync && pfn > zone->compact_cached_migrate_pfn[1])
+               if (cc->mode != MIGRATE_ASYNC &&
+                   pfn > zone->compact_cached_migrate_pfn[1])
                        zone->compact_cached_migrate_pfn[1] = pfn;
        } else {
                if (cc->finished_update_free)
@@ -208,7 +209,7 @@ static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags,
                }
 
                /* async aborts if taking too long or contended */
-               if (!cc->sync) {
+               if (cc->mode == MIGRATE_ASYNC) {
                        cc->contended = true;
                        return false;
                }
@@ -473,7 +474,8 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
        bool locked = false;
        struct page *page = NULL, *valid_page = NULL;
        bool set_unsuitable = true;
-       const isolate_mode_t mode = (!cc->sync ? ISOLATE_ASYNC_MIGRATE : 0) |
+       const isolate_mode_t mode = (cc->mode == MIGRATE_ASYNC ?
+                                       ISOLATE_ASYNC_MIGRATE : 0) |
                                    (unevictable ? ISOLATE_UNEVICTABLE : 0);
 
        /*
@@ -483,7 +485,7 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
         */
        while (unlikely(too_many_isolated(zone))) {
                /* async migration should just abort */
-               if (!cc->sync)
+               if (cc->mode == MIGRATE_ASYNC)
                        return 0;
 
                congestion_wait(BLK_RW_ASYNC, HZ/10);
@@ -548,7 +550,8 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
                         * the minimum amount of work satisfies the allocation
                         */
                        mt = get_pageblock_migratetype(page);
-                       if (!cc->sync && !migrate_async_suitable(mt)) {
+                       if (cc->mode == MIGRATE_ASYNC &&
+                           !migrate_async_suitable(mt)) {
                                set_unsuitable = false;
                                goto next_pageblock;
                        }
@@ -981,6 +984,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
        int ret;
        unsigned long start_pfn = zone->zone_start_pfn;
        unsigned long end_pfn = zone_end_pfn(zone);
+       const bool sync = cc->mode != MIGRATE_ASYNC;
 
        ret = compaction_suitable(zone, cc->order);
        switch (ret) {
@@ -1006,7 +1010,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
         * information on where the scanners should start but check that it
         * is initialised by ensuring the values are within zone boundaries.
         */
-       cc->migrate_pfn = zone->compact_cached_migrate_pfn[cc->sync];
+       cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];
        cc->free_pfn = zone->compact_cached_free_pfn;
        if (cc->free_pfn < start_pfn || cc->free_pfn > end_pfn) {
                cc->free_pfn = end_pfn & ~(pageblock_nr_pages-1);
@@ -1040,8 +1044,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
 
                nr_migrate = cc->nr_migratepages;
                err = migrate_pages(&cc->migratepages, compaction_alloc,
-                               compaction_free, (unsigned long)cc,
-                               cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC,
+                               compaction_free, (unsigned long)cc, cc->mode,
                                MR_COMPACTION);
                update_nr_listpages(cc);
                nr_remaining = cc->nr_migratepages;
@@ -1074,9 +1077,8 @@ out:
        return ret;
 }
 
-static unsigned long compact_zone_order(struct zone *zone,
-                                int order, gfp_t gfp_mask,
-                                bool sync, bool *contended)
+static unsigned long compact_zone_order(struct zone *zone, int order,
+               gfp_t gfp_mask, enum migrate_mode mode, bool *contended)
 {
        unsigned long ret;
        struct compact_control cc = {
@@ -1085,7 +1087,7 @@ static unsigned long compact_zone_order(struct zone *zone,
                .order = order,
                .migratetype = allocflags_to_migratetype(gfp_mask),
                .zone = zone,
-               .sync = sync,
+               .mode = mode,
        };
        INIT_LIST_HEAD(&cc.freepages);
        INIT_LIST_HEAD(&cc.migratepages);
@@ -1107,7 +1109,7 @@ int sysctl_extfrag_threshold = 500;
  * @order: The order of the current allocation
  * @gfp_mask: The GFP mask of the current allocation
  * @nodemask: The allowed nodes to allocate from
- * @sync: Whether migration is synchronous or not
+ * @mode: The migration mode for async, sync light, or sync migration
  * @contended: Return value that is true if compaction was aborted due to lock contention
  * @page: Optionally capture a free page of the requested order during compaction
  *
@@ -1115,7 +1117,7 @@ int sysctl_extfrag_threshold = 500;
  */
 unsigned long try_to_compact_pages(struct zonelist *zonelist,
                        int order, gfp_t gfp_mask, nodemask_t *nodemask,
-                       bool sync, bool *contended)
+                       enum migrate_mode mode, bool *contended)
 {
        enum zone_type high_zoneidx = gfp_zone(gfp_mask);
        int may_enter_fs = gfp_mask & __GFP_FS;
@@ -1140,7 +1142,7 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist,
                                                                nodemask) {
                int status;
 
-               status = compact_zone_order(zone, order, gfp_mask, sync,
+               status = compact_zone_order(zone, order, gfp_mask, mode,
                                                contended);
                rc = max(status, rc);
 
@@ -1190,7 +1192,7 @@ void compact_pgdat(pg_data_t *pgdat, int order)
 {
        struct compact_control cc = {
                .order = order,
-               .sync = false,
+               .mode = MIGRATE_ASYNC,
        };
 
        if (!order)
@@ -1203,7 +1205,7 @@ static void compact_node(int nid)
 {
        struct compact_control cc = {
                .order = -1,
-               .sync = true,
+               .mode = MIGRATE_SYNC,
                .ignore_skip_hint = true,
        };
 

diff --git a/mm/internal.h b/mm/internal.h
index 6ee580d..a25424a 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -134,7 +134,7 @@ struct compact_control {
        unsigned long nr_migratepages;  /* Number of pages to migrate */
        unsigned long free_pfn;         /* isolate_freepages search base */
        unsigned long migrate_pfn;      /* isolate_migratepages search base */
-       bool sync;                      /* Synchronous migration */
+       enum migrate_mode mode;         /* Async or sync migration mode */
        bool ignore_skip_hint;          /* Scan blocks even if marked skip */
        bool finished_update_free;      /* True when the zone cached pfns are
                                         * no longer being updated

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 027d029..afb29da 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2217,7 +2217,7 @@ static struct page *
 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
        struct zonelist *zonelist, enum zone_type high_zoneidx,
        nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-       int migratetype, bool sync_migration,
+       int migratetype, enum migrate_mode mode,
        bool *contended_compaction, bool *deferred_compaction,
        unsigned long *did_some_progress)
 {
@@ -2231,7 +2231,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 
        current->flags |= PF_MEMALLOC;
        *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
-                                               nodemask, sync_migration,
+                                               nodemask, mode,
                                                contended_compaction);
        current->flags &= ~PF_MEMALLOC;
 
@@ -2264,7 +2264,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
                 * As async compaction considers a subset of pageblocks, only
                 * defer if the failure was a sync compaction failure.
                 */
-               if (sync_migration)
+               if (mode != MIGRATE_ASYNC)
                        defer_compaction(preferred_zone, order);
 
                cond_resched();
@@ -2277,9 +2277,8 @@ static inline struct page *
 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
        struct zonelist *zonelist, enum zone_type high_zoneidx,
        nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-       int migratetype, bool sync_migration,
-       bool *contended_compaction, bool *deferred_compaction,
-       unsigned long *did_some_progress)
+       int migratetype, enum migrate_mode mode, bool *contended_compaction,
+       bool *deferred_compaction, unsigned long *did_some_progress)
 {
        return NULL;
 }
@@ -2474,7 +2473,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
        int alloc_flags;
        unsigned long pages_reclaimed = 0;
        unsigned long did_some_progress;
-       bool sync_migration = false;
+       enum migrate_mode migration_mode = MIGRATE_ASYNC;
        bool deferred_compaction = false;
        bool contended_compaction = false;
 
@@ -2568,17 +2567,15 @@ rebalance:
         * Try direct compaction. The first pass is asynchronous. Subsequent
         * attempts after direct reclaim are synchronous
         */
-       page = __alloc_pages_direct_compact(gfp_mask, order,
-                                       zonelist, high_zoneidx,
-                                       nodemask,
-                                       alloc_flags, preferred_zone,
-                                       migratetype, sync_migration,
-                                       &contended_compaction,
+       page = __alloc_pages_direct_compact(gfp_mask, order, zonelist,
+                                       high_zoneidx, nodemask, alloc_flags,
+                                       preferred_zone, migratetype,
+                                       migration_mode, &contended_compaction,
                                        &deferred_compaction,
                                        &did_some_progress);
        if (page)
                goto got_pg;
-       sync_migration = true;
+       migration_mode = MIGRATE_SYNC_LIGHT;
 
        /*
         * If compaction is deferred for high-order allocations, it is because
@@ -2653,12 +2650,10 @@ rebalance:
                 * direct reclaim and reclaim/compaction depends on compaction
                 * being called after reclaim so call directly if necessary
                 */
-               page = __alloc_pages_direct_compact(gfp_mask, order,
-                                       zonelist, high_zoneidx,
-                                       nodemask,
-                                       alloc_flags, preferred_zone,
-                                       migratetype, sync_migration,
-                                       &contended_compaction,
+               page = __alloc_pages_direct_compact(gfp_mask, order, zonelist,
+                                       high_zoneidx, nodemask, alloc_flags,
+                                       preferred_zone, migratetype,
+                                       migration_mode, &contended_compaction,
                                        &deferred_compaction,
                                        &did_some_progress);
                if (page)
@@ -6218,7 +6213,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc,
                cc->nr_migratepages -= nr_reclaimed;
 
                ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
-                                   NULL, 0, MIGRATE_SYNC, MR_CMA);
+                                   NULL, 0, cc->mode, MR_CMA);
        }
        if (ret < 0) {
                putback_movable_pages(&cc->migratepages);
@@ -6257,7 +6252,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
                .nr_migratepages = 0,
                .order = -1,
                .zone = page_zone(pfn_to_page(start)),
-               .sync = true,
+               .mode = MIGRATE_SYNC,
                .ignore_skip_hint = true,
        };
        INIT_LIST_HEAD(&cc.migratepages);