Merge tag 'perf-urgent-2023-10-21' of git://git.kernel.org/pub/scm/linux/kernel/git...

[platform/kernel/linux-starfive.git] / mm / mempolicy.c

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 42b5567..f1b00d6 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -426,6 +426,7 @@ struct queue_pages {
        unsigned long start;
        unsigned long end;
        struct vm_area_struct *first;
+       bool has_unmovable;
 };
 
 /*
@@ -446,9 +447,8 @@ static inline bool queue_folio_required(struct folio *folio,
 /*
  * queue_folios_pmd() has three possible return values:
  * 0 - folios are placed on the right node or queued successfully, or
- *     special page is met, i.e. huge zero page.
- * 1 - there is unmovable folio, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
- *     specified.
+ *     special page is met, i.e. zero page, or unmovable page is found
+ *     but continue walking (indicated by queue_pages.has_unmovable).
  * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an
  *        existing folio was already on a node that does not follow the
  *        policy.
@@ -479,7 +479,7 @@ static int queue_folios_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr,
        if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
                if (!vma_migratable(walk->vma) ||
                    migrate_folio_add(folio, qp->pagelist, flags)) {
-                       ret = 1;
+                       qp->has_unmovable = true;
                        goto unlock;
                }
        } else
@@ -495,9 +495,8 @@ unlock:
  *
  * queue_folios_pte_range() has three possible return values:
  * 0 - folios are placed on the right node or queued successfully, or
- *     special page is met, i.e. zero page.
- * 1 - there is unmovable folio, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
- *     specified.
+ *     special page is met, i.e. zero page, or unmovable page is found
+ *     but continue walking (indicated by queue_pages.has_unmovable).
  * -EIO - only MPOL_MF_STRICT was specified and an existing folio was already
  *        on a node that does not follow the policy.
  */
@@ -508,7 +507,6 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
        struct folio *folio;
        struct queue_pages *qp = walk->private;
        unsigned long flags = qp->flags;
-       bool has_unmovable = false;
        pte_t *pte, *mapped_pte;
        pte_t ptent;
        spinlock_t *ptl;
@@ -538,11 +536,12 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
                if (!queue_folio_required(folio, qp))
                        continue;
                if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
-                       /* MPOL_MF_STRICT must be specified if we get here */
-                       if (!vma_migratable(vma)) {
-                               has_unmovable = true;
-                               break;
-                       }
+                       /*
+                        * MPOL_MF_STRICT must be specified if we get here.
+                        * Continue walking vmas due to MPOL_MF_MOVE* flags.
+                        */
+                       if (!vma_migratable(vma))
+                               qp->has_unmovable = true;
 
                        /*
                         * Do not abort immediately since there may be
@@ -550,16 +549,13 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
                         * need migrate other LRU pages.
                         */
                        if (migrate_folio_add(folio, qp->pagelist, flags))
-                               has_unmovable = true;
+                               qp->has_unmovable = true;
                } else
                        break;
        }
        pte_unmap_unlock(mapped_pte, ptl);
        cond_resched();
 
-       if (has_unmovable)
-               return 1;
-
        return addr != end ? -EIO : 0;
 }
 
@@ -599,7 +595,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,
                 * Detecting misplaced folio but allow migrating folios which
                 * have been queued.
                 */
-               ret = 1;
+               qp->has_unmovable = true;
                goto unlock;
        }
 
@@ -620,7 +616,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,
                         * Failed to isolate folio but allow migrating pages
                         * which have been queued.
                         */
-                       ret = 1;
+                       qp->has_unmovable = true;
        }
 unlock:
        spin_unlock(ptl);
@@ -756,12 +752,15 @@ queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
                .start = start,
                .end = end,
                .first = NULL,
+               .has_unmovable = false,
        };
        const struct mm_walk_ops *ops = lock_vma ?
                        &queue_pages_lock_vma_walk_ops : &queue_pages_walk_ops;
 
        err = walk_page_range(mm, start, end, ops, &qp);
 
+       if (qp.has_unmovable)
+               err = 1;
        if (!qp.first)
                /* whole range in hole */
                err = -EFAULT;
@@ -1358,7 +1357,7 @@ static long do_mbind(unsigned long start, unsigned long len,
                                putback_movable_pages(&pagelist);
                }
 
-               if ((ret > 0) || (nr_failed && (flags & MPOL_MF_STRICT)))
+               if (((ret > 0) || nr_failed) && (flags & MPOL_MF_STRICT))
                        err = -EIO;
        } else {
 up_out: