mm: numa: Migrate on reference policy

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / mempolicy.c

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 75d4600..4c1c8d8 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -118,6 +118,26 @@ static struct mempolicy default_policy = {
        .flags = MPOL_F_LOCAL,
 };
 
+static struct mempolicy preferred_node_policy[MAX_NUMNODES];
+
+static struct mempolicy *get_task_policy(struct task_struct *p)
+{
+       struct mempolicy *pol = p->mempolicy;
+       int node;
+
+       if (!pol) {
+               node = numa_node_id();
+               if (node != -1)
+                       pol = &preferred_node_policy[node];
+
+               /* preferred_node_policy is not initialised early in boot */
+               if (!pol->mode)
+                       pol = NULL;
+       }
+
+       return pol;
+}
+
 static const struct mempolicy_operations {
        int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
        /*
@@ -252,7 +272,7 @@ static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
        pr_debug("setting mode %d flags %d nodes[0] %lx\n",
                 mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
 
-       if (mode == MPOL_DEFAULT || mode == MPOL_NOOP) {
+       if (mode == MPOL_DEFAULT) {
                if (nodes && !nodes_empty(*nodes))
                        return ERR_PTR(-EINVAL);
                return NULL;
@@ -583,6 +603,8 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
        BUILD_BUG_ON(_PAGE_NUMA != _PAGE_PROTNONE);
 
        nr_updated = change_protection(vma, addr, end, vma->vm_page_prot, 0, 1);
+       if (nr_updated)
+               count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
 
        return nr_updated;
 }
@@ -1186,7 +1208,7 @@ static long do_mbind(unsigned long start, unsigned long len,
        if (start & ~PAGE_MASK)
                return -EINVAL;
 
-       if (mode == MPOL_DEFAULT || mode == MPOL_NOOP)
+       if (mode == MPOL_DEFAULT)
                flags &= ~MPOL_MF_STRICT;
 
        len = (len + PAGE_SIZE - 1) & PAGE_MASK;
@@ -1241,7 +1263,7 @@ static long do_mbind(unsigned long start, unsigned long len,
                          flags | MPOL_MF_INVERT, &pagelist);
 
        err = PTR_ERR(vma);     /* maybe ... */
-       if (!IS_ERR(vma) && mode != MPOL_NOOP)
+       if (!IS_ERR(vma))
                err = mbind_range(mm, start, end, new);
 
        if (!err) {
@@ -1596,7 +1618,7 @@ asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
 struct mempolicy *get_vma_policy(struct task_struct *task,
                struct vm_area_struct *vma, unsigned long addr)
 {
-       struct mempolicy *pol = task->mempolicy;
+       struct mempolicy *pol = get_task_policy(task);
 
        if (vma) {
                if (vma->vm_ops && vma->vm_ops->get_policy) {
@@ -2019,7 +2041,7 @@ retry_cpuset:
  */
 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
-       struct mempolicy *pol = current->mempolicy;
+       struct mempolicy *pol = get_task_policy(current);
        struct page *page;
        unsigned int cpuset_mems_cookie;
 
@@ -2293,6 +2315,11 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
        default:
                BUG();
        }
+
+       /* Migrate the page towards the node whose CPU is referencing it */
+       if (pol->flags & MPOL_F_MORON)
+               polnid = numa_node_id();
+
        if (curnid != polnid)
                ret = polnid;
 out:
@@ -2481,6 +2508,15 @@ void __init numa_policy_init(void)
                                     sizeof(struct sp_node),
                                     0, SLAB_PANIC, NULL);
 
+       for_each_node(nid) {
+               preferred_node_policy[nid] = (struct mempolicy) {
+                       .refcnt = ATOMIC_INIT(1),
+                       .mode = MPOL_PREFERRED,
+                       .flags = MPOL_F_MOF | MPOL_F_MORON,
+                       .v = { .preferred_node = nid, },
+               };
+       }
+
        /*
         * Set interleaving policy for system init. Interleaving is only
         * enabled across suitably sized nodes (default is >= 16MB), or
@@ -2530,7 +2566,6 @@ static const char * const policy_modes[] =
        [MPOL_BIND]       = "bind",
        [MPOL_INTERLEAVE] = "interleave",
        [MPOL_LOCAL]      = "local",
-       [MPOL_NOOP]       = "noop",     /* should not actually be used */
 };
 
 
@@ -2581,7 +2616,7 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                        break;
                }
        }
-       if (mode >= MPOL_MAX || mode == MPOL_NOOP)
+       if (mode >= MPOL_MAX)
                goto out;
 
        switch (mode) {