memcg: fix rmdir, force_empty with THP

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 848b421..5b562b3 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -814,7 +814,8 @@ void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
         * removed from global LRU.
         */
        mz = page_cgroup_zoneinfo(pc);
-       MEM_CGROUP_ZSTAT(mz, lru) -= 1;
+       /* huge page split is done under lru_lock. so, we have no races. */
+       MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
        if (mem_cgroup_is_root(pc->mem_cgroup))
                return;
        VM_BUG_ON(list_empty(&pc->lru));
@@ -865,7 +866,8 @@ void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
                return;
 
        mz = page_cgroup_zoneinfo(pc);
-       MEM_CGROUP_ZSTAT(mz, lru) += 1;
+       /* huge page split is done under lru_lock. so, we have no races. */
+       MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
        SetPageCgroupAcctLRU(pc);
        if (mem_cgroup_is_root(pc->mem_cgroup))
                return;
@@ -2152,14 +2154,26 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
        unsigned long flags;
 
        /*
-        * We have no races witch charge/uncharge but will have races with
+        * We have no races with charge/uncharge but will have races with
         * page state accounting.
         */
        move_lock_page_cgroup(head_pc, &flags);
 
        tail_pc->mem_cgroup = head_pc->mem_cgroup;
        smp_wmb(); /* see __commit_charge() */
-       /* we don't need to copy all flags...*/
+       if (PageCgroupAcctLRU(head_pc)) {
+               enum lru_list lru;
+               struct mem_cgroup_per_zone *mz;
+
+               /*
+                * LRU flags cannot be copied because we need to add tail
+                *.page to LRU by generic call and our hook will be called.
+                * We hold lru_lock, then, reduce counter directly.
+                */
+               lru = page_lru(head);
+               mz = page_cgroup_zoneinfo(head_pc);
+               MEM_CGROUP_ZSTAT(mz, lru) -= 1;
+       }
        tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
        move_unlock_page_cgroup(head_pc, &flags);
 }
@@ -2183,8 +2197,11 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
  */
 
 static void __mem_cgroup_move_account(struct page_cgroup *pc,
-       struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
+       struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge,
+       int charge_size)
 {
+       int nr_pages = charge_size >> PAGE_SHIFT;
+
        VM_BUG_ON(from == to);
        VM_BUG_ON(PageLRU(pc->page));
        VM_BUG_ON(!page_is_cgroup_locked(pc));
@@ -2198,14 +2215,14 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
                __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
                preempt_enable();
        }
-       mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -1);
+       mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages);
        if (uncharge)
                /* This is not "cancel", but cancel_charge does all we need. */
-               mem_cgroup_cancel_charge(from, PAGE_SIZE);
+               mem_cgroup_cancel_charge(from, charge_size);
 
        /* caller should have done css_get */
        pc->mem_cgroup = to;
-       mem_cgroup_charge_statistics(to, PageCgroupCache(pc), 1);
+       mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages);
        /*
         * We charges against "to" which may not have any tasks. Then, "to"
         * can be under rmdir(). But in current implementation, caller of
@@ -2220,15 +2237,19 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
  * __mem_cgroup_move_account()
  */
 static int mem_cgroup_move_account(struct page_cgroup *pc,
-               struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
+               struct mem_cgroup *from, struct mem_cgroup *to,
+               bool uncharge, int charge_size)
 {
        int ret = -EINVAL;
        unsigned long flags;
 
+       if ((charge_size > PAGE_SIZE) && !PageTransHuge(pc->page))
+               return -EBUSY;
+
        lock_page_cgroup(pc);
        if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
                move_lock_page_cgroup(pc, &flags);
-               __mem_cgroup_move_account(pc, from, to, uncharge);
+               __mem_cgroup_move_account(pc, from, to, uncharge, charge_size);
                move_unlock_page_cgroup(pc, &flags);
                ret = 0;
        }
@@ -2253,6 +2274,8 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc,
        struct cgroup *cg = child->css.cgroup;
        struct cgroup *pcg = cg->parent;
        struct mem_cgroup *parent;
+       int charge = PAGE_SIZE;
+       unsigned long flags;
        int ret;
 
        /* Is ROOT ? */
@@ -2264,17 +2287,23 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc,
                goto out;
        if (isolate_lru_page(page))
                goto put;
+       /* The page is isolated from LRU and we have no race with splitting */
+       charge = PAGE_SIZE << compound_order(page);
 
        parent = mem_cgroup_from_cont(pcg);
-       ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false,
-                                     PAGE_SIZE);
+       ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false, charge);
        if (ret || !parent)
                goto put_back;
 
-       ret = mem_cgroup_move_account(pc, child, parent, true);
+       if (charge > PAGE_SIZE)
+               flags = compound_lock_irqsave(page);
+
+       ret = mem_cgroup_move_account(pc, child, parent, true, charge);
        if (ret)
-               mem_cgroup_cancel_charge(parent, PAGE_SIZE);
+               mem_cgroup_cancel_charge(parent, charge);
 put_back:
+       if (charge > PAGE_SIZE)
+               compound_unlock_irqrestore(page, flags);
        putback_lru_page(page);
 put:
        put_page(page);
@@ -4854,7 +4883,7 @@ retry:
                                goto put;
                        pc = lookup_page_cgroup(page);
                        if (!mem_cgroup_move_account(pc,
-                                               mc.from, mc.to, false)) {
+                                       mc.from, mc.to, false, PAGE_SIZE)) {
                                mc.precharge--;
                                /* we uncharge from mc.from later. */
                                mc.moved_charge++;