mm/swap_cgroup.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/swap_cgroup.h>
   3 #include <linux/vmalloc.h>
   4 #include <linux/mm.h>
   5
   6 #include <linux/swapops.h> /* depends on mm.h include */
   7
   8 static DEFINE_MUTEX(swap_cgroup_mutex);
   9 struct swap_cgroup_ctrl {
  10         struct page **map;
  11         unsigned long length;
  12         spinlock_t      lock;
  13 };
  14
  15 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
  16
  17 struct swap_cgroup {
  18         unsigned short          id;
  19 };
  20 #define SC_PER_PAGE     (PAGE_SIZE/sizeof(struct swap_cgroup))
  21
  22 /*
  23  * SwapCgroup implements "lookup" and "exchange" operations.
  24  * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
  25  * against SwapCache. At swap_free(), this is accessed directly from swap.
  26  *
  27  * This means,
  28  *  - we have no race in "exchange" when we're accessed via SwapCache because
  29  *    SwapCache(and its swp_entry) is under lock.
  30  *  - When called via swap_free(), there is no user of this entry and no race.
  31  * Then, we don't need lock around "exchange".
  32  *
  33  * TODO: we can push these buffers out to HIGHMEM.
  34  */
  35
  36 /*
  37  * allocate buffer for swap_cgroup.
  38  */
  39 static int swap_cgroup_prepare(int type)
  40 {
  41         struct page *page;
  42         struct swap_cgroup_ctrl *ctrl;
  43         unsigned long idx, max;
  44
  45         ctrl = &swap_cgroup_ctrl[type];
  46
  47         for (idx = 0; idx < ctrl->length; idx++) {
  48                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  49                 if (!page)
  50                         goto not_enough_page;
  51                 ctrl->map[idx] = page;
  52
  53                 if (!(idx % SWAP_CLUSTER_MAX))
  54                         cond_resched();
  55         }
  56         return 0;
  57 not_enough_page:
  58         max = idx;
  59         for (idx = 0; idx < max; idx++)
  60                 __free_page(ctrl->map[idx]);
  61
  62         return -ENOMEM;
  63 }
  64
  65 static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
  66                                                 pgoff_t offset)
  67 {
  68         struct page *mappage;
  69         struct swap_cgroup *sc;
  70
  71         mappage = ctrl->map[offset / SC_PER_PAGE];
  72         sc = page_address(mappage);
  73         return sc + offset % SC_PER_PAGE;
  74 }
  75
  76 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
  77                                         struct swap_cgroup_ctrl **ctrlp)
  78 {
  79         pgoff_t offset = swp_offset(ent);
  80         struct swap_cgroup_ctrl *ctrl;
  81
  82         ctrl = &swap_cgroup_ctrl[swp_type(ent)];
  83         if (ctrlp)
  84                 *ctrlp = ctrl;
  85         return __lookup_swap_cgroup(ctrl, offset);
  86 }
  87
  88 /**
  89  * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
  90  * @ent: swap entry to be cmpxchged
  91  * @old: old id
  92  * @new: new id
  93  *
  94  * Returns old id at success, 0 at failure.
  95  * (There is no mem_cgroup using 0 as its id)
  96  */
  97 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
  98                                         unsigned short old, unsigned short new)
  99 {
 100         struct swap_cgroup_ctrl *ctrl;
 101         struct swap_cgroup *sc;
 102         unsigned long flags;
 103         unsigned short retval;
 104
 105         sc = lookup_swap_cgroup(ent, &ctrl);
 106
 107         spin_lock_irqsave(&ctrl->lock, flags);
 108         retval = sc->id;
 109         if (retval == old)
 110                 sc->id = new;
 111         else
 112                 retval = 0;
 113         spin_unlock_irqrestore(&ctrl->lock, flags);
 114         return retval;
 115 }
 116
 117 /**
 118  * swap_cgroup_record - record mem_cgroup for a set of swap entries
 119  * @ent: the first swap entry to be recorded into
 120  * @id: mem_cgroup to be recorded
 121  * @nr_ents: number of swap entries to be recorded
 122  *
 123  * Returns old value at success, 0 at failure.
 124  * (Of course, old value can be 0.)
 125  */
 126 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
 127                                   unsigned int nr_ents)
 128 {
 129         struct swap_cgroup_ctrl *ctrl;
 130         struct swap_cgroup *sc;
 131         unsigned short old;
 132         unsigned long flags;
 133         pgoff_t offset = swp_offset(ent);
 134         pgoff_t end = offset + nr_ents;
 135
 136         sc = lookup_swap_cgroup(ent, &ctrl);
 137
 138         spin_lock_irqsave(&ctrl->lock, flags);
 139         old = sc->id;
 140         for (;;) {
 141                 VM_BUG_ON(sc->id != old);
 142                 sc->id = id;
 143                 offset++;
 144                 if (offset == end)
 145                         break;
 146                 if (offset % SC_PER_PAGE)
 147                         sc++;
 148                 else
 149                         sc = __lookup_swap_cgroup(ctrl, offset);
 150         }
 151         spin_unlock_irqrestore(&ctrl->lock, flags);
 152
 153         return old;
 154 }
 155
 156 /**
 157  * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
 158  * @ent: swap entry to be looked up.
 159  *
 160  * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 161  */
 162 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
 163 {
 164         return lookup_swap_cgroup(ent, NULL)->id;
 165 }
 166
 167 int swap_cgroup_swapon(int type, unsigned long max_pages)
 168 {
 169         void *array;
 170         unsigned long array_size;
 171         unsigned long length;
 172         struct swap_cgroup_ctrl *ctrl;
 173
 174         if (!do_swap_account)
 175                 return 0;
 176
 177         length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
 178         array_size = length * sizeof(void *);
 179
 180         array = vzalloc(array_size);
 181         if (!array)
 182                 goto nomem;
 183
 184         ctrl = &swap_cgroup_ctrl[type];
 185         mutex_lock(&swap_cgroup_mutex);
 186         ctrl->length = length;
 187         ctrl->map = array;
 188         spin_lock_init(&ctrl->lock);
 189         if (swap_cgroup_prepare(type)) {
 190                 /* memory shortage */
 191                 ctrl->map = NULL;
 192                 ctrl->length = 0;
 193                 mutex_unlock(&swap_cgroup_mutex);
 194                 vfree(array);
 195                 goto nomem;
 196         }
 197         mutex_unlock(&swap_cgroup_mutex);
 198
 199         return 0;
 200 nomem:
 201         pr_info("couldn't allocate enough memory for swap_cgroup\n");
 202         pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
 203         return -ENOMEM;
 204 }
 205
 206 void swap_cgroup_swapoff(int type)
 207 {
 208         struct page **map;
 209         unsigned long i, length;
 210         struct swap_cgroup_ctrl *ctrl;
 211
 212         if (!do_swap_account)
 213                 return;
 214
 215         mutex_lock(&swap_cgroup_mutex);
 216         ctrl = &swap_cgroup_ctrl[type];
 217         map = ctrl->map;
 218         length = ctrl->length;
 219         ctrl->map = NULL;
 220         ctrl->length = 0;
 221         mutex_unlock(&swap_cgroup_mutex);
 222
 223         if (map) {
 224                 for (i = 0; i < length; i++) {
 225                         struct page *page = map[i];
 226                         if (page)
 227                                 __free_page(page);
 228                         if (!(i % SWAP_CLUSTER_MAX))
 229                                 cond_resched();
 230                 }
 231                 vfree(map);
 232         }
 233 }