swap: fix races exposed by swap discard
[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / swapfile.c
1 /*
2  *  linux/mm/swapfile.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *  Swap reorganised 29.12.95, Stephen Tweedie
6  */
7
8 #include <linux/mm.h>
9 #include <linux/hugetlb.h>
10 #include <linux/mman.h>
11 #include <linux/slab.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/swap.h>
14 #include <linux/vmalloc.h>
15 #include <linux/pagemap.h>
16 #include <linux/namei.h>
17 #include <linux/shmem_fs.h>
18 #include <linux/blkdev.h>
19 #include <linux/random.h>
20 #include <linux/writeback.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/init.h>
24 #include <linux/ksm.h>
25 #include <linux/rmap.h>
26 #include <linux/security.h>
27 #include <linux/backing-dev.h>
28 #include <linux/mutex.h>
29 #include <linux/capability.h>
30 #include <linux/syscalls.h>
31 #include <linux/memcontrol.h>
32 #include <linux/poll.h>
33 #include <linux/oom.h>
34 #include <linux/frontswap.h>
35 #include <linux/swapfile.h>
36 #include <linux/export.h>
37
38 #include <asm/pgtable.h>
39 #include <asm/tlbflush.h>
40 #include <linux/swapops.h>
41 #include <linux/page_cgroup.h>
42
43 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
44                                  unsigned char);
45 static void free_swap_count_continuations(struct swap_info_struct *);
46 static sector_t map_swap_entry(swp_entry_t, struct block_device**);
47
48 DEFINE_SPINLOCK(swap_lock);
49 static unsigned int nr_swapfiles;
50 atomic_long_t nr_swap_pages;
51 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
52 long total_swap_pages;
53 static int least_priority;
54 static atomic_t highest_priority_index = ATOMIC_INIT(-1);
55
56 static const char Bad_file[] = "Bad swap file entry ";
57 static const char Unused_file[] = "Unused swap file entry ";
58 static const char Bad_offset[] = "Bad swap offset entry ";
59 static const char Unused_offset[] = "Unused swap offset entry ";
60
61 struct swap_list_t swap_list = {-1, -1};
62
63 struct swap_info_struct *swap_info[MAX_SWAPFILES];
64
65 static DEFINE_MUTEX(swapon_mutex);
66
67 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
68 /* Activity counter to indicate that a swapon or swapoff has occurred */
69 static atomic_t proc_poll_event = ATOMIC_INIT(0);
70
71 static inline unsigned char swap_count(unsigned char ent)
72 {
73         return ent & ~SWAP_HAS_CACHE;   /* may include SWAP_HAS_CONT flag */
74 }
75
76 /* returns 1 if swap entry is freed */
77 static int
78 __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
79 {
80         swp_entry_t entry = swp_entry(si->type, offset);
81         struct page *page;
82         int ret = 0;
83
84         page = find_get_page(swap_address_space(entry), entry.val);
85         if (!page)
86                 return 0;
87         /*
88          * This function is called from scan_swap_map() and it's called
89          * by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
90          * We have to use trylock for avoiding deadlock. This is a special
91          * case and you should use try_to_free_swap() with explicit lock_page()
92          * in usual operations.
93          */
94         if (trylock_page(page)) {
95                 ret = try_to_free_swap(page);
96                 unlock_page(page);
97         }
98         page_cache_release(page);
99         return ret;
100 }
101
102 /*
103  * swapon tell device that all the old swap contents can be discarded,
104  * to allow the swap device to optimize its wear-levelling.
105  */
106 static int discard_swap(struct swap_info_struct *si)
107 {
108         struct swap_extent *se;
109         sector_t start_block;
110         sector_t nr_blocks;
111         int err = 0;
112
113         /* Do not discard the swap header page! */
114         se = &si->first_swap_extent;
115         start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
116         nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
117         if (nr_blocks) {
118                 err = blkdev_issue_discard(si->bdev, start_block,
119                                 nr_blocks, GFP_KERNEL, 0);
120                 if (err)
121                         return err;
122                 cond_resched();
123         }
124
125         list_for_each_entry(se, &si->first_swap_extent.list, list) {
126                 start_block = se->start_block << (PAGE_SHIFT - 9);
127                 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
128
129                 err = blkdev_issue_discard(si->bdev, start_block,
130                                 nr_blocks, GFP_KERNEL, 0);
131                 if (err)
132                         break;
133
134                 cond_resched();
135         }
136         return err;             /* That will often be -EOPNOTSUPP */
137 }
138
139 /*
140  * swap allocation tell device that a cluster of swap can now be discarded,
141  * to allow the swap device to optimize its wear-levelling.
142  */
143 static void discard_swap_cluster(struct swap_info_struct *si,
144                                  pgoff_t start_page, pgoff_t nr_pages)
145 {
146         struct swap_extent *se = si->curr_swap_extent;
147         int found_extent = 0;
148
149         while (nr_pages) {
150                 struct list_head *lh;
151
152                 if (se->start_page <= start_page &&
153                     start_page < se->start_page + se->nr_pages) {
154                         pgoff_t offset = start_page - se->start_page;
155                         sector_t start_block = se->start_block + offset;
156                         sector_t nr_blocks = se->nr_pages - offset;
157
158                         if (nr_blocks > nr_pages)
159                                 nr_blocks = nr_pages;
160                         start_page += nr_blocks;
161                         nr_pages -= nr_blocks;
162
163                         if (!found_extent++)
164                                 si->curr_swap_extent = se;
165
166                         start_block <<= PAGE_SHIFT - 9;
167                         nr_blocks <<= PAGE_SHIFT - 9;
168                         if (blkdev_issue_discard(si->bdev, start_block,
169                                     nr_blocks, GFP_NOIO, 0))
170                                 break;
171                 }
172
173                 lh = se->list.next;
174                 se = list_entry(lh, struct swap_extent, list);
175         }
176 }
177
178 #define SWAPFILE_CLUSTER        256
179 #define LATENCY_LIMIT           256
180
181 static inline void cluster_set_flag(struct swap_cluster_info *info,
182         unsigned int flag)
183 {
184         info->flags = flag;
185 }
186
187 static inline unsigned int cluster_count(struct swap_cluster_info *info)
188 {
189         return info->data;
190 }
191
192 static inline void cluster_set_count(struct swap_cluster_info *info,
193                                      unsigned int c)
194 {
195         info->data = c;
196 }
197
198 static inline void cluster_set_count_flag(struct swap_cluster_info *info,
199                                          unsigned int c, unsigned int f)
200 {
201         info->flags = f;
202         info->data = c;
203 }
204
205 static inline unsigned int cluster_next(struct swap_cluster_info *info)
206 {
207         return info->data;
208 }
209
210 static inline void cluster_set_next(struct swap_cluster_info *info,
211                                     unsigned int n)
212 {
213         info->data = n;
214 }
215
216 static inline void cluster_set_next_flag(struct swap_cluster_info *info,
217                                          unsigned int n, unsigned int f)
218 {
219         info->flags = f;
220         info->data = n;
221 }
222
223 static inline bool cluster_is_free(struct swap_cluster_info *info)
224 {
225         return info->flags & CLUSTER_FLAG_FREE;
226 }
227
228 static inline bool cluster_is_null(struct swap_cluster_info *info)
229 {
230         return info->flags & CLUSTER_FLAG_NEXT_NULL;
231 }
232
233 static inline void cluster_set_null(struct swap_cluster_info *info)
234 {
235         info->flags = CLUSTER_FLAG_NEXT_NULL;
236         info->data = 0;
237 }
238
239 /* Add a cluster to discard list and schedule it to do discard */
240 static void swap_cluster_schedule_discard(struct swap_info_struct *si,
241                 unsigned int idx)
242 {
243         /*
244          * If scan_swap_map() can't find a free cluster, it will check
245          * si->swap_map directly. To make sure the discarding cluster isn't
246          * taken by scan_swap_map(), mark the swap entries bad (occupied). It
247          * will be cleared after discard
248          */
249         memset(si->swap_map + idx * SWAPFILE_CLUSTER,
250                         SWAP_MAP_BAD, SWAPFILE_CLUSTER);
251
252         if (cluster_is_null(&si->discard_cluster_head)) {
253                 cluster_set_next_flag(&si->discard_cluster_head,
254                                                 idx, 0);
255                 cluster_set_next_flag(&si->discard_cluster_tail,
256                                                 idx, 0);
257         } else {
258                 unsigned int tail = cluster_next(&si->discard_cluster_tail);
259                 cluster_set_next(&si->cluster_info[tail], idx);
260                 cluster_set_next_flag(&si->discard_cluster_tail,
261                                                 idx, 0);
262         }
263
264         schedule_work(&si->discard_work);
265 }
266
267 /*
268  * Doing discard actually. After a cluster discard is finished, the cluster
269  * will be added to free cluster list. caller should hold si->lock.
270 */
271 static void swap_do_scheduled_discard(struct swap_info_struct *si)
272 {
273         struct swap_cluster_info *info;
274         unsigned int idx;
275
276         info = si->cluster_info;
277
278         while (!cluster_is_null(&si->discard_cluster_head)) {
279                 idx = cluster_next(&si->discard_cluster_head);
280
281                 cluster_set_next_flag(&si->discard_cluster_head,
282                                                 cluster_next(&info[idx]), 0);
283                 if (cluster_next(&si->discard_cluster_tail) == idx) {
284                         cluster_set_null(&si->discard_cluster_head);
285                         cluster_set_null(&si->discard_cluster_tail);
286                 }
287                 spin_unlock(&si->lock);
288
289                 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
290                                 SWAPFILE_CLUSTER);
291
292                 spin_lock(&si->lock);
293                 cluster_set_flag(&info[idx], CLUSTER_FLAG_FREE);
294                 if (cluster_is_null(&si->free_cluster_head)) {
295                         cluster_set_next_flag(&si->free_cluster_head,
296                                                 idx, 0);
297                         cluster_set_next_flag(&si->free_cluster_tail,
298                                                 idx, 0);
299                 } else {
300                         unsigned int tail;
301
302                         tail = cluster_next(&si->free_cluster_tail);
303                         cluster_set_next(&info[tail], idx);
304                         cluster_set_next_flag(&si->free_cluster_tail,
305                                                 idx, 0);
306                 }
307                 memset(si->swap_map + idx * SWAPFILE_CLUSTER,
308                                 0, SWAPFILE_CLUSTER);
309         }
310 }
311
312 static void swap_discard_work(struct work_struct *work)
313 {
314         struct swap_info_struct *si;
315
316         si = container_of(work, struct swap_info_struct, discard_work);
317
318         spin_lock(&si->lock);
319         swap_do_scheduled_discard(si);
320         spin_unlock(&si->lock);
321 }
322
323 /*
324  * The cluster corresponding to page_nr will be used. The cluster will be
325  * removed from free cluster list and its usage counter will be increased.
326  */
327 static void inc_cluster_info_page(struct swap_info_struct *p,
328         struct swap_cluster_info *cluster_info, unsigned long page_nr)
329 {
330         unsigned long idx = page_nr / SWAPFILE_CLUSTER;
331
332         if (!cluster_info)
333                 return;
334         if (cluster_is_free(&cluster_info[idx])) {
335                 VM_BUG_ON(cluster_next(&p->free_cluster_head) != idx);
336                 cluster_set_next_flag(&p->free_cluster_head,
337                         cluster_next(&cluster_info[idx]), 0);
338                 if (cluster_next(&p->free_cluster_tail) == idx) {
339                         cluster_set_null(&p->free_cluster_tail);
340                         cluster_set_null(&p->free_cluster_head);
341                 }
342                 cluster_set_count_flag(&cluster_info[idx], 0, 0);
343         }
344
345         VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
346         cluster_set_count(&cluster_info[idx],
347                 cluster_count(&cluster_info[idx]) + 1);
348 }
349
350 /*
351  * The cluster corresponding to page_nr decreases one usage. If the usage
352  * counter becomes 0, which means no page in the cluster is in using, we can
353  * optionally discard the cluster and add it to free cluster list.
354  */
355 static void dec_cluster_info_page(struct swap_info_struct *p,
356         struct swap_cluster_info *cluster_info, unsigned long page_nr)
357 {
358         unsigned long idx = page_nr / SWAPFILE_CLUSTER;
359
360         if (!cluster_info)
361                 return;
362
363         VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
364         cluster_set_count(&cluster_info[idx],
365                 cluster_count(&cluster_info[idx]) - 1);
366
367         if (cluster_count(&cluster_info[idx]) == 0) {
368                 /*
369                  * If the swap is discardable, prepare discard the cluster
370                  * instead of free it immediately. The cluster will be freed
371                  * after discard.
372                  */
373                 if ((p->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
374                                  (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
375                         swap_cluster_schedule_discard(p, idx);
376                         return;
377                 }
378
379                 cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
380                 if (cluster_is_null(&p->free_cluster_head)) {
381                         cluster_set_next_flag(&p->free_cluster_head, idx, 0);
382                         cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
383                 } else {
384                         unsigned int tail = cluster_next(&p->free_cluster_tail);
385                         cluster_set_next(&cluster_info[tail], idx);
386                         cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
387                 }
388         }
389 }
390
391 /*
392  * It's possible scan_swap_map() uses a free cluster in the middle of free
393  * cluster list. Avoiding such abuse to avoid list corruption.
394  */
395 static inline bool scan_swap_map_recheck_cluster(struct swap_info_struct *si,
396         unsigned long offset)
397 {
398         offset /= SWAPFILE_CLUSTER;
399         return !cluster_is_null(&si->free_cluster_head) &&
400                 offset != cluster_next(&si->free_cluster_head) &&
401                 cluster_is_free(&si->cluster_info[offset]);
402 }
403
404 static unsigned long scan_swap_map(struct swap_info_struct *si,
405                                    unsigned char usage)
406 {
407         unsigned long offset;
408         unsigned long scan_base;
409         unsigned long last_in_cluster = 0;
410         int latency_ration = LATENCY_LIMIT;
411
412         /*
413          * We try to cluster swap pages by allocating them sequentially
414          * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
415          * way, however, we resort to first-free allocation, starting
416          * a new cluster.  This prevents us from scattering swap pages
417          * all over the entire swap partition, so that we reduce
418          * overall disk seek times between swap pages.  -- sct
419          * But we do now try to find an empty cluster.  -Andrea
420          * And we let swap pages go all over an SSD partition.  Hugh
421          */
422
423         si->flags += SWP_SCANNING;
424         scan_base = offset = si->cluster_next;
425
426         if (unlikely(!si->cluster_nr--)) {
427                 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
428                         si->cluster_nr = SWAPFILE_CLUSTER - 1;
429                         goto checks;
430                 }
431 check_cluster:
432                 if (!cluster_is_null(&si->free_cluster_head)) {
433                         offset = cluster_next(&si->free_cluster_head) *
434                                                 SWAPFILE_CLUSTER;
435                         last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
436                         si->cluster_next = offset;
437                         si->cluster_nr = SWAPFILE_CLUSTER - 1;
438                         goto checks;
439                 } else if (si->cluster_info) {
440                         /*
441                          * we don't have free cluster but have some clusters in
442                          * discarding, do discard now and reclaim them
443                          */
444                         if (!cluster_is_null(&si->discard_cluster_head)) {
445                                 si->cluster_nr = 0;
446                                 swap_do_scheduled_discard(si);
447                                 scan_base = offset = si->cluster_next;
448                                 if (!si->cluster_nr)
449                                         goto check_cluster;
450                                 si->cluster_nr--;
451                                 goto checks;
452                         }
453
454                         /*
455                          * Checking free cluster is fast enough, we can do the
456                          * check every time
457                          */
458                         si->cluster_nr = 0;
459                         goto checks;
460                 }
461
462                 spin_unlock(&si->lock);
463
464                 /*
465                  * If seek is expensive, start searching for new cluster from
466                  * start of partition, to minimize the span of allocated swap.
467                  * But if seek is cheap, search from our current position, so
468                  * that swap is allocated from all over the partition: if the
469                  * Flash Translation Layer only remaps within limited zones,
470                  * we don't want to wear out the first zone too quickly.
471                  */
472                 if (!(si->flags & SWP_SOLIDSTATE))
473                         scan_base = offset = si->lowest_bit;
474                 last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
475
476                 /* Locate the first empty (unaligned) cluster */
477                 for (; last_in_cluster <= si->highest_bit; offset++) {
478                         if (si->swap_map[offset])
479                                 last_in_cluster = offset + SWAPFILE_CLUSTER;
480                         else if (offset == last_in_cluster) {
481                                 spin_lock(&si->lock);
482                                 offset -= SWAPFILE_CLUSTER - 1;
483                                 si->cluster_next = offset;
484                                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
485                                 goto checks;
486                         }
487                         if (unlikely(--latency_ration < 0)) {
488                                 cond_resched();
489                                 latency_ration = LATENCY_LIMIT;
490                         }
491                 }
492
493                 offset = si->lowest_bit;
494                 last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
495
496                 /* Locate the first empty (unaligned) cluster */
497                 for (; last_in_cluster < scan_base; offset++) {
498                         if (si->swap_map[offset])
499                                 last_in_cluster = offset + SWAPFILE_CLUSTER;
500                         else if (offset == last_in_cluster) {
501                                 spin_lock(&si->lock);
502                                 offset -= SWAPFILE_CLUSTER - 1;
503                                 si->cluster_next = offset;
504                                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
505                                 goto checks;
506                         }
507                         if (unlikely(--latency_ration < 0)) {
508                                 cond_resched();
509                                 latency_ration = LATENCY_LIMIT;
510                         }
511                 }
512
513                 offset = scan_base;
514                 spin_lock(&si->lock);
515                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
516         }
517
518 checks:
519         if (scan_swap_map_recheck_cluster(si, offset))
520                 goto check_cluster;
521         if (!(si->flags & SWP_WRITEOK))
522                 goto no_page;
523         if (!si->highest_bit)
524                 goto no_page;
525         if (offset > si->highest_bit)
526                 scan_base = offset = si->lowest_bit;
527
528         /* reuse swap entry of cache-only swap if not busy. */
529         if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
530                 int swap_was_freed;
531                 spin_unlock(&si->lock);
532                 swap_was_freed = __try_to_reclaim_swap(si, offset);
533                 spin_lock(&si->lock);
534                 /* entry was freed successfully, try to use this again */
535                 if (swap_was_freed)
536                         goto checks;
537                 goto scan; /* check next one */
538         }
539
540         if (si->swap_map[offset])
541                 goto scan;
542
543         if (offset == si->lowest_bit)
544                 si->lowest_bit++;
545         if (offset == si->highest_bit)
546                 si->highest_bit--;
547         si->inuse_pages++;
548         if (si->inuse_pages == si->pages) {
549                 si->lowest_bit = si->max;
550                 si->highest_bit = 0;
551         }
552         si->swap_map[offset] = usage;
553         inc_cluster_info_page(si, si->cluster_info, offset);
554         si->cluster_next = offset + 1;
555         si->flags -= SWP_SCANNING;
556
557         return offset;
558
559 scan:
560         spin_unlock(&si->lock);
561         while (++offset <= si->highest_bit) {
562                 if (!si->swap_map[offset]) {
563                         spin_lock(&si->lock);
564                         goto checks;
565                 }
566                 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
567                         spin_lock(&si->lock);
568                         goto checks;
569                 }
570                 if (unlikely(--latency_ration < 0)) {
571                         cond_resched();
572                         latency_ration = LATENCY_LIMIT;
573                 }
574         }
575         offset = si->lowest_bit;
576         while (++offset < scan_base) {
577                 if (!si->swap_map[offset]) {
578                         spin_lock(&si->lock);
579                         goto checks;
580                 }
581                 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
582                         spin_lock(&si->lock);
583                         goto checks;
584                 }
585                 if (unlikely(--latency_ration < 0)) {
586                         cond_resched();
587                         latency_ration = LATENCY_LIMIT;
588                 }
589         }
590         spin_lock(&si->lock);
591
592 no_page:
593         si->flags -= SWP_SCANNING;
594         return 0;
595 }
596
597 swp_entry_t get_swap_page(void)
598 {
599         struct swap_info_struct *si;
600         pgoff_t offset;
601         int type, next;
602         int wrapped = 0;
603         int hp_index;
604
605         spin_lock(&swap_lock);
606         if (atomic_long_read(&nr_swap_pages) <= 0)
607                 goto noswap;
608         atomic_long_dec(&nr_swap_pages);
609
610         for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
611                 hp_index = atomic_xchg(&highest_priority_index, -1);
612                 /*
613                  * highest_priority_index records current highest priority swap
614                  * type which just frees swap entries. If its priority is
615                  * higher than that of swap_list.next swap type, we use it.  It
616                  * isn't protected by swap_lock, so it can be an invalid value
617                  * if the corresponding swap type is swapoff. We double check
618                  * the flags here. It's even possible the swap type is swapoff
619                  * and swapon again and its priority is changed. In such rare
620                  * case, low prority swap type might be used, but eventually
621                  * high priority swap will be used after several rounds of
622                  * swap.
623                  */
624                 if (hp_index != -1 && hp_index != type &&
625                     swap_info[type]->prio < swap_info[hp_index]->prio &&
626                     (swap_info[hp_index]->flags & SWP_WRITEOK)) {
627                         type = hp_index;
628                         swap_list.next = type;
629                 }
630
631                 si = swap_info[type];
632                 next = si->next;
633                 if (next < 0 ||
634                     (!wrapped && si->prio != swap_info[next]->prio)) {
635                         next = swap_list.head;
636                         wrapped++;
637                 }
638
639                 spin_lock(&si->lock);
640                 if (!si->highest_bit) {
641                         spin_unlock(&si->lock);
642                         continue;
643                 }
644                 if (!(si->flags & SWP_WRITEOK)) {
645                         spin_unlock(&si->lock);
646                         continue;
647                 }
648
649                 swap_list.next = next;
650
651                 spin_unlock(&swap_lock);
652                 /* This is called for allocating swap entry for cache */
653                 offset = scan_swap_map(si, SWAP_HAS_CACHE);
654                 spin_unlock(&si->lock);
655                 if (offset)
656                         return swp_entry(type, offset);
657                 spin_lock(&swap_lock);
658                 next = swap_list.next;
659         }
660
661         atomic_long_inc(&nr_swap_pages);
662 noswap:
663         spin_unlock(&swap_lock);
664         return (swp_entry_t) {0};
665 }
666
667 /* The only caller of this function is now susupend routine */
668 swp_entry_t get_swap_page_of_type(int type)
669 {
670         struct swap_info_struct *si;
671         pgoff_t offset;
672
673         si = swap_info[type];
674         spin_lock(&si->lock);
675         if (si && (si->flags & SWP_WRITEOK)) {
676                 atomic_long_dec(&nr_swap_pages);
677                 /* This is called for allocating swap entry, not cache */
678                 offset = scan_swap_map(si, 1);
679                 if (offset) {
680                         spin_unlock(&si->lock);
681                         return swp_entry(type, offset);
682                 }
683                 atomic_long_inc(&nr_swap_pages);
684         }
685         spin_unlock(&si->lock);
686         return (swp_entry_t) {0};
687 }
688
689 static struct swap_info_struct *swap_info_get(swp_entry_t entry)
690 {
691         struct swap_info_struct *p;
692         unsigned long offset, type;
693
694         if (!entry.val)
695                 goto out;
696         type = swp_type(entry);
697         if (type >= nr_swapfiles)
698                 goto bad_nofile;
699         p = swap_info[type];
700         if (!(p->flags & SWP_USED))
701                 goto bad_device;
702         offset = swp_offset(entry);
703         if (offset >= p->max)
704                 goto bad_offset;
705         if (!p->swap_map[offset])
706                 goto bad_free;
707         spin_lock(&p->lock);
708         return p;
709
710 bad_free:
711         pr_err("swap_free: %s%08lx\n", Unused_offset, entry.val);
712         goto out;
713 bad_offset:
714         pr_err("swap_free: %s%08lx\n", Bad_offset, entry.val);
715         goto out;
716 bad_device:
717         pr_err("swap_free: %s%08lx\n", Unused_file, entry.val);
718         goto out;
719 bad_nofile:
720         pr_err("swap_free: %s%08lx\n", Bad_file, entry.val);
721 out:
722         return NULL;
723 }
724
725 /*
726  * This swap type frees swap entry, check if it is the highest priority swap
727  * type which just frees swap entry. get_swap_page() uses
728  * highest_priority_index to search highest priority swap type. The
729  * swap_info_struct.lock can't protect us if there are multiple swap types
730  * active, so we use atomic_cmpxchg.
731  */
732 static void set_highest_priority_index(int type)
733 {
734         int old_hp_index, new_hp_index;
735
736         do {
737                 old_hp_index = atomic_read(&highest_priority_index);
738                 if (old_hp_index != -1 &&
739                         swap_info[old_hp_index]->prio >= swap_info[type]->prio)
740                         break;
741                 new_hp_index = type;
742         } while (atomic_cmpxchg(&highest_priority_index,
743                 old_hp_index, new_hp_index) != old_hp_index);
744 }
745
746 static unsigned char swap_entry_free(struct swap_info_struct *p,
747                                      swp_entry_t entry, unsigned char usage)
748 {
749         unsigned long offset = swp_offset(entry);
750         unsigned char count;
751         unsigned char has_cache;
752
753         count = p->swap_map[offset];
754         has_cache = count & SWAP_HAS_CACHE;
755         count &= ~SWAP_HAS_CACHE;
756
757         if (usage == SWAP_HAS_CACHE) {
758                 VM_BUG_ON(!has_cache);
759                 has_cache = 0;
760         } else if (count == SWAP_MAP_SHMEM) {
761                 /*
762                  * Or we could insist on shmem.c using a special
763                  * swap_shmem_free() and free_shmem_swap_and_cache()...
764                  */
765                 count = 0;
766         } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
767                 if (count == COUNT_CONTINUED) {
768                         if (swap_count_continued(p, offset, count))
769                                 count = SWAP_MAP_MAX | COUNT_CONTINUED;
770                         else
771                                 count = SWAP_MAP_MAX;
772                 } else
773                         count--;
774         }
775
776         if (!count)
777                 mem_cgroup_uncharge_swap(entry);
778
779         usage = count | has_cache;
780         p->swap_map[offset] = usage;
781
782         /* free if no reference */
783         if (!usage) {
784                 dec_cluster_info_page(p, p->cluster_info, offset);
785                 if (offset < p->lowest_bit)
786                         p->lowest_bit = offset;
787                 if (offset > p->highest_bit)
788                         p->highest_bit = offset;
789                 set_highest_priority_index(p->type);
790                 atomic_long_inc(&nr_swap_pages);
791                 p->inuse_pages--;
792                 frontswap_invalidate_page(p->type, offset);
793                 if (p->flags & SWP_BLKDEV) {
794                         struct gendisk *disk = p->bdev->bd_disk;
795                         if (disk->fops->swap_slot_free_notify)
796                                 disk->fops->swap_slot_free_notify(p->bdev,
797                                                                   offset);
798                 }
799         }
800
801         return usage;
802 }
803
804 /*
805  * Caller has made sure that the swapdevice corresponding to entry
806  * is still around or has not been recycled.
807  */
808 void swap_free(swp_entry_t entry)
809 {
810         struct swap_info_struct *p;
811
812         p = swap_info_get(entry);
813         if (p) {
814                 swap_entry_free(p, entry, 1);
815                 spin_unlock(&p->lock);
816         }
817 }
818
819 /*
820  * Called after dropping swapcache to decrease refcnt to swap entries.
821  */
822 void swapcache_free(swp_entry_t entry, struct page *page)
823 {
824         struct swap_info_struct *p;
825         unsigned char count;
826
827         p = swap_info_get(entry);
828         if (p) {
829                 count = swap_entry_free(p, entry, SWAP_HAS_CACHE);
830                 if (page)
831                         mem_cgroup_uncharge_swapcache(page, entry, count != 0);
832                 spin_unlock(&p->lock);
833         }
834 }
835
836 /*
837  * How many references to page are currently swapped out?
838  * This does not give an exact answer when swap count is continued,
839  * but does include the high COUNT_CONTINUED flag to allow for that.
840  */
841 int page_swapcount(struct page *page)
842 {
843         int count = 0;
844         struct swap_info_struct *p;
845         swp_entry_t entry;
846
847         entry.val = page_private(page);
848         p = swap_info_get(entry);
849         if (p) {
850                 count = swap_count(p->swap_map[swp_offset(entry)]);
851                 spin_unlock(&p->lock);
852         }
853         return count;
854 }
855
856 /*
857  * We can write to an anon page without COW if there are no other references
858  * to it.  And as a side-effect, free up its swap: because the old content
859  * on disk will never be read, and seeking back there to write new content
860  * later would only waste time away from clustering.
861  */
862 int reuse_swap_page(struct page *page)
863 {
864         int count;
865
866         VM_BUG_ON(!PageLocked(page));
867         if (unlikely(PageKsm(page)))
868                 return 0;
869         count = page_mapcount(page);
870         if (count <= 1 && PageSwapCache(page)) {
871                 count += page_swapcount(page);
872                 if (count == 1 && !PageWriteback(page)) {
873                         delete_from_swap_cache(page);
874                         SetPageDirty(page);
875                 }
876         }
877         return count <= 1;
878 }
879
880 /*
881  * If swap is getting full, or if there are no more mappings of this page,
882  * then try_to_free_swap is called to free its swap space.
883  */
884 int try_to_free_swap(struct page *page)
885 {
886         VM_BUG_ON(!PageLocked(page));
887
888         if (!PageSwapCache(page))
889                 return 0;
890         if (PageWriteback(page))
891                 return 0;
892         if (page_swapcount(page))
893                 return 0;
894
895         /*
896          * Once hibernation has begun to create its image of memory,
897          * there's a danger that one of the calls to try_to_free_swap()
898          * - most probably a call from __try_to_reclaim_swap() while
899          * hibernation is allocating its own swap pages for the image,
900          * but conceivably even a call from memory reclaim - will free
901          * the swap from a page which has already been recorded in the
902          * image as a clean swapcache page, and then reuse its swap for
903          * another page of the image.  On waking from hibernation, the
904          * original page might be freed under memory pressure, then
905          * later read back in from swap, now with the wrong data.
906          *
907          * Hibration suspends storage while it is writing the image
908          * to disk so check that here.
909          */
910         if (pm_suspended_storage())
911                 return 0;
912
913         delete_from_swap_cache(page);
914         SetPageDirty(page);
915         return 1;
916 }
917
918 /*
919  * Free the swap entry like above, but also try to
920  * free the page cache entry if it is the last user.
921  */
922 int free_swap_and_cache(swp_entry_t entry)
923 {
924         struct swap_info_struct *p;
925         struct page *page = NULL;
926
927         if (non_swap_entry(entry))
928                 return 1;
929
930         p = swap_info_get(entry);
931         if (p) {
932                 if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) {
933                         page = find_get_page(swap_address_space(entry),
934                                                 entry.val);
935                         if (page && !trylock_page(page)) {
936                                 page_cache_release(page);
937                                 page = NULL;
938                         }
939                 }
940                 spin_unlock(&p->lock);
941         }
942         if (page) {
943                 /*
944                  * Not mapped elsewhere, or swap space full? Free it!
945                  * Also recheck PageSwapCache now page is locked (above).
946                  */
947                 if (PageSwapCache(page) && !PageWriteback(page) &&
948                                 (!page_mapped(page) || vm_swap_full())) {
949                         delete_from_swap_cache(page);
950                         SetPageDirty(page);
951                 }
952                 unlock_page(page);
953                 page_cache_release(page);
954         }
955         return p != NULL;
956 }
957
958 #ifdef CONFIG_HIBERNATION
959 /*
960  * Find the swap type that corresponds to given device (if any).
961  *
962  * @offset - number of the PAGE_SIZE-sized block of the device, starting
963  * from 0, in which the swap header is expected to be located.
964  *
965  * This is needed for the suspend to disk (aka swsusp).
966  */
967 int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
968 {
969         struct block_device *bdev = NULL;
970         int type;
971
972         if (device)
973                 bdev = bdget(device);
974
975         spin_lock(&swap_lock);
976         for (type = 0; type < nr_swapfiles; type++) {
977                 struct swap_info_struct *sis = swap_info[type];
978
979                 if (!(sis->flags & SWP_WRITEOK))
980                         continue;
981
982                 if (!bdev) {
983                         if (bdev_p)
984                                 *bdev_p = bdgrab(sis->bdev);
985
986                         spin_unlock(&swap_lock);
987                         return type;
988                 }
989                 if (bdev == sis->bdev) {
990                         struct swap_extent *se = &sis->first_swap_extent;
991
992                         if (se->start_block == offset) {
993                                 if (bdev_p)
994                                         *bdev_p = bdgrab(sis->bdev);
995
996                                 spin_unlock(&swap_lock);
997                                 bdput(bdev);
998                                 return type;
999                         }
1000                 }
1001         }
1002         spin_unlock(&swap_lock);
1003         if (bdev)
1004                 bdput(bdev);
1005
1006         return -ENODEV;
1007 }
1008
1009 /*
1010  * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1011  * corresponding to given index in swap_info (swap type).
1012  */
1013 sector_t swapdev_block(int type, pgoff_t offset)
1014 {
1015         struct block_device *bdev;
1016
1017         if ((unsigned int)type >= nr_swapfiles)
1018                 return 0;
1019         if (!(swap_info[type]->flags & SWP_WRITEOK))
1020                 return 0;
1021         return map_swap_entry(swp_entry(type, offset), &bdev);
1022 }
1023
1024 /*
1025  * Return either the total number of swap pages of given type, or the number
1026  * of free pages of that type (depending on @free)
1027  *
1028  * This is needed for software suspend
1029  */
1030 unsigned int count_swap_pages(int type, int free)
1031 {
1032         unsigned int n = 0;
1033
1034         spin_lock(&swap_lock);
1035         if ((unsigned int)type < nr_swapfiles) {
1036                 struct swap_info_struct *sis = swap_info[type];
1037
1038                 spin_lock(&sis->lock);
1039                 if (sis->flags & SWP_WRITEOK) {
1040                         n = sis->pages;
1041                         if (free)
1042                                 n -= sis->inuse_pages;
1043                 }
1044                 spin_unlock(&sis->lock);
1045         }
1046         spin_unlock(&swap_lock);
1047         return n;
1048 }
1049 #endif /* CONFIG_HIBERNATION */
1050
1051 static inline int maybe_same_pte(pte_t pte, pte_t swp_pte)
1052 {
1053 #ifdef CONFIG_MEM_SOFT_DIRTY
1054         /*
1055          * When pte keeps soft dirty bit the pte generated
1056          * from swap entry does not has it, still it's same
1057          * pte from logical point of view.
1058          */
1059         pte_t swp_pte_dirty = pte_swp_mksoft_dirty(swp_pte);
1060         return pte_same(pte, swp_pte) || pte_same(pte, swp_pte_dirty);
1061 #else
1062         return pte_same(pte, swp_pte);
1063 #endif
1064 }
1065
1066 /*
1067  * No need to decide whether this PTE shares the swap entry with others,
1068  * just let do_wp_page work it out if a write is requested later - to
1069  * force COW, vm_page_prot omits write permission from any private vma.
1070  */
1071 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1072                 unsigned long addr, swp_entry_t entry, struct page *page)
1073 {
1074         struct page *swapcache;
1075         struct mem_cgroup *memcg;
1076         spinlock_t *ptl;
1077         pte_t *pte;
1078         int ret = 1;
1079
1080         swapcache = page;
1081         page = ksm_might_need_to_copy(page, vma, addr);
1082         if (unlikely(!page))
1083                 return -ENOMEM;
1084
1085         if (mem_cgroup_try_charge_swapin(vma->vm_mm, page,
1086                                          GFP_KERNEL, &memcg)) {
1087                 ret = -ENOMEM;
1088                 goto out_nolock;
1089         }
1090
1091         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1092         if (unlikely(!maybe_same_pte(*pte, swp_entry_to_pte(entry)))) {
1093                 mem_cgroup_cancel_charge_swapin(memcg);
1094                 ret = 0;
1095                 goto out;
1096         }
1097
1098         dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1099         inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1100         get_page(page);
1101         set_pte_at(vma->vm_mm, addr, pte,
1102                    pte_mkold(mk_pte(page, vma->vm_page_prot)));
1103         if (page == swapcache)
1104                 page_add_anon_rmap(page, vma, addr);
1105         else /* ksm created a completely new copy */
1106                 page_add_new_anon_rmap(page, vma, addr);
1107         mem_cgroup_commit_charge_swapin(page, memcg);
1108         swap_free(entry);
1109         /*
1110          * Move the page to the active list so it is not
1111          * immediately swapped out again after swapon.
1112          */
1113         activate_page(page);
1114 out:
1115         pte_unmap_unlock(pte, ptl);
1116 out_nolock:
1117         if (page != swapcache) {
1118                 unlock_page(page);
1119                 put_page(page);
1120         }
1121         return ret;
1122 }
1123
1124 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1125                                 unsigned long addr, unsigned long end,
1126                                 swp_entry_t entry, struct page *page)
1127 {
1128         pte_t swp_pte = swp_entry_to_pte(entry);
1129         pte_t *pte;
1130         int ret = 0;
1131
1132         /*
1133          * We don't actually need pte lock while scanning for swp_pte: since
1134          * we hold page lock and mmap_sem, swp_pte cannot be inserted into the
1135          * page table while we're scanning; though it could get zapped, and on
1136          * some architectures (e.g. x86_32 with PAE) we might catch a glimpse
1137          * of unmatched parts which look like swp_pte, so unuse_pte must
1138          * recheck under pte lock.  Scanning without pte lock lets it be
1139          * preemptible whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE.
1140          */
1141         pte = pte_offset_map(pmd, addr);
1142         do {
1143                 /*
1144                  * swapoff spends a _lot_ of time in this loop!
1145                  * Test inline before going to call unuse_pte.
1146                  */
1147                 if (unlikely(maybe_same_pte(*pte, swp_pte))) {
1148                         pte_unmap(pte);
1149                         ret = unuse_pte(vma, pmd, addr, entry, page);
1150                         if (ret)
1151                                 goto out;
1152                         pte = pte_offset_map(pmd, addr);
1153                 }
1154         } while (pte++, addr += PAGE_SIZE, addr != end);
1155         pte_unmap(pte - 1);
1156 out:
1157         return ret;
1158 }
1159
1160 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1161                                 unsigned long addr, unsigned long end,
1162                                 swp_entry_t entry, struct page *page)
1163 {
1164         pmd_t *pmd;
1165         unsigned long next;
1166         int ret;
1167
1168         pmd = pmd_offset(pud, addr);
1169         do {
1170                 next = pmd_addr_end(addr, end);
1171                 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1172                         continue;
1173                 ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
1174                 if (ret)
1175                         return ret;
1176         } while (pmd++, addr = next, addr != end);
1177         return 0;
1178 }
1179
1180 static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
1181                                 unsigned long addr, unsigned long end,
1182                                 swp_entry_t entry, struct page *page)
1183 {
1184         pud_t *pud;
1185         unsigned long next;
1186         int ret;
1187
1188         pud = pud_offset(pgd, addr);
1189         do {
1190                 next = pud_addr_end(addr, end);
1191                 if (pud_none_or_clear_bad(pud))
1192                         continue;
1193                 ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
1194                 if (ret)
1195                         return ret;
1196         } while (pud++, addr = next, addr != end);
1197         return 0;
1198 }
1199
1200 static int unuse_vma(struct vm_area_struct *vma,
1201                                 swp_entry_t entry, struct page *page)
1202 {
1203         pgd_t *pgd;
1204         unsigned long addr, end, next;
1205         int ret;
1206
1207         if (page_anon_vma(page)) {
1208                 addr = page_address_in_vma(page, vma);
1209                 if (addr == -EFAULT)
1210                         return 0;
1211                 else
1212                         end = addr + PAGE_SIZE;
1213         } else {
1214                 addr = vma->vm_start;
1215                 end = vma->vm_end;
1216         }
1217
1218         pgd = pgd_offset(vma->vm_mm, addr);
1219         do {
1220                 next = pgd_addr_end(addr, end);
1221                 if (pgd_none_or_clear_bad(pgd))
1222                         continue;
1223                 ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
1224                 if (ret)
1225                         return ret;
1226         } while (pgd++, addr = next, addr != end);
1227         return 0;
1228 }
1229
1230 static int unuse_mm(struct mm_struct *mm,
1231                                 swp_entry_t entry, struct page *page)
1232 {
1233         struct vm_area_struct *vma;
1234         int ret = 0;
1235
1236         if (!down_read_trylock(&mm->mmap_sem)) {
1237                 /*
1238                  * Activate page so shrink_inactive_list is unlikely to unmap
1239                  * its ptes while lock is dropped, so swapoff can make progress.
1240                  */
1241                 activate_page(page);
1242                 unlock_page(page);
1243                 down_read(&mm->mmap_sem);
1244                 lock_page(page);
1245         }
1246         for (vma = mm->mmap; vma; vma = vma->vm_next) {
1247                 if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
1248                         break;
1249         }
1250         up_read(&mm->mmap_sem);
1251         return (ret < 0)? ret: 0;
1252 }
1253
1254 /*
1255  * Scan swap_map (or frontswap_map if frontswap parameter is true)
1256  * from current position to next entry still in use.
1257  * Recycle to start on reaching the end, returning 0 when empty.
1258  */
1259 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1260                                         unsigned int prev, bool frontswap)
1261 {
1262         unsigned int max = si->max;
1263         unsigned int i = prev;
1264         unsigned char count;
1265
1266         /*
1267          * No need for swap_lock here: we're just looking
1268          * for whether an entry is in use, not modifying it; false
1269          * hits are okay, and sys_swapoff() has already prevented new
1270          * allocations from this area (while holding swap_lock).
1271          */
1272         for (;;) {
1273                 if (++i >= max) {
1274                         if (!prev) {
1275                                 i = 0;
1276                                 break;
1277                         }
1278                         /*
1279                          * No entries in use at top of swap_map,
1280                          * loop back to start and recheck there.
1281                          */
1282                         max = prev + 1;
1283                         prev = 0;
1284                         i = 1;
1285                 }
1286                 if (frontswap) {
1287                         if (frontswap_test(si, i))
1288                                 break;
1289                         else
1290                                 continue;
1291                 }
1292                 count = ACCESS_ONCE(si->swap_map[i]);
1293                 if (count && swap_count(count) != SWAP_MAP_BAD)
1294                         break;
1295         }
1296         return i;
1297 }
1298
1299 /*
1300  * We completely avoid races by reading each swap page in advance,
1301  * and then search for the process using it.  All the necessary
1302  * page table adjustments can then be made atomically.
1303  *
1304  * if the boolean frontswap is true, only unuse pages_to_unuse pages;
1305  * pages_to_unuse==0 means all pages; ignored if frontswap is false
1306  */
1307 int try_to_unuse(unsigned int type, bool frontswap,
1308                  unsigned long pages_to_unuse)
1309 {
1310         struct swap_info_struct *si = swap_info[type];
1311         struct mm_struct *start_mm;
1312         volatile unsigned char *swap_map; /* swap_map is accessed without
1313                                            * locking. Mark it as volatile
1314                                            * to prevent compiler doing
1315                                            * something odd.
1316                                            */
1317         unsigned char swcount;
1318         struct page *page;
1319         swp_entry_t entry;
1320         unsigned int i = 0;
1321         int retval = 0;
1322
1323         /*
1324          * When searching mms for an entry, a good strategy is to
1325          * start at the first mm we freed the previous entry from
1326          * (though actually we don't notice whether we or coincidence
1327          * freed the entry).  Initialize this start_mm with a hold.
1328          *
1329          * A simpler strategy would be to start at the last mm we
1330          * freed the previous entry from; but that would take less
1331          * advantage of mmlist ordering, which clusters forked mms
1332          * together, child after parent.  If we race with dup_mmap(), we
1333          * prefer to resolve parent before child, lest we miss entries
1334          * duplicated after we scanned child: using last mm would invert
1335          * that.
1336          */
1337         start_mm = &init_mm;
1338         atomic_inc(&init_mm.mm_users);
1339
1340         /*
1341          * Keep on scanning until all entries have gone.  Usually,
1342          * one pass through swap_map is enough, but not necessarily:
1343          * there are races when an instance of an entry might be missed.
1344          */
1345         while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
1346                 if (signal_pending(current)) {
1347                         retval = -EINTR;
1348                         break;
1349                 }
1350
1351                 /*
1352                  * Get a page for the entry, using the existing swap
1353                  * cache page if there is one.  Otherwise, get a clean
1354                  * page and read the swap into it.
1355                  */
1356                 swap_map = &si->swap_map[i];
1357                 entry = swp_entry(type, i);
1358                 page = read_swap_cache_async(entry,
1359                                         GFP_HIGHUSER_MOVABLE, NULL, 0);
1360                 if (!page) {
1361                         /*
1362                          * Either swap_duplicate() failed because entry
1363                          * has been freed independently, and will not be
1364                          * reused since sys_swapoff() already disabled
1365                          * allocation from here, or alloc_page() failed.
1366                          */
1367                         swcount = *swap_map;
1368                         /*
1369                          * We don't hold lock here, so the swap entry could be
1370                          * SWAP_MAP_BAD (when the cluster is discarding).
1371                          * Instead of fail out, We can just skip the swap
1372                          * entry because swapoff will wait for discarding
1373                          * finish anyway.
1374                          */
1375                         if (!swcount || swcount == SWAP_MAP_BAD)
1376                                 continue;
1377                         retval = -ENOMEM;
1378                         break;
1379                 }
1380
1381                 /*
1382                  * Don't hold on to start_mm if it looks like exiting.
1383                  */
1384                 if (atomic_read(&start_mm->mm_users) == 1) {
1385                         mmput(start_mm);
1386                         start_mm = &init_mm;
1387                         atomic_inc(&init_mm.mm_users);
1388                 }
1389
1390                 /*
1391                  * Wait for and lock page.  When do_swap_page races with
1392                  * try_to_unuse, do_swap_page can handle the fault much
1393                  * faster than try_to_unuse can locate the entry.  This
1394                  * apparently redundant "wait_on_page_locked" lets try_to_unuse
1395                  * defer to do_swap_page in such a case - in some tests,
1396                  * do_swap_page and try_to_unuse repeatedly compete.
1397                  */
1398                 wait_on_page_locked(page);
1399                 wait_on_page_writeback(page);
1400                 lock_page(page);
1401                 wait_on_page_writeback(page);
1402
1403                 /*
1404                  * Remove all references to entry.
1405                  */
1406                 swcount = *swap_map;
1407                 if (swap_count(swcount) == SWAP_MAP_SHMEM) {
1408                         retval = shmem_unuse(entry, page);
1409                         /* page has already been unlocked and released */
1410                         if (retval < 0)
1411                                 break;
1412                         continue;
1413                 }
1414                 if (swap_count(swcount) && start_mm != &init_mm)
1415                         retval = unuse_mm(start_mm, entry, page);
1416
1417                 if (swap_count(*swap_map)) {
1418                         int set_start_mm = (*swap_map >= swcount);
1419                         struct list_head *p = &start_mm->mmlist;
1420                         struct mm_struct *new_start_mm = start_mm;
1421                         struct mm_struct *prev_mm = start_mm;
1422                         struct mm_struct *mm;
1423
1424                         atomic_inc(&new_start_mm->mm_users);
1425                         atomic_inc(&prev_mm->mm_users);
1426                         spin_lock(&mmlist_lock);
1427                         while (swap_count(*swap_map) && !retval &&
1428                                         (p = p->next) != &start_mm->mmlist) {
1429                                 mm = list_entry(p, struct mm_struct, mmlist);
1430                                 if (!atomic_inc_not_zero(&mm->mm_users))
1431                                         continue;
1432                                 spin_unlock(&mmlist_lock);
1433                                 mmput(prev_mm);
1434                                 prev_mm = mm;
1435
1436                                 cond_resched();
1437
1438                                 swcount = *swap_map;
1439                                 if (!swap_count(swcount)) /* any usage ? */
1440                                         ;
1441                                 else if (mm == &init_mm)
1442                                         set_start_mm = 1;
1443                                 else
1444                                         retval = unuse_mm(mm, entry, page);
1445
1446                                 if (set_start_mm && *swap_map < swcount) {
1447                                         mmput(new_start_mm);
1448                                         atomic_inc(&mm->mm_users);
1449                                         new_start_mm = mm;
1450                                         set_start_mm = 0;
1451                                 }
1452                                 spin_lock(&mmlist_lock);
1453                         }
1454                         spin_unlock(&mmlist_lock);
1455                         mmput(prev_mm);
1456                         mmput(start_mm);
1457                         start_mm = new_start_mm;
1458                 }
1459                 if (retval) {
1460                         unlock_page(page);
1461                         page_cache_release(page);
1462                         break;
1463                 }
1464
1465                 /*
1466                  * If a reference remains (rare), we would like to leave
1467                  * the page in the swap cache; but try_to_unmap could
1468                  * then re-duplicate the entry once we drop page lock,
1469                  * so we might loop indefinitely; also, that page could
1470                  * not be swapped out to other storage meanwhile.  So:
1471                  * delete from cache even if there's another reference,
1472                  * after ensuring that the data has been saved to disk -
1473                  * since if the reference remains (rarer), it will be
1474                  * read from disk into another page.  Splitting into two
1475                  * pages would be incorrect if swap supported "shared
1476                  * private" pages, but they are handled by tmpfs files.
1477                  *
1478                  * Given how unuse_vma() targets one particular offset
1479                  * in an anon_vma, once the anon_vma has been determined,
1480                  * this splitting happens to be just what is needed to
1481                  * handle where KSM pages have been swapped out: re-reading
1482                  * is unnecessarily slow, but we can fix that later on.
1483                  */
1484                 if (swap_count(*swap_map) &&
1485                      PageDirty(page) && PageSwapCache(page)) {
1486                         struct writeback_control wbc = {
1487                                 .sync_mode = WB_SYNC_NONE,
1488                         };
1489
1490                         swap_writepage(page, &wbc);
1491                         lock_page(page);
1492                         wait_on_page_writeback(page);
1493                 }
1494
1495                 /*
1496                  * It is conceivable that a racing task removed this page from
1497                  * swap cache just before we acquired the page lock at the top,
1498                  * or while we dropped it in unuse_mm().  The page might even
1499                  * be back in swap cache on another swap area: that we must not
1500                  * delete, since it may not have been written out to swap yet.
1501                  */
1502                 if (PageSwapCache(page) &&
1503                     likely(page_private(page) == entry.val))
1504                         delete_from_swap_cache(page);
1505
1506                 /*
1507                  * So we could skip searching mms once swap count went
1508                  * to 1, we did not mark any present ptes as dirty: must
1509                  * mark page dirty so shrink_page_list will preserve it.
1510                  */
1511                 SetPageDirty(page);
1512                 unlock_page(page);
1513                 page_cache_release(page);
1514
1515                 /*
1516                  * Make sure that we aren't completely killing
1517                  * interactive performance.
1518                  */
1519                 cond_resched();
1520                 if (frontswap && pages_to_unuse > 0) {
1521                         if (!--pages_to_unuse)
1522                                 break;
1523                 }
1524         }
1525
1526         mmput(start_mm);
1527         return retval;
1528 }
1529
1530 /*
1531  * After a successful try_to_unuse, if no swap is now in use, we know
1532  * we can empty the mmlist.  swap_lock must be held on entry and exit.
1533  * Note that mmlist_lock nests inside swap_lock, and an mm must be
1534  * added to the mmlist just after page_duplicate - before would be racy.
1535  */
1536 static void drain_mmlist(void)
1537 {
1538         struct list_head *p, *next;
1539         unsigned int type;
1540
1541         for (type = 0; type < nr_swapfiles; type++)
1542                 if (swap_info[type]->inuse_pages)
1543                         return;
1544         spin_lock(&mmlist_lock);
1545         list_for_each_safe(p, next, &init_mm.mmlist)
1546                 list_del_init(p);
1547         spin_unlock(&mmlist_lock);
1548 }
1549
1550 /*
1551  * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
1552  * corresponds to page offset for the specified swap entry.
1553  * Note that the type of this function is sector_t, but it returns page offset
1554  * into the bdev, not sector offset.
1555  */
1556 static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
1557 {
1558         struct swap_info_struct *sis;
1559         struct swap_extent *start_se;
1560         struct swap_extent *se;
1561         pgoff_t offset;
1562
1563         sis = swap_info[swp_type(entry)];
1564         *bdev = sis->bdev;
1565
1566         offset = swp_offset(entry);
1567         start_se = sis->curr_swap_extent;
1568         se = start_se;
1569
1570         for ( ; ; ) {
1571                 struct list_head *lh;
1572
1573                 if (se->start_page <= offset &&
1574                                 offset < (se->start_page + se->nr_pages)) {
1575                         return se->start_block + (offset - se->start_page);
1576                 }
1577                 lh = se->list.next;
1578                 se = list_entry(lh, struct swap_extent, list);
1579                 sis->curr_swap_extent = se;
1580                 BUG_ON(se == start_se);         /* It *must* be present */
1581         }
1582 }
1583
1584 /*
1585  * Returns the page offset into bdev for the specified page's swap entry.
1586  */
1587 sector_t map_swap_page(struct page *page, struct block_device **bdev)
1588 {
1589         swp_entry_t entry;
1590         entry.val = page_private(page);
1591         return map_swap_entry(entry, bdev);
1592 }
1593
1594 /*
1595  * Free all of a swapdev's extent information
1596  */
1597 static void destroy_swap_extents(struct swap_info_struct *sis)
1598 {
1599         while (!list_empty(&sis->first_swap_extent.list)) {
1600                 struct swap_extent *se;
1601
1602                 se = list_entry(sis->first_swap_extent.list.next,
1603                                 struct swap_extent, list);
1604                 list_del(&se->list);
1605                 kfree(se);
1606         }
1607
1608         if (sis->flags & SWP_FILE) {
1609                 struct file *swap_file = sis->swap_file;
1610                 struct address_space *mapping = swap_file->f_mapping;
1611
1612                 sis->flags &= ~SWP_FILE;
1613                 mapping->a_ops->swap_deactivate(swap_file);
1614         }
1615 }
1616
1617 /*
1618  * Add a block range (and the corresponding page range) into this swapdev's
1619  * extent list.  The extent list is kept sorted in page order.
1620  *
1621  * This function rather assumes that it is called in ascending page order.
1622  */
1623 int
1624 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
1625                 unsigned long nr_pages, sector_t start_block)
1626 {
1627         struct swap_extent *se;
1628         struct swap_extent *new_se;
1629         struct list_head *lh;
1630
1631         if (start_page == 0) {
1632                 se = &sis->first_swap_extent;
1633                 sis->curr_swap_extent = se;
1634                 se->start_page = 0;
1635                 se->nr_pages = nr_pages;
1636                 se->start_block = start_block;
1637                 return 1;
1638         } else {
1639                 lh = sis->first_swap_extent.list.prev;  /* Highest extent */
1640                 se = list_entry(lh, struct swap_extent, list);
1641                 BUG_ON(se->start_page + se->nr_pages != start_page);
1642                 if (se->start_block + se->nr_pages == start_block) {
1643                         /* Merge it */
1644                         se->nr_pages += nr_pages;
1645                         return 0;
1646                 }
1647         }
1648
1649         /*
1650          * No merge.  Insert a new extent, preserving ordering.
1651          */
1652         new_se = kmalloc(sizeof(*se), GFP_KERNEL);
1653         if (new_se == NULL)
1654                 return -ENOMEM;
1655         new_se->start_page = start_page;
1656         new_se->nr_pages = nr_pages;
1657         new_se->start_block = start_block;
1658
1659         list_add_tail(&new_se->list, &sis->first_swap_extent.list);
1660         return 1;
1661 }
1662
1663 /*
1664  * A `swap extent' is a simple thing which maps a contiguous range of pages
1665  * onto a contiguous range of disk blocks.  An ordered list of swap extents
1666  * is built at swapon time and is then used at swap_writepage/swap_readpage
1667  * time for locating where on disk a page belongs.
1668  *
1669  * If the swapfile is an S_ISBLK block device, a single extent is installed.
1670  * This is done so that the main operating code can treat S_ISBLK and S_ISREG
1671  * swap files identically.
1672  *
1673  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
1674  * extent list operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
1675  * swapfiles are handled *identically* after swapon time.
1676  *
1677  * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
1678  * and will parse them into an ordered extent list, in PAGE_SIZE chunks.  If
1679  * some stray blocks are found which do not fall within the PAGE_SIZE alignment
1680  * requirements, they are simply tossed out - we will never use those blocks
1681  * for swapping.
1682  *
1683  * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon.  This
1684  * prevents root from shooting her foot off by ftruncating an in-use swapfile,
1685  * which will scribble on the fs.
1686  *
1687  * The amount of disk space which a single swap extent represents varies.
1688  * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
1689  * extents in the list.  To avoid much list walking, we cache the previous
1690  * search location in `curr_swap_extent', and start new searches from there.
1691  * This is extremely effective.  The average number of iterations in
1692  * map_swap_page() has been measured at about 0.3 per page.  - akpm.
1693  */
1694 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
1695 {
1696         struct file *swap_file = sis->swap_file;
1697         struct address_space *mapping = swap_file->f_mapping;
1698         struct inode *inode = mapping->host;
1699         int ret;
1700
1701         if (S_ISBLK(inode->i_mode)) {
1702                 ret = add_swap_extent(sis, 0, sis->max, 0);
1703                 *span = sis->pages;
1704                 return ret;
1705         }
1706
1707         if (mapping->a_ops->swap_activate) {
1708                 ret = mapping->a_ops->swap_activate(sis, swap_file, span);
1709                 if (!ret) {
1710                         sis->flags |= SWP_FILE;
1711                         ret = add_swap_extent(sis, 0, sis->max, 0);
1712                         *span = sis->pages;
1713                 }
1714                 return ret;
1715         }
1716
1717         return generic_swapfile_activate(sis, swap_file, span);
1718 }
1719
1720 static void _enable_swap_info(struct swap_info_struct *p, int prio,
1721                                 unsigned char *swap_map,
1722                                 struct swap_cluster_info *cluster_info)
1723 {
1724         int i, prev;
1725
1726         if (prio >= 0)
1727                 p->prio = prio;
1728         else
1729                 p->prio = --least_priority;
1730         p->swap_map = swap_map;
1731         p->cluster_info = cluster_info;
1732         p->flags |= SWP_WRITEOK;
1733         atomic_long_add(p->pages, &nr_swap_pages);
1734         total_swap_pages += p->pages;
1735
1736         /* insert swap space into swap_list: */
1737         prev = -1;
1738         for (i = swap_list.head; i >= 0; i = swap_info[i]->next) {
1739                 if (p->prio >= swap_info[i]->prio)
1740                         break;
1741                 prev = i;
1742         }
1743         p->next = i;
1744         if (prev < 0)
1745                 swap_list.head = swap_list.next = p->type;
1746         else
1747                 swap_info[prev]->next = p->type;
1748 }
1749
1750 static void enable_swap_info(struct swap_info_struct *p, int prio,
1751                                 unsigned char *swap_map,
1752                                 struct swap_cluster_info *cluster_info,
1753                                 unsigned long *frontswap_map)
1754 {
1755         frontswap_init(p->type, frontswap_map);
1756         spin_lock(&swap_lock);
1757         spin_lock(&p->lock);
1758          _enable_swap_info(p, prio, swap_map, cluster_info);
1759         spin_unlock(&p->lock);
1760         spin_unlock(&swap_lock);
1761 }
1762
1763 static void reinsert_swap_info(struct swap_info_struct *p)
1764 {
1765         spin_lock(&swap_lock);
1766         spin_lock(&p->lock);
1767         _enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
1768         spin_unlock(&p->lock);
1769         spin_unlock(&swap_lock);
1770 }
1771
1772 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
1773 {
1774         struct swap_info_struct *p = NULL;
1775         unsigned char *swap_map;
1776         struct swap_cluster_info *cluster_info;
1777         unsigned long *frontswap_map;
1778         struct file *swap_file, *victim;
1779         struct address_space *mapping;
1780         struct inode *inode;
1781         struct filename *pathname;
1782         int i, type, prev;
1783         int err;
1784
1785         if (!capable(CAP_SYS_ADMIN))
1786                 return -EPERM;
1787
1788         BUG_ON(!current->mm);
1789
1790         pathname = getname(specialfile);
1791         if (IS_ERR(pathname))
1792                 return PTR_ERR(pathname);
1793
1794         victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
1795         err = PTR_ERR(victim);
1796         if (IS_ERR(victim))
1797                 goto out;
1798
1799         mapping = victim->f_mapping;
1800         prev = -1;
1801         spin_lock(&swap_lock);
1802         for (type = swap_list.head; type >= 0; type = swap_info[type]->next) {
1803                 p = swap_info[type];
1804                 if (p->flags & SWP_WRITEOK) {
1805                         if (p->swap_file->f_mapping == mapping)
1806                                 break;
1807                 }
1808                 prev = type;
1809         }
1810         if (type < 0) {
1811                 err = -EINVAL;
1812                 spin_unlock(&swap_lock);
1813                 goto out_dput;
1814         }
1815         if (!security_vm_enough_memory_mm(current->mm, p->pages))
1816                 vm_unacct_memory(p->pages);
1817         else {
1818                 err = -ENOMEM;
1819                 spin_unlock(&swap_lock);
1820                 goto out_dput;
1821         }
1822         if (prev < 0)
1823                 swap_list.head = p->next;
1824         else
1825                 swap_info[prev]->next = p->next;
1826         if (type == swap_list.next) {
1827                 /* just pick something that's safe... */
1828                 swap_list.next = swap_list.head;
1829         }
1830         spin_lock(&p->lock);
1831         if (p->prio < 0) {
1832                 for (i = p->next; i >= 0; i = swap_info[i]->next)
1833                         swap_info[i]->prio = p->prio--;
1834                 least_priority++;
1835         }
1836         atomic_long_sub(p->pages, &nr_swap_pages);
1837         total_swap_pages -= p->pages;
1838         p->flags &= ~SWP_WRITEOK;
1839         spin_unlock(&p->lock);
1840         spin_unlock(&swap_lock);
1841
1842         set_current_oom_origin();
1843         err = try_to_unuse(type, false, 0); /* force all pages to be unused */
1844         clear_current_oom_origin();
1845
1846         if (err) {
1847                 /* re-insert swap space back into swap_list */
1848                 reinsert_swap_info(p);
1849                 goto out_dput;
1850         }
1851
1852         flush_work(&p->discard_work);
1853
1854         destroy_swap_extents(p);
1855         if (p->flags & SWP_CONTINUED)
1856                 free_swap_count_continuations(p);
1857
1858         mutex_lock(&swapon_mutex);
1859         spin_lock(&swap_lock);
1860         spin_lock(&p->lock);
1861         drain_mmlist();
1862
1863         /* wait for anyone still in scan_swap_map */
1864         p->highest_bit = 0;             /* cuts scans short */
1865         while (p->flags >= SWP_SCANNING) {
1866                 spin_unlock(&p->lock);
1867                 spin_unlock(&swap_lock);
1868                 schedule_timeout_uninterruptible(1);
1869                 spin_lock(&swap_lock);
1870                 spin_lock(&p->lock);
1871         }
1872
1873         swap_file = p->swap_file;
1874         p->swap_file = NULL;
1875         p->max = 0;
1876         swap_map = p->swap_map;
1877         p->swap_map = NULL;
1878         cluster_info = p->cluster_info;
1879         p->cluster_info = NULL;
1880         p->flags = 0;
1881         frontswap_map = frontswap_map_get(p);
1882         frontswap_map_set(p, NULL);
1883         spin_unlock(&p->lock);
1884         spin_unlock(&swap_lock);
1885         frontswap_invalidate_area(type);
1886         mutex_unlock(&swapon_mutex);
1887         vfree(swap_map);
1888         vfree(cluster_info);
1889         vfree(frontswap_map);
1890         /* Destroy swap account informatin */
1891         swap_cgroup_swapoff(type);
1892
1893         inode = mapping->host;
1894         if (S_ISBLK(inode->i_mode)) {
1895                 struct block_device *bdev = I_BDEV(inode);
1896                 set_blocksize(bdev, p->old_block_size);
1897                 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
1898         } else {
1899                 mutex_lock(&inode->i_mutex);
1900                 inode->i_flags &= ~S_SWAPFILE;
1901                 mutex_unlock(&inode->i_mutex);
1902         }
1903         filp_close(swap_file, NULL);
1904         err = 0;
1905         atomic_inc(&proc_poll_event);
1906         wake_up_interruptible(&proc_poll_wait);
1907
1908 out_dput:
1909         filp_close(victim, NULL);
1910 out:
1911         putname(pathname);
1912         return err;
1913 }
1914
1915 #ifdef CONFIG_PROC_FS
1916 static unsigned swaps_poll(struct file *file, poll_table *wait)
1917 {
1918         struct seq_file *seq = file->private_data;
1919
1920         poll_wait(file, &proc_poll_wait, wait);
1921
1922         if (seq->poll_event != atomic_read(&proc_poll_event)) {
1923                 seq->poll_event = atomic_read(&proc_poll_event);
1924                 return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
1925         }
1926
1927         return POLLIN | POLLRDNORM;
1928 }
1929
1930 /* iterator */
1931 static void *swap_start(struct seq_file *swap, loff_t *pos)
1932 {
1933         struct swap_info_struct *si;
1934         int type;
1935         loff_t l = *pos;
1936
1937         mutex_lock(&swapon_mutex);
1938
1939         if (!l)
1940                 return SEQ_START_TOKEN;
1941
1942         for (type = 0; type < nr_swapfiles; type++) {
1943                 smp_rmb();      /* read nr_swapfiles before swap_info[type] */
1944                 si = swap_info[type];
1945                 if (!(si->flags & SWP_USED) || !si->swap_map)
1946                         continue;
1947                 if (!--l)
1948                         return si;
1949         }
1950
1951         return NULL;
1952 }
1953
1954 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
1955 {
1956         struct swap_info_struct *si = v;
1957         int type;
1958
1959         if (v == SEQ_START_TOKEN)
1960                 type = 0;
1961         else
1962                 type = si->type + 1;
1963
1964         for (; type < nr_swapfiles; type++) {
1965                 smp_rmb();      /* read nr_swapfiles before swap_info[type] */
1966                 si = swap_info[type];
1967                 if (!(si->flags & SWP_USED) || !si->swap_map)
1968                         continue;
1969                 ++*pos;
1970                 return si;
1971         }
1972
1973         return NULL;
1974 }
1975
1976 static void swap_stop(struct seq_file *swap, void *v)
1977 {
1978         mutex_unlock(&swapon_mutex);
1979 }
1980
1981 static int swap_show(struct seq_file *swap, void *v)
1982 {
1983         struct swap_info_struct *si = v;
1984         struct file *file;
1985         int len;
1986
1987         if (si == SEQ_START_TOKEN) {
1988                 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
1989                 return 0;
1990         }
1991
1992         file = si->swap_file;
1993         len = seq_path(swap, &file->f_path, " \t\n\\");
1994         seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
1995                         len < 40 ? 40 - len : 1, " ",
1996                         S_ISBLK(file_inode(file)->i_mode) ?
1997                                 "partition" : "file\t",
1998                         si->pages << (PAGE_SHIFT - 10),
1999                         si->inuse_pages << (PAGE_SHIFT - 10),
2000                         si->prio);
2001         return 0;
2002 }
2003
2004 static const struct seq_operations swaps_op = {
2005         .start =        swap_start,
2006         .next =         swap_next,
2007         .stop =         swap_stop,
2008         .show =         swap_show
2009 };
2010
2011 static int swaps_open(struct inode *inode, struct file *file)
2012 {
2013         struct seq_file *seq;
2014         int ret;
2015
2016         ret = seq_open(file, &swaps_op);
2017         if (ret)
2018                 return ret;
2019
2020         seq = file->private_data;
2021         seq->poll_event = atomic_read(&proc_poll_event);
2022         return 0;
2023 }
2024
2025 static const struct file_operations proc_swaps_operations = {
2026         .open           = swaps_open,
2027         .read           = seq_read,
2028         .llseek         = seq_lseek,
2029         .release        = seq_release,
2030         .poll           = swaps_poll,
2031 };
2032
2033 static int __init procswaps_init(void)
2034 {
2035         proc_create("swaps", 0, NULL, &proc_swaps_operations);
2036         return 0;
2037 }
2038 __initcall(procswaps_init);
2039 #endif /* CONFIG_PROC_FS */
2040
2041 #ifdef MAX_SWAPFILES_CHECK
2042 static int __init max_swapfiles_check(void)
2043 {
2044         MAX_SWAPFILES_CHECK();
2045         return 0;
2046 }
2047 late_initcall(max_swapfiles_check);
2048 #endif
2049
2050 static struct swap_info_struct *alloc_swap_info(void)
2051 {
2052         struct swap_info_struct *p;
2053         unsigned int type;
2054
2055         p = kzalloc(sizeof(*p), GFP_KERNEL);
2056         if (!p)
2057                 return ERR_PTR(-ENOMEM);
2058
2059         spin_lock(&swap_lock);
2060         for (type = 0; type < nr_swapfiles; type++) {
2061                 if (!(swap_info[type]->flags & SWP_USED))
2062                         break;
2063         }
2064         if (type >= MAX_SWAPFILES) {
2065                 spin_unlock(&swap_lock);
2066                 kfree(p);
2067                 return ERR_PTR(-EPERM);
2068         }
2069         if (type >= nr_swapfiles) {
2070                 p->type = type;
2071                 swap_info[type] = p;
2072                 /*
2073                  * Write swap_info[type] before nr_swapfiles, in case a
2074                  * racing procfs swap_start() or swap_next() is reading them.
2075                  * (We never shrink nr_swapfiles, we never free this entry.)
2076                  */
2077                 smp_wmb();
2078                 nr_swapfiles++;
2079         } else {
2080                 kfree(p);
2081                 p = swap_info[type];
2082                 /*
2083                  * Do not memset this entry: a racing procfs swap_next()
2084                  * would be relying on p->type to remain valid.
2085                  */
2086         }
2087         INIT_LIST_HEAD(&p->first_swap_extent.list);
2088         p->flags = SWP_USED;
2089         p->next = -1;
2090         spin_unlock(&swap_lock);
2091         spin_lock_init(&p->lock);
2092
2093         return p;
2094 }
2095
2096 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2097 {
2098         int error;
2099
2100         if (S_ISBLK(inode->i_mode)) {
2101                 p->bdev = bdgrab(I_BDEV(inode));
2102                 error = blkdev_get(p->bdev,
2103                                    FMODE_READ | FMODE_WRITE | FMODE_EXCL,
2104                                    sys_swapon);
2105                 if (error < 0) {
2106                         p->bdev = NULL;
2107                         return -EINVAL;
2108                 }
2109                 p->old_block_size = block_size(p->bdev);
2110                 error = set_blocksize(p->bdev, PAGE_SIZE);
2111                 if (error < 0)
2112                         return error;
2113                 p->flags |= SWP_BLKDEV;
2114         } else if (S_ISREG(inode->i_mode)) {
2115                 p->bdev = inode->i_sb->s_bdev;
2116                 mutex_lock(&inode->i_mutex);
2117                 if (IS_SWAPFILE(inode))
2118                         return -EBUSY;
2119         } else
2120                 return -EINVAL;
2121
2122         return 0;
2123 }
2124
2125 static unsigned long read_swap_header(struct swap_info_struct *p,
2126                                         union swap_header *swap_header,
2127                                         struct inode *inode)
2128 {
2129         int i;
2130         unsigned long maxpages;
2131         unsigned long swapfilepages;
2132         unsigned long last_page;
2133
2134         if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2135                 pr_err("Unable to find swap-space signature\n");
2136                 return 0;
2137         }
2138
2139         /* swap partition endianess hack... */
2140         if (swab32(swap_header->info.version) == 1) {
2141                 swab32s(&swap_header->info.version);
2142                 swab32s(&swap_header->info.last_page);
2143                 swab32s(&swap_header->info.nr_badpages);
2144                 for (i = 0; i < swap_header->info.nr_badpages; i++)
2145                         swab32s(&swap_header->info.badpages[i]);
2146         }
2147         /* Check the swap header's sub-version */
2148         if (swap_header->info.version != 1) {
2149                 pr_warn("Unable to handle swap header version %d\n",
2150                         swap_header->info.version);
2151                 return 0;
2152         }
2153
2154         p->lowest_bit  = 1;
2155         p->cluster_next = 1;
2156         p->cluster_nr = 0;
2157
2158         /*
2159          * Find out how many pages are allowed for a single swap
2160          * device. There are two limiting factors: 1) the number
2161          * of bits for the swap offset in the swp_entry_t type, and
2162          * 2) the number of bits in the swap pte as defined by the
2163          * different architectures. In order to find the
2164          * largest possible bit mask, a swap entry with swap type 0
2165          * and swap offset ~0UL is created, encoded to a swap pte,
2166          * decoded to a swp_entry_t again, and finally the swap
2167          * offset is extracted. This will mask all the bits from
2168          * the initial ~0UL mask that can't be encoded in either
2169          * the swp_entry_t or the architecture definition of a
2170          * swap pte.
2171          */
2172         maxpages = swp_offset(pte_to_swp_entry(
2173                         swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2174         last_page = swap_header->info.last_page;
2175         if (last_page > maxpages) {
2176                 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2177                         maxpages << (PAGE_SHIFT - 10),
2178                         last_page << (PAGE_SHIFT - 10));
2179         }
2180         if (maxpages > last_page) {
2181                 maxpages = last_page + 1;
2182                 /* p->max is an unsigned int: don't overflow it */
2183                 if ((unsigned int)maxpages == 0)
2184                         maxpages = UINT_MAX;
2185         }
2186         p->highest_bit = maxpages - 1;
2187
2188         if (!maxpages)
2189                 return 0;
2190         swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2191         if (swapfilepages && maxpages > swapfilepages) {
2192                 pr_warn("Swap area shorter than signature indicates\n");
2193                 return 0;
2194         }
2195         if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2196                 return 0;
2197         if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2198                 return 0;
2199
2200         return maxpages;
2201 }
2202
2203 static int setup_swap_map_and_extents(struct swap_info_struct *p,
2204                                         union swap_header *swap_header,
2205                                         unsigned char *swap_map,
2206                                         struct swap_cluster_info *cluster_info,
2207                                         unsigned long maxpages,
2208                                         sector_t *span)
2209 {
2210         int i;
2211         unsigned int nr_good_pages;
2212         int nr_extents;
2213         unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2214         unsigned long idx = p->cluster_next / SWAPFILE_CLUSTER;
2215
2216         nr_good_pages = maxpages - 1;   /* omit header page */
2217
2218         cluster_set_null(&p->free_cluster_head);
2219         cluster_set_null(&p->free_cluster_tail);
2220         cluster_set_null(&p->discard_cluster_head);
2221         cluster_set_null(&p->discard_cluster_tail);
2222
2223         for (i = 0; i < swap_header->info.nr_badpages; i++) {
2224                 unsigned int page_nr = swap_header->info.badpages[i];
2225                 if (page_nr == 0 || page_nr > swap_header->info.last_page)
2226                         return -EINVAL;
2227                 if (page_nr < maxpages) {
2228                         swap_map[page_nr] = SWAP_MAP_BAD;
2229                         nr_good_pages--;
2230                         /*
2231                          * Haven't marked the cluster free yet, no list
2232                          * operation involved
2233                          */
2234                         inc_cluster_info_page(p, cluster_info, page_nr);
2235                 }
2236         }
2237
2238         /* Haven't marked the cluster free yet, no list operation involved */
2239         for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2240                 inc_cluster_info_page(p, cluster_info, i);
2241
2242         if (nr_good_pages) {
2243                 swap_map[0] = SWAP_MAP_BAD;
2244                 /*
2245                  * Not mark the cluster free yet, no list
2246                  * operation involved
2247                  */
2248                 inc_cluster_info_page(p, cluster_info, 0);
2249                 p->max = maxpages;
2250                 p->pages = nr_good_pages;
2251                 nr_extents = setup_swap_extents(p, span);
2252                 if (nr_extents < 0)
2253                         return nr_extents;
2254                 nr_good_pages = p->pages;
2255         }
2256         if (!nr_good_pages) {
2257                 pr_warn("Empty swap-file\n");
2258                 return -EINVAL;
2259         }
2260
2261         if (!cluster_info)
2262                 return nr_extents;
2263
2264         for (i = 0; i < nr_clusters; i++) {
2265                 if (!cluster_count(&cluster_info[idx])) {
2266                         cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2267                         if (cluster_is_null(&p->free_cluster_head)) {
2268                                 cluster_set_next_flag(&p->free_cluster_head,
2269                                                                 idx, 0);
2270                                 cluster_set_next_flag(&p->free_cluster_tail,
2271                                                                 idx, 0);
2272                         } else {
2273                                 unsigned int tail;
2274
2275                                 tail = cluster_next(&p->free_cluster_tail);
2276                                 cluster_set_next(&cluster_info[tail], idx);
2277                                 cluster_set_next_flag(&p->free_cluster_tail,
2278                                                                 idx, 0);
2279                         }
2280                 }
2281                 idx++;
2282                 if (idx == nr_clusters)
2283                         idx = 0;
2284         }
2285         return nr_extents;
2286 }
2287
2288 /*
2289  * Helper to sys_swapon determining if a given swap
2290  * backing device queue supports DISCARD operations.
2291  */
2292 static bool swap_discardable(struct swap_info_struct *si)
2293 {
2294         struct request_queue *q = bdev_get_queue(si->bdev);
2295
2296         if (!q || !blk_queue_discard(q))
2297                 return false;
2298
2299         return true;
2300 }
2301
2302 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2303 {
2304         struct swap_info_struct *p;
2305         struct filename *name;
2306         struct file *swap_file = NULL;
2307         struct address_space *mapping;
2308         int i;
2309         int prio;
2310         int error;
2311         union swap_header *swap_header;
2312         int nr_extents;
2313         sector_t span;
2314         unsigned long maxpages;
2315         unsigned char *swap_map = NULL;
2316         struct swap_cluster_info *cluster_info = NULL;
2317         unsigned long *frontswap_map = NULL;
2318         struct page *page = NULL;
2319         struct inode *inode = NULL;
2320
2321         if (swap_flags & ~SWAP_FLAGS_VALID)
2322                 return -EINVAL;
2323
2324         if (!capable(CAP_SYS_ADMIN))
2325                 return -EPERM;
2326
2327         p = alloc_swap_info();
2328         if (IS_ERR(p))
2329                 return PTR_ERR(p);
2330
2331         INIT_WORK(&p->discard_work, swap_discard_work);
2332
2333         name = getname(specialfile);
2334         if (IS_ERR(name)) {
2335                 error = PTR_ERR(name);
2336                 name = NULL;
2337                 goto bad_swap;
2338         }
2339         swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
2340         if (IS_ERR(swap_file)) {
2341                 error = PTR_ERR(swap_file);
2342                 swap_file = NULL;
2343                 goto bad_swap;
2344         }
2345
2346         p->swap_file = swap_file;
2347         mapping = swap_file->f_mapping;
2348
2349         for (i = 0; i < nr_swapfiles; i++) {
2350                 struct swap_info_struct *q = swap_info[i];
2351
2352                 if (q == p || !q->swap_file)
2353                         continue;
2354                 if (mapping == q->swap_file->f_mapping) {
2355                         error = -EBUSY;
2356                         goto bad_swap;
2357                 }
2358         }
2359
2360         inode = mapping->host;
2361         /* If S_ISREG(inode->i_mode) will do mutex_lock(&inode->i_mutex); */
2362         error = claim_swapfile(p, inode);
2363         if (unlikely(error))
2364                 goto bad_swap;
2365
2366         /*
2367          * Read the swap header.
2368          */
2369         if (!mapping->a_ops->readpage) {
2370                 error = -EINVAL;
2371                 goto bad_swap;
2372         }
2373         page = read_mapping_page(mapping, 0, swap_file);
2374         if (IS_ERR(page)) {
2375                 error = PTR_ERR(page);
2376                 goto bad_swap;
2377         }
2378         swap_header = kmap(page);
2379
2380         maxpages = read_swap_header(p, swap_header, inode);
2381         if (unlikely(!maxpages)) {
2382                 error = -EINVAL;
2383                 goto bad_swap;
2384         }
2385
2386         /* OK, set up the swap map and apply the bad block list */
2387         swap_map = vzalloc(maxpages);
2388         if (!swap_map) {
2389                 error = -ENOMEM;
2390                 goto bad_swap;
2391         }
2392         if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
2393                 p->flags |= SWP_SOLIDSTATE;
2394                 /*
2395                  * select a random position to start with to help wear leveling
2396                  * SSD
2397                  */
2398                 p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
2399
2400                 cluster_info = vzalloc(DIV_ROUND_UP(maxpages,
2401                         SWAPFILE_CLUSTER) * sizeof(*cluster_info));
2402                 if (!cluster_info) {
2403                         error = -ENOMEM;
2404                         goto bad_swap;
2405                 }
2406         }
2407
2408         error = swap_cgroup_swapon(p->type, maxpages);
2409         if (error)
2410                 goto bad_swap;
2411
2412         nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
2413                 cluster_info, maxpages, &span);
2414         if (unlikely(nr_extents < 0)) {
2415                 error = nr_extents;
2416                 goto bad_swap;
2417         }
2418         /* frontswap enabled? set up bit-per-page map for frontswap */
2419         if (frontswap_enabled)
2420                 frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));
2421
2422         if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
2423                 /*
2424                  * When discard is enabled for swap with no particular
2425                  * policy flagged, we set all swap discard flags here in
2426                  * order to sustain backward compatibility with older
2427                  * swapon(8) releases.
2428                  */
2429                 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
2430                              SWP_PAGE_DISCARD);
2431
2432                 /*
2433                  * By flagging sys_swapon, a sysadmin can tell us to
2434                  * either do single-time area discards only, or to just
2435                  * perform discards for released swap page-clusters.
2436                  * Now it's time to adjust the p->flags accordingly.
2437                  */
2438                 if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
2439                         p->flags &= ~SWP_PAGE_DISCARD;
2440                 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
2441                         p->flags &= ~SWP_AREA_DISCARD;
2442
2443                 /* issue a swapon-time discard if it's still required */
2444                 if (p->flags & SWP_AREA_DISCARD) {
2445                         int err = discard_swap(p);
2446                         if (unlikely(err))
2447                                 pr_err("swapon: discard_swap(%p): %d\n",
2448                                         p, err);
2449                 }
2450         }
2451
2452         mutex_lock(&swapon_mutex);
2453         prio = -1;
2454         if (swap_flags & SWAP_FLAG_PREFER)
2455                 prio =
2456                   (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
2457         enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
2458
2459         pr_info("Adding %uk swap on %s.  "
2460                         "Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2461                 p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
2462                 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
2463                 (p->flags & SWP_SOLIDSTATE) ? "SS" : "",
2464                 (p->flags & SWP_DISCARDABLE) ? "D" : "",
2465                 (p->flags & SWP_AREA_DISCARD) ? "s" : "",
2466                 (p->flags & SWP_PAGE_DISCARD) ? "c" : "",
2467                 (frontswap_map) ? "FS" : "");
2468
2469         mutex_unlock(&swapon_mutex);
2470         atomic_inc(&proc_poll_event);
2471         wake_up_interruptible(&proc_poll_wait);
2472
2473         if (S_ISREG(inode->i_mode))
2474                 inode->i_flags |= S_SWAPFILE;
2475         error = 0;
2476         goto out;
2477 bad_swap:
2478         if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
2479                 set_blocksize(p->bdev, p->old_block_size);
2480                 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2481         }
2482         destroy_swap_extents(p);
2483         swap_cgroup_swapoff(p->type);
2484         spin_lock(&swap_lock);
2485         p->swap_file = NULL;
2486         p->flags = 0;
2487         spin_unlock(&swap_lock);
2488         vfree(swap_map);
2489         vfree(cluster_info);
2490         if (swap_file) {
2491                 if (inode && S_ISREG(inode->i_mode)) {
2492                         mutex_unlock(&inode->i_mutex);
2493                         inode = NULL;
2494                 }
2495                 filp_close(swap_file, NULL);
2496         }
2497 out:
2498         if (page && !IS_ERR(page)) {
2499                 kunmap(page);
2500                 page_cache_release(page);
2501         }
2502         if (name)
2503                 putname(name);
2504         if (inode && S_ISREG(inode->i_mode))
2505                 mutex_unlock(&inode->i_mutex);
2506         return error;
2507 }
2508
2509 void si_swapinfo(struct sysinfo *val)
2510 {
2511         unsigned int type;
2512         unsigned long nr_to_be_unused = 0;
2513
2514         spin_lock(&swap_lock);
2515         for (type = 0; type < nr_swapfiles; type++) {
2516                 struct swap_info_struct *si = swap_info[type];
2517
2518                 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
2519                         nr_to_be_unused += si->inuse_pages;
2520         }
2521         val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
2522         val->totalswap = total_swap_pages + nr_to_be_unused;
2523         spin_unlock(&swap_lock);
2524 }
2525
2526 /*
2527  * Verify that a swap entry is valid and increment its swap map count.
2528  *
2529  * Returns error code in following case.
2530  * - success -> 0
2531  * - swp_entry is invalid -> EINVAL
2532  * - swp_entry is migration entry -> EINVAL
2533  * - swap-cache reference is requested but there is already one. -> EEXIST
2534  * - swap-cache reference is requested but the entry is not used. -> ENOENT
2535  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
2536  */
2537 static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
2538 {
2539         struct swap_info_struct *p;
2540         unsigned long offset, type;
2541         unsigned char count;
2542         unsigned char has_cache;
2543         int err = -EINVAL;
2544
2545         if (non_swap_entry(entry))
2546                 goto out;
2547
2548         type = swp_type(entry);
2549         if (type >= nr_swapfiles)
2550                 goto bad_file;
2551         p = swap_info[type];
2552         offset = swp_offset(entry);
2553
2554         spin_lock(&p->lock);
2555         if (unlikely(offset >= p->max))
2556                 goto unlock_out;
2557
2558         count = p->swap_map[offset];
2559
2560         /*
2561          * swapin_readahead() doesn't check if a swap entry is valid, so the
2562          * swap entry could be SWAP_MAP_BAD. Check here with lock held.
2563          */
2564         if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
2565                 err = -ENOENT;
2566                 goto unlock_out;
2567         }
2568
2569         has_cache = count & SWAP_HAS_CACHE;
2570         count &= ~SWAP_HAS_CACHE;
2571         err = 0;
2572
2573         if (usage == SWAP_HAS_CACHE) {
2574
2575                 /* set SWAP_HAS_CACHE if there is no cache and entry is used */
2576                 if (!has_cache && count)
2577                         has_cache = SWAP_HAS_CACHE;
2578                 else if (has_cache)             /* someone else added cache */
2579                         err = -EEXIST;
2580                 else                            /* no users remaining */
2581                         err = -ENOENT;
2582
2583         } else if (count || has_cache) {
2584
2585                 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
2586                         count += usage;
2587                 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
2588                         err = -EINVAL;
2589                 else if (swap_count_continued(p, offset, count))
2590                         count = COUNT_CONTINUED;
2591                 else
2592                         err = -ENOMEM;
2593         } else
2594                 err = -ENOENT;                  /* unused swap entry */
2595
2596         p->swap_map[offset] = count | has_cache;
2597
2598 unlock_out:
2599         spin_unlock(&p->lock);
2600 out:
2601         return err;
2602
2603 bad_file:
2604         pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
2605         goto out;
2606 }
2607
2608 /*
2609  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
2610  * (in which case its reference count is never incremented).
2611  */
2612 void swap_shmem_alloc(swp_entry_t entry)
2613 {
2614         __swap_duplicate(entry, SWAP_MAP_SHMEM);
2615 }
2616
2617 /*
2618  * Increase reference count of swap entry by 1.
2619  * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
2620  * but could not be atomically allocated.  Returns 0, just as if it succeeded,
2621  * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
2622  * might occur if a page table entry has got corrupted.
2623  */
2624 int swap_duplicate(swp_entry_t entry)
2625 {
2626         int err = 0;
2627
2628         while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
2629                 err = add_swap_count_continuation(entry, GFP_ATOMIC);
2630         return err;
2631 }
2632
2633 /*
2634  * @entry: swap entry for which we allocate swap cache.
2635  *
2636  * Called when allocating swap cache for existing swap entry,
2637  * This can return error codes. Returns 0 at success.
2638  * -EBUSY means there is a swap cache.
2639  * Note: return code is different from swap_duplicate().
2640  */
2641 int swapcache_prepare(swp_entry_t entry)
2642 {
2643         return __swap_duplicate(entry, SWAP_HAS_CACHE);
2644 }
2645
2646 struct swap_info_struct *page_swap_info(struct page *page)
2647 {
2648         swp_entry_t swap = { .val = page_private(page) };
2649         BUG_ON(!PageSwapCache(page));
2650         return swap_info[swp_type(swap)];
2651 }
2652
2653 /*
2654  * out-of-line __page_file_ methods to avoid include hell.
2655  */
2656 struct address_space *__page_file_mapping(struct page *page)
2657 {
2658         VM_BUG_ON(!PageSwapCache(page));
2659         return page_swap_info(page)->swap_file->f_mapping;
2660 }
2661 EXPORT_SYMBOL_GPL(__page_file_mapping);
2662
2663 pgoff_t __page_file_index(struct page *page)
2664 {
2665         swp_entry_t swap = { .val = page_private(page) };
2666         VM_BUG_ON(!PageSwapCache(page));
2667         return swp_offset(swap);
2668 }
2669 EXPORT_SYMBOL_GPL(__page_file_index);
2670
2671 /*
2672  * add_swap_count_continuation - called when a swap count is duplicated
2673  * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
2674  * page of the original vmalloc'ed swap_map, to hold the continuation count
2675  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
2676  * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
2677  *
2678  * These continuation pages are seldom referenced: the common paths all work
2679  * on the original swap_map, only referring to a continuation page when the
2680  * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
2681  *
2682  * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
2683  * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
2684  * can be called after dropping locks.
2685  */
2686 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
2687 {
2688         struct swap_info_struct *si;
2689         struct page *head;
2690         struct page *page;
2691         struct page *list_page;
2692         pgoff_t offset;
2693         unsigned char count;
2694
2695         /*
2696          * When debugging, it's easier to use __GFP_ZERO here; but it's better
2697          * for latency not to zero a page while GFP_ATOMIC and holding locks.
2698          */
2699         page = alloc_page(gfp_mask | __GFP_HIGHMEM);
2700
2701         si = swap_info_get(entry);
2702         if (!si) {
2703                 /*
2704                  * An acceptable race has occurred since the failing
2705                  * __swap_duplicate(): the swap entry has been freed,
2706                  * perhaps even the whole swap_map cleared for swapoff.
2707                  */
2708                 goto outer;
2709         }
2710
2711         offset = swp_offset(entry);
2712         count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
2713
2714         if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
2715                 /*
2716                  * The higher the swap count, the more likely it is that tasks
2717                  * will race to add swap count continuation: we need to avoid
2718                  * over-provisioning.
2719                  */
2720                 goto out;
2721         }
2722
2723         if (!page) {
2724                 spin_unlock(&si->lock);
2725                 return -ENOMEM;
2726         }
2727
2728         /*
2729          * We are fortunate that although vmalloc_to_page uses pte_offset_map,
2730          * no architecture is using highmem pages for kernel pagetables: so it
2731          * will not corrupt the GFP_ATOMIC caller's atomic pagetable kmaps.
2732          */
2733         head = vmalloc_to_page(si->swap_map + offset);
2734         offset &= ~PAGE_MASK;
2735
2736         /*
2737          * Page allocation does not initialize the page's lru field,
2738          * but it does always reset its private field.
2739          */
2740         if (!page_private(head)) {
2741                 BUG_ON(count & COUNT_CONTINUED);
2742                 INIT_LIST_HEAD(&head->lru);
2743                 set_page_private(head, SWP_CONTINUED);
2744                 si->flags |= SWP_CONTINUED;
2745         }
2746
2747         list_for_each_entry(list_page, &head->lru, lru) {
2748                 unsigned char *map;
2749
2750                 /*
2751                  * If the previous map said no continuation, but we've found
2752                  * a continuation page, free our allocation and use this one.
2753                  */
2754                 if (!(count & COUNT_CONTINUED))
2755                         goto out;
2756
2757                 map = kmap_atomic(list_page) + offset;
2758                 count = *map;
2759                 kunmap_atomic(map);
2760
2761                 /*
2762                  * If this continuation count now has some space in it,
2763                  * free our allocation and use this one.
2764                  */
2765                 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
2766                         goto out;
2767         }
2768
2769         list_add_tail(&page->lru, &head->lru);
2770         page = NULL;                    /* now it's attached, don't free it */
2771 out:
2772         spin_unlock(&si->lock);
2773 outer:
2774         if (page)
2775                 __free_page(page);
2776         return 0;
2777 }
2778
2779 /*
2780  * swap_count_continued - when the original swap_map count is incremented
2781  * from SWAP_MAP_MAX, check if there is already a continuation page to carry
2782  * into, carry if so, or else fail until a new continuation page is allocated;
2783  * when the original swap_map count is decremented from 0 with continuation,
2784  * borrow from the continuation and report whether it still holds more.
2785  * Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
2786  */
2787 static bool swap_count_continued(struct swap_info_struct *si,
2788                                  pgoff_t offset, unsigned char count)
2789 {
2790         struct page *head;
2791         struct page *page;
2792         unsigned char *map;
2793
2794         head = vmalloc_to_page(si->swap_map + offset);
2795         if (page_private(head) != SWP_CONTINUED) {
2796                 BUG_ON(count & COUNT_CONTINUED);
2797                 return false;           /* need to add count continuation */
2798         }
2799
2800         offset &= ~PAGE_MASK;
2801         page = list_entry(head->lru.next, struct page, lru);
2802         map = kmap_atomic(page) + offset;
2803
2804         if (count == SWAP_MAP_MAX)      /* initial increment from swap_map */
2805                 goto init_map;          /* jump over SWAP_CONT_MAX checks */
2806
2807         if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
2808                 /*
2809                  * Think of how you add 1 to 999
2810                  */
2811                 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
2812                         kunmap_atomic(map);
2813                         page = list_entry(page->lru.next, struct page, lru);
2814                         BUG_ON(page == head);
2815                         map = kmap_atomic(page) + offset;
2816                 }
2817                 if (*map == SWAP_CONT_MAX) {
2818                         kunmap_atomic(map);
2819                         page = list_entry(page->lru.next, struct page, lru);
2820                         if (page == head)
2821                                 return false;   /* add count continuation */
2822                         map = kmap_atomic(page) + offset;
2823 init_map:               *map = 0;               /* we didn't zero the page */
2824                 }
2825                 *map += 1;
2826                 kunmap_atomic(map);
2827                 page = list_entry(page->lru.prev, struct page, lru);
2828                 while (page != head) {
2829                         map = kmap_atomic(page) + offset;
2830                         *map = COUNT_CONTINUED;
2831                         kunmap_atomic(map);
2832                         page = list_entry(page->lru.prev, struct page, lru);
2833                 }
2834                 return true;                    /* incremented */
2835
2836         } else {                                /* decrementing */
2837                 /*
2838                  * Think of how you subtract 1 from 1000
2839                  */
2840                 BUG_ON(count != COUNT_CONTINUED);
2841                 while (*map == COUNT_CONTINUED) {
2842                         kunmap_atomic(map);
2843                         page = list_entry(page->lru.next, struct page, lru);
2844                         BUG_ON(page == head);
2845                         map = kmap_atomic(page) + offset;
2846                 }
2847                 BUG_ON(*map == 0);
2848                 *map -= 1;
2849                 if (*map == 0)
2850                         count = 0;
2851                 kunmap_atomic(map);
2852                 page = list_entry(page->lru.prev, struct page, lru);
2853                 while (page != head) {
2854                         map = kmap_atomic(page) + offset;
2855                         *map = SWAP_CONT_MAX | count;
2856                         count = COUNT_CONTINUED;
2857                         kunmap_atomic(map);
2858                         page = list_entry(page->lru.prev, struct page, lru);
2859                 }
2860                 return count == COUNT_CONTINUED;
2861         }
2862 }
2863
2864 /*
2865  * free_swap_count_continuations - swapoff free all the continuation pages
2866  * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
2867  */
2868 static void free_swap_count_continuations(struct swap_info_struct *si)
2869 {
2870         pgoff_t offset;
2871
2872         for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
2873                 struct page *head;
2874                 head = vmalloc_to_page(si->swap_map + offset);
2875                 if (page_private(head)) {
2876                         struct list_head *this, *next;
2877                         list_for_each_safe(this, next, &head->lru) {
2878                                 struct page *page;
2879                                 page = list_entry(this, struct page, lru);
2880                                 list_del(this);
2881                                 __free_page(page);
2882                         }
2883                 }
2884         }
2885 }