mm/shmem.c

   1 /*
   2  * Resizable virtual memory filesystem for Linux.
   3  *
   4  * Copyright (C) 2000 Linus Torvalds.
   5  *               2000 Transmeta Corp.
   6  *               2000-2001 Christoph Rohland
   7  *               2000-2001 SAP AG
   8  *               2002 Red Hat Inc.
   9  * Copyright (C) 2002-2011 Hugh Dickins.
  10  * Copyright (C) 2011 Google Inc.
  11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
  12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
  13  *
  14  * Extended attribute support for tmpfs:
  15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  17  *
  18  * tiny-shmem:
  19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  20  *
  21  * This file is released under the GPL.
  22  */
  23
  24 #include <linux/fs.h>
  25 #include <linux/init.h>
  26 #include <linux/vfs.h>
  27 #include <linux/mount.h>
  28 #include <linux/ramfs.h>
  29 #include <linux/pagemap.h>
  30 #include <linux/file.h>
  31 #include <linux/mm.h>
  32 #include <linux/random.h>
  33 #include <linux/sched/signal.h>
  34 #include <linux/export.h>
  35 #include <linux/swap.h>
  36 #include <linux/uio.h>
  37 #include <linux/khugepaged.h>
  38 #include <linux/hugetlb.h>
  39 #include <linux/frontswap.h>
  40 #include <linux/fs_parser.h>
  41
  42 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
  43
  44 static struct vfsmount *shm_mnt;
  45
  46 #ifdef CONFIG_SHMEM
  47 /*
  48  * This virtual memory filesystem is heavily based on the ramfs. It
  49  * extends ramfs by the ability to use swap and honor resource limits
  50  * which makes it a completely usable filesystem.
  51  */
  52
  53 #include <linux/xattr.h>
  54 #include <linux/exportfs.h>
  55 #include <linux/posix_acl.h>
  56 #include <linux/posix_acl_xattr.h>
  57 #include <linux/mman.h>
  58 #include <linux/string.h>
  59 #include <linux/slab.h>
  60 #include <linux/backing-dev.h>
  61 #include <linux/shmem_fs.h>
  62 #include <linux/writeback.h>
  63 #include <linux/blkdev.h>
  64 #include <linux/pagevec.h>
  65 #include <linux/percpu_counter.h>
  66 #include <linux/falloc.h>
  67 #include <linux/splice.h>
  68 #include <linux/security.h>
  69 #include <linux/swapops.h>
  70 #include <linux/mempolicy.h>
  71 #include <linux/namei.h>
  72 #include <linux/ctype.h>
  73 #include <linux/migrate.h>
  74 #include <linux/highmem.h>
  75 #include <linux/seq_file.h>
  76 #include <linux/magic.h>
  77 #include <linux/syscalls.h>
  78 #include <linux/fcntl.h>
  79 #include <uapi/linux/memfd.h>
  80 #include <linux/userfaultfd_k.h>
  81 #include <linux/rmap.h>
  82 #include <linux/uuid.h>
  83
  84 #include <linux/uaccess.h>
  85
  86 #include "internal.h"
  87
  88 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
  89 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
  90
  91 /* Pretend that each entry is of this size in directory's i_size */
  92 #define BOGO_DIRENT_SIZE 20
  93
  94 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
  95 #define SHORT_SYMLINK_LEN 128
  96
  97 /*
  98  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
  99  * inode->i_private (with i_mutex making sure that it has only one user at
 100  * a time): we would prefer not to enlarge the shmem inode just for that.
 101  */
 102 struct shmem_falloc {
 103         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
 104         pgoff_t start;          /* start of range currently being fallocated */
 105         pgoff_t next;           /* the next page offset to be fallocated */
 106         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
 107         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
 108 };
 109
 110 struct shmem_options {
 111         unsigned long long blocks;
 112         unsigned long long inodes;
 113         struct mempolicy *mpol;
 114         kuid_t uid;
 115         kgid_t gid;
 116         umode_t mode;
 117         int huge;
 118         int seen;
 119 #define SHMEM_SEEN_BLOCKS 1
 120 #define SHMEM_SEEN_INODES 2
 121 #define SHMEM_SEEN_HUGE 4
 122 };
 123
 124 #ifdef CONFIG_TMPFS
 125 static unsigned long shmem_default_max_blocks(void)
 126 {
 127         return totalram_pages() / 2;
 128 }
 129
 130 static unsigned long shmem_default_max_inodes(void)
 131 {
 132         unsigned long nr_pages = totalram_pages();
 133
 134         return min(nr_pages - totalhigh_pages(), nr_pages / 2);
 135 }
 136 #endif
 137
 138 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
 139 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
 140                                 struct shmem_inode_info *info, pgoff_t index);
 141 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
 142                              struct page **pagep, enum sgp_type sgp,
 143                              gfp_t gfp, struct vm_area_struct *vma,
 144                              vm_fault_t *fault_type);
 145 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
 146                 struct page **pagep, enum sgp_type sgp,
 147                 gfp_t gfp, struct vm_area_struct *vma,
 148                 struct vm_fault *vmf, vm_fault_t *fault_type);
 149
 150 int shmem_getpage(struct inode *inode, pgoff_t index,
 151                 struct page **pagep, enum sgp_type sgp)
 152 {
 153         return shmem_getpage_gfp(inode, index, pagep, sgp,
 154                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
 155 }
 156
 157 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
 158 {
 159         return sb->s_fs_info;
 160 }
 161
 162 /*
 163  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 164  * for shared memory and for shared anonymous (/dev/zero) mappings
 165  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 166  * consistent with the pre-accounting of private mappings ...
 167  */
 168 static inline int shmem_acct_size(unsigned long flags, loff_t size)
 169 {
 170         return (flags & VM_NORESERVE) ?
 171                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
 172 }
 173
 174 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 175 {
 176         if (!(flags & VM_NORESERVE))
 177                 vm_unacct_memory(VM_ACCT(size));
 178 }
 179
 180 static inline int shmem_reacct_size(unsigned long flags,
 181                 loff_t oldsize, loff_t newsize)
 182 {
 183         if (!(flags & VM_NORESERVE)) {
 184                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
 185                         return security_vm_enough_memory_mm(current->mm,
 186                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
 187                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
 188                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
 189         }
 190         return 0;
 191 }
 192
 193 /*
 194  * ... whereas tmpfs objects are accounted incrementally as
 195  * pages are allocated, in order to allow large sparse files.
 196  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
 197  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 198  */
 199 static inline int shmem_acct_block(unsigned long flags, long pages)
 200 {
 201         if (!(flags & VM_NORESERVE))
 202                 return 0;
 203
 204         return security_vm_enough_memory_mm(current->mm,
 205                         pages * VM_ACCT(PAGE_SIZE));
 206 }
 207
 208 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
 209 {
 210         if (flags & VM_NORESERVE)
 211                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
 212 }
 213
 214 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
 215 {
 216         struct shmem_inode_info *info = SHMEM_I(inode);
 217         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 218
 219         if (shmem_acct_block(info->flags, pages))
 220                 return false;
 221
 222         if (sbinfo->max_blocks) {
 223                 if (percpu_counter_compare(&sbinfo->used_blocks,
 224                                            sbinfo->max_blocks - pages) > 0)
 225                         goto unacct;
 226                 percpu_counter_add(&sbinfo->used_blocks, pages);
 227         }
 228
 229         return true;
 230
 231 unacct:
 232         shmem_unacct_blocks(info->flags, pages);
 233         return false;
 234 }
 235
 236 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
 237 {
 238         struct shmem_inode_info *info = SHMEM_I(inode);
 239         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 240
 241         if (sbinfo->max_blocks)
 242                 percpu_counter_sub(&sbinfo->used_blocks, pages);
 243         shmem_unacct_blocks(info->flags, pages);
 244 }
 245
 246 static const struct super_operations shmem_ops;
 247 static const struct address_space_operations shmem_aops;
 248 static const struct file_operations shmem_file_operations;
 249 static const struct inode_operations shmem_inode_operations;
 250 static const struct inode_operations shmem_dir_inode_operations;
 251 static const struct inode_operations shmem_special_inode_operations;
 252 static const struct vm_operations_struct shmem_vm_ops;
 253 static struct file_system_type shmem_fs_type;
 254
 255 bool vma_is_shmem(struct vm_area_struct *vma)
 256 {
 257         return vma->vm_ops == &shmem_vm_ops;
 258 }
 259
 260 static LIST_HEAD(shmem_swaplist);
 261 static DEFINE_MUTEX(shmem_swaplist_mutex);
 262
 263 static int shmem_reserve_inode(struct super_block *sb)
 264 {
 265         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 266         if (sbinfo->max_inodes) {
 267                 spin_lock(&sbinfo->stat_lock);
 268                 if (!sbinfo->free_inodes) {
 269                         spin_unlock(&sbinfo->stat_lock);
 270                         return -ENOSPC;
 271                 }
 272                 sbinfo->free_inodes--;
 273                 spin_unlock(&sbinfo->stat_lock);
 274         }
 275         return 0;
 276 }
 277
 278 static void shmem_free_inode(struct super_block *sb)
 279 {
 280         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 281         if (sbinfo->max_inodes) {
 282                 spin_lock(&sbinfo->stat_lock);
 283                 sbinfo->free_inodes++;
 284                 spin_unlock(&sbinfo->stat_lock);
 285         }
 286 }
 287
 288 /**
 289  * shmem_recalc_inode - recalculate the block usage of an inode
 290  * @inode: inode to recalc
 291  *
 292  * We have to calculate the free blocks since the mm can drop
 293  * undirtied hole pages behind our back.
 294  *
 295  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 296  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 297  *
 298  * It has to be called with the spinlock held.
 299  */
 300 static void shmem_recalc_inode(struct inode *inode)
 301 {
 302         struct shmem_inode_info *info = SHMEM_I(inode);
 303         long freed;
 304
 305         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
 306         if (freed > 0) {
 307                 info->alloced -= freed;
 308                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
 309                 shmem_inode_unacct_blocks(inode, freed);
 310         }
 311 }
 312
 313 bool shmem_charge(struct inode *inode, long pages)
 314 {
 315         struct shmem_inode_info *info = SHMEM_I(inode);
 316         unsigned long flags;
 317
 318         if (!shmem_inode_acct_block(inode, pages))
 319                 return false;
 320
 321         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
 322         inode->i_mapping->nrpages += pages;
 323
 324         spin_lock_irqsave(&info->lock, flags);
 325         info->alloced += pages;
 326         inode->i_blocks += pages * BLOCKS_PER_PAGE;
 327         shmem_recalc_inode(inode);
 328         spin_unlock_irqrestore(&info->lock, flags);
 329
 330         return true;
 331 }
 332
 333 void shmem_uncharge(struct inode *inode, long pages)
 334 {
 335         struct shmem_inode_info *info = SHMEM_I(inode);
 336         unsigned long flags;
 337
 338         /* nrpages adjustment done by __delete_from_page_cache() or caller */
 339
 340         spin_lock_irqsave(&info->lock, flags);
 341         info->alloced -= pages;
 342         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
 343         shmem_recalc_inode(inode);
 344         spin_unlock_irqrestore(&info->lock, flags);
 345
 346         shmem_inode_unacct_blocks(inode, pages);
 347 }
 348
 349 /*
 350  * Replace item expected in xarray by a new item, while holding xa_lock.
 351  */
 352 static int shmem_replace_entry(struct address_space *mapping,
 353                         pgoff_t index, void *expected, void *replacement)
 354 {
 355         XA_STATE(xas, &mapping->i_pages, index);
 356         void *item;
 357
 358         VM_BUG_ON(!expected);
 359         VM_BUG_ON(!replacement);
 360         item = xas_load(&xas);
 361         if (item != expected)
 362                 return -ENOENT;
 363         xas_store(&xas, replacement);
 364         return 0;
 365 }
 366
 367 /*
 368  * Sometimes, before we decide whether to proceed or to fail, we must check
 369  * that an entry was not already brought back from swap by a racing thread.
 370  *
 371  * Checking page is not enough: by the time a SwapCache page is locked, it
 372  * might be reused, and again be SwapCache, using the same swap as before.
 373  */
 374 static bool shmem_confirm_swap(struct address_space *mapping,
 375                                pgoff_t index, swp_entry_t swap)
 376 {
 377         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
 378 }
 379
 380 /*
 381  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
 382  *
 383  * SHMEM_HUGE_NEVER:
 384  *      disables huge pages for the mount;
 385  * SHMEM_HUGE_ALWAYS:
 386  *      enables huge pages for the mount;
 387  * SHMEM_HUGE_WITHIN_SIZE:
 388  *      only allocate huge pages if the page will be fully within i_size,
 389  *      also respect fadvise()/madvise() hints;
 390  * SHMEM_HUGE_ADVISE:
 391  *      only allocate huge pages if requested with fadvise()/madvise();
 392  */
 393
 394 #define SHMEM_HUGE_NEVER        0
 395 #define SHMEM_HUGE_ALWAYS       1
 396 #define SHMEM_HUGE_WITHIN_SIZE  2
 397 #define SHMEM_HUGE_ADVISE       3
 398
 399 /*
 400  * Special values.
 401  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
 402  *
 403  * SHMEM_HUGE_DENY:
 404  *      disables huge on shm_mnt and all mounts, for emergency use;
 405  * SHMEM_HUGE_FORCE:
 406  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
 407  *
 408  */
 409 #define SHMEM_HUGE_DENY         (-1)
 410 #define SHMEM_HUGE_FORCE        (-2)
 411
 412 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 413 /* ifdef here to avoid bloating shmem.o when not necessary */
 414
 415 static int shmem_huge __read_mostly;
 416
 417 #if defined(CONFIG_SYSFS)
 418 static int shmem_parse_huge(const char *str)
 419 {
 420         if (!strcmp(str, "never"))
 421                 return SHMEM_HUGE_NEVER;
 422         if (!strcmp(str, "always"))
 423                 return SHMEM_HUGE_ALWAYS;
 424         if (!strcmp(str, "within_size"))
 425                 return SHMEM_HUGE_WITHIN_SIZE;
 426         if (!strcmp(str, "advise"))
 427                 return SHMEM_HUGE_ADVISE;
 428         if (!strcmp(str, "deny"))
 429                 return SHMEM_HUGE_DENY;
 430         if (!strcmp(str, "force"))
 431                 return SHMEM_HUGE_FORCE;
 432         return -EINVAL;
 433 }
 434 #endif
 435
 436 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
 437 static const char *shmem_format_huge(int huge)
 438 {
 439         switch (huge) {
 440         case SHMEM_HUGE_NEVER:
 441                 return "never";
 442         case SHMEM_HUGE_ALWAYS:
 443                 return "always";
 444         case SHMEM_HUGE_WITHIN_SIZE:
 445                 return "within_size";
 446         case SHMEM_HUGE_ADVISE:
 447                 return "advise";
 448         case SHMEM_HUGE_DENY:
 449                 return "deny";
 450         case SHMEM_HUGE_FORCE:
 451                 return "force";
 452         default:
 453                 VM_BUG_ON(1);
 454                 return "bad_val";
 455         }
 456 }
 457 #endif
 458
 459 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 460                 struct shrink_control *sc, unsigned long nr_to_split)
 461 {
 462         LIST_HEAD(list), *pos, *next;
 463         LIST_HEAD(to_remove);
 464         struct inode *inode;
 465         struct shmem_inode_info *info;
 466         struct page *page;
 467         unsigned long batch = sc ? sc->nr_to_scan : 128;
 468         int removed = 0, split = 0;
 469
 470         if (list_empty(&sbinfo->shrinklist))
 471                 return SHRINK_STOP;
 472
 473         spin_lock(&sbinfo->shrinklist_lock);
 474         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
 475                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 476
 477                 /* pin the inode */
 478                 inode = igrab(&info->vfs_inode);
 479
 480                 /* inode is about to be evicted */
 481                 if (!inode) {
 482                         list_del_init(&info->shrinklist);
 483                         removed++;
 484                         goto next;
 485                 }
 486
 487                 /* Check if there's anything to gain */
 488                 if (round_up(inode->i_size, PAGE_SIZE) ==
 489                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
 490                         list_move(&info->shrinklist, &to_remove);
 491                         removed++;
 492                         goto next;
 493                 }
 494
 495                 list_move(&info->shrinklist, &list);
 496 next:
 497                 if (!--batch)
 498                         break;
 499         }
 500         spin_unlock(&sbinfo->shrinklist_lock);
 501
 502         list_for_each_safe(pos, next, &to_remove) {
 503                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 504                 inode = &info->vfs_inode;
 505                 list_del_init(&info->shrinklist);
 506                 iput(inode);
 507         }
 508
 509         list_for_each_safe(pos, next, &list) {
 510                 int ret;
 511
 512                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 513                 inode = &info->vfs_inode;
 514
 515                 if (nr_to_split && split >= nr_to_split)
 516                         goto leave;
 517
 518                 page = find_get_page(inode->i_mapping,
 519                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
 520                 if (!page)
 521                         goto drop;
 522
 523                 /* No huge page at the end of the file: nothing to split */
 524                 if (!PageTransHuge(page)) {
 525                         put_page(page);
 526                         goto drop;
 527                 }
 528
 529                 /*
 530                  * Leave the inode on the list if we failed to lock
 531                  * the page at this time.
 532                  *
 533                  * Waiting for the lock may lead to deadlock in the
 534                  * reclaim path.
 535                  */
 536                 if (!trylock_page(page)) {
 537                         put_page(page);
 538                         goto leave;
 539                 }
 540
 541                 ret = split_huge_page(page);
 542                 unlock_page(page);
 543                 put_page(page);
 544
 545                 /* If split failed leave the inode on the list */
 546                 if (ret)
 547                         goto leave;
 548
 549                 split++;
 550 drop:
 551                 list_del_init(&info->shrinklist);
 552                 removed++;
 553 leave:
 554                 iput(inode);
 555         }
 556
 557         spin_lock(&sbinfo->shrinklist_lock);
 558         list_splice_tail(&list, &sbinfo->shrinklist);
 559         sbinfo->shrinklist_len -= removed;
 560         spin_unlock(&sbinfo->shrinklist_lock);
 561
 562         return split;
 563 }
 564
 565 static long shmem_unused_huge_scan(struct super_block *sb,
 566                 struct shrink_control *sc)
 567 {
 568         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 569
 570         if (!READ_ONCE(sbinfo->shrinklist_len))
 571                 return SHRINK_STOP;
 572
 573         return shmem_unused_huge_shrink(sbinfo, sc, 0);
 574 }
 575
 576 static long shmem_unused_huge_count(struct super_block *sb,
 577                 struct shrink_control *sc)
 578 {
 579         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 580         return READ_ONCE(sbinfo->shrinklist_len);
 581 }
 582 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
 583
 584 #define shmem_huge SHMEM_HUGE_DENY
 585
 586 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 587                 struct shrink_control *sc, unsigned long nr_to_split)
 588 {
 589         return 0;
 590 }
 591 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 592
 593 static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
 594 {
 595         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
 596             (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
 597             shmem_huge != SHMEM_HUGE_DENY)
 598                 return true;
 599         return false;
 600 }
 601
 602 /*
 603  * Like add_to_page_cache_locked, but error if expected item has gone.
 604  */
 605 static int shmem_add_to_page_cache(struct page *page,
 606                                    struct address_space *mapping,
 607                                    pgoff_t index, void *expected, gfp_t gfp,
 608                                    struct mm_struct *charge_mm)
 609 {
 610         XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
 611         unsigned long i = 0;
 612         unsigned long nr = compound_nr(page);
 613         int error;
 614
 615         VM_BUG_ON_PAGE(PageTail(page), page);
 616         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
 617         VM_BUG_ON_PAGE(!PageLocked(page), page);
 618         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
 619         VM_BUG_ON(expected && PageTransHuge(page));
 620
 621         page_ref_add(page, nr);
 622         page->mapping = mapping;
 623         page->index = index;
 624
 625         if (!PageSwapCache(page)) {
 626                 error = mem_cgroup_charge(page, charge_mm, gfp);
 627                 if (error) {
 628                         if (PageTransHuge(page)) {
 629                                 count_vm_event(THP_FILE_FALLBACK);
 630                                 count_vm_event(THP_FILE_FALLBACK_CHARGE);
 631                         }
 632                         goto error;
 633                 }
 634         }
 635         cgroup_throttle_swaprate(page, gfp);
 636
 637         do {
 638                 void *entry;
 639                 xas_lock_irq(&xas);
 640                 entry = xas_find_conflict(&xas);
 641                 if (entry != expected)
 642                         xas_set_err(&xas, -EEXIST);
 643                 xas_create_range(&xas);
 644                 if (xas_error(&xas))
 645                         goto unlock;
 646 next:
 647                 xas_store(&xas, page);
 648                 if (++i < nr) {
 649                         xas_next(&xas);
 650                         goto next;
 651                 }
 652                 if (PageTransHuge(page)) {
 653                         count_vm_event(THP_FILE_ALLOC);
 654                         __inc_node_page_state(page, NR_SHMEM_THPS);
 655                 }
 656                 mapping->nrpages += nr;
 657                 __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
 658                 __mod_lruvec_page_state(page, NR_SHMEM, nr);
 659 unlock:
 660                 xas_unlock_irq(&xas);
 661         } while (xas_nomem(&xas, gfp));
 662
 663         if (xas_error(&xas)) {
 664                 error = xas_error(&xas);
 665                 goto error;
 666         }
 667
 668         return 0;
 669 error:
 670         page->mapping = NULL;
 671         page_ref_sub(page, nr);
 672         return error;
 673 }
 674
 675 /*
 676  * Like delete_from_page_cache, but substitutes swap for page.
 677  */
 678 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
 679 {
 680         struct address_space *mapping = page->mapping;
 681         int error;
 682
 683         VM_BUG_ON_PAGE(PageCompound(page), page);
 684
 685         xa_lock_irq(&mapping->i_pages);
 686         error = shmem_replace_entry(mapping, page->index, page, radswap);
 687         page->mapping = NULL;
 688         mapping->nrpages--;
 689         __dec_lruvec_page_state(page, NR_FILE_PAGES);
 690         __dec_lruvec_page_state(page, NR_SHMEM);
 691         xa_unlock_irq(&mapping->i_pages);
 692         put_page(page);
 693         BUG_ON(error);
 694 }
 695
 696 /*
 697  * Remove swap entry from page cache, free the swap and its page cache.
 698  */
 699 static int shmem_free_swap(struct address_space *mapping,
 700                            pgoff_t index, void *radswap)
 701 {
 702         void *old;
 703
 704         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
 705         if (old != radswap)
 706                 return -ENOENT;
 707         free_swap_and_cache(radix_to_swp_entry(radswap));
 708         return 0;
 709 }
 710
 711 /*
 712  * Determine (in bytes) how many of the shmem object's pages mapped by the
 713  * given offsets are swapped out.
 714  *
 715  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
 716  * as long as the inode doesn't go away and racy results are not a problem.
 717  */
 718 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 719                                                 pgoff_t start, pgoff_t end)
 720 {
 721         XA_STATE(xas, &mapping->i_pages, start);
 722         struct page *page;
 723         unsigned long swapped = 0;
 724
 725         rcu_read_lock();
 726         xas_for_each(&xas, page, end - 1) {
 727                 if (xas_retry(&xas, page))
 728                         continue;
 729                 if (xa_is_value(page))
 730                         swapped++;
 731
 732                 if (need_resched()) {
 733                         xas_pause(&xas);
 734                         cond_resched_rcu();
 735                 }
 736         }
 737
 738         rcu_read_unlock();
 739
 740         return swapped << PAGE_SHIFT;
 741 }
 742
 743 /*
 744  * Determine (in bytes) how many of the shmem object's pages mapped by the
 745  * given vma is swapped out.
 746  *
 747  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
 748  * as long as the inode doesn't go away and racy results are not a problem.
 749  */
 750 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
 751 {
 752         struct inode *inode = file_inode(vma->vm_file);
 753         struct shmem_inode_info *info = SHMEM_I(inode);
 754         struct address_space *mapping = inode->i_mapping;
 755         unsigned long swapped;
 756
 757         /* Be careful as we don't hold info->lock */
 758         swapped = READ_ONCE(info->swapped);
 759
 760         /*
 761          * The easier cases are when the shmem object has nothing in swap, or
 762          * the vma maps it whole. Then we can simply use the stats that we
 763          * already track.
 764          */
 765         if (!swapped)
 766                 return 0;
 767
 768         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
 769                 return swapped << PAGE_SHIFT;
 770
 771         /* Here comes the more involved part */
 772         return shmem_partial_swap_usage(mapping,
 773                         linear_page_index(vma, vma->vm_start),
 774                         linear_page_index(vma, vma->vm_end));
 775 }
 776
 777 /*
 778  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
 779  */
 780 void shmem_unlock_mapping(struct address_space *mapping)
 781 {
 782         struct pagevec pvec;
 783         pgoff_t indices[PAGEVEC_SIZE];
 784         pgoff_t index = 0;
 785
 786         pagevec_init(&pvec);
 787         /*
 788          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
 789          */
 790         while (!mapping_unevictable(mapping)) {
 791                 /*
 792                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
 793                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
 794                  */
 795                 pvec.nr = find_get_entries(mapping, index,
 796                                            PAGEVEC_SIZE, pvec.pages, indices);
 797                 if (!pvec.nr)
 798                         break;
 799                 index = indices[pvec.nr - 1] + 1;
 800                 pagevec_remove_exceptionals(&pvec);
 801                 check_move_unevictable_pages(&pvec);
 802                 pagevec_release(&pvec);
 803                 cond_resched();
 804         }
 805 }
 806
 807 /*
 808  * Check whether a hole-punch or truncation needs to split a huge page,
 809  * returning true if no split was required, or the split has been successful.
 810  *
 811  * Eviction (or truncation to 0 size) should never need to split a huge page;
 812  * but in rare cases might do so, if shmem_undo_range() failed to trylock on
 813  * head, and then succeeded to trylock on tail.
 814  *
 815  * A split can only succeed when there are no additional references on the
 816  * huge page: so the split below relies upon find_get_entries() having stopped
 817  * when it found a subpage of the huge page, without getting further references.
 818  */
 819 static bool shmem_punch_compound(struct page *page, pgoff_t start, pgoff_t end)
 820 {
 821         if (!PageTransCompound(page))
 822                 return true;
 823
 824         /* Just proceed to delete a huge page wholly within the range punched */
 825         if (PageHead(page) &&
 826             page->index >= start && page->index + HPAGE_PMD_NR <= end)
 827                 return true;
 828
 829         /* Try to split huge page, so we can truly punch the hole or truncate */
 830         return split_huge_page(page) >= 0;
 831 }
 832
 833 /*
 834  * Remove range of pages and swap entries from page cache, and free them.
 835  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
 836  */
 837 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
 838                                                                  bool unfalloc)
 839 {
 840         struct address_space *mapping = inode->i_mapping;
 841         struct shmem_inode_info *info = SHMEM_I(inode);
 842         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
 843         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
 844         unsigned int partial_start = lstart & (PAGE_SIZE - 1);
 845         unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
 846         struct pagevec pvec;
 847         pgoff_t indices[PAGEVEC_SIZE];
 848         long nr_swaps_freed = 0;
 849         pgoff_t index;
 850         int i;
 851
 852         if (lend == -1)
 853                 end = -1;       /* unsigned, so actually very big */
 854
 855         pagevec_init(&pvec);
 856         index = start;
 857         while (index < end) {
 858                 pvec.nr = find_get_entries(mapping, index,
 859                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
 860                         pvec.pages, indices);
 861                 if (!pvec.nr)
 862                         break;
 863                 for (i = 0; i < pagevec_count(&pvec); i++) {
 864                         struct page *page = pvec.pages[i];
 865
 866                         index = indices[i];
 867                         if (index >= end)
 868                                 break;
 869
 870                         if (xa_is_value(page)) {
 871                                 if (unfalloc)
 872                                         continue;
 873                                 nr_swaps_freed += !shmem_free_swap(mapping,
 874                                                                 index, page);
 875                                 continue;
 876                         }
 877
 878                         VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
 879
 880                         if (!trylock_page(page))
 881                                 continue;
 882
 883                         if ((!unfalloc || !PageUptodate(page)) &&
 884                             page_mapping(page) == mapping) {
 885                                 VM_BUG_ON_PAGE(PageWriteback(page), page);
 886                                 if (shmem_punch_compound(page, start, end))
 887                                         truncate_inode_page(mapping, page);
 888                         }
 889                         unlock_page(page);
 890                 }
 891                 pagevec_remove_exceptionals(&pvec);
 892                 pagevec_release(&pvec);
 893                 cond_resched();
 894                 index++;
 895         }
 896
 897         if (partial_start) {
 898                 struct page *page = NULL;
 899                 shmem_getpage(inode, start - 1, &page, SGP_READ);
 900                 if (page) {
 901                         unsigned int top = PAGE_SIZE;
 902                         if (start > end) {
 903                                 top = partial_end;
 904                                 partial_end = 0;
 905                         }
 906                         zero_user_segment(page, partial_start, top);
 907                         set_page_dirty(page);
 908                         unlock_page(page);
 909                         put_page(page);
 910                 }
 911         }
 912         if (partial_end) {
 913                 struct page *page = NULL;
 914                 shmem_getpage(inode, end, &page, SGP_READ);
 915                 if (page) {
 916                         zero_user_segment(page, 0, partial_end);
 917                         set_page_dirty(page);
 918                         unlock_page(page);
 919                         put_page(page);
 920                 }
 921         }
 922         if (start >= end)
 923                 return;
 924
 925         index = start;
 926         while (index < end) {
 927                 cond_resched();
 928
 929                 pvec.nr = find_get_entries(mapping, index,
 930                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
 931                                 pvec.pages, indices);
 932                 if (!pvec.nr) {
 933                         /* If all gone or hole-punch or unfalloc, we're done */
 934                         if (index == start || end != -1)
 935                                 break;
 936                         /* But if truncating, restart to make sure all gone */
 937                         index = start;
 938                         continue;
 939                 }
 940                 for (i = 0; i < pagevec_count(&pvec); i++) {
 941                         struct page *page = pvec.pages[i];
 942
 943                         index = indices[i];
 944                         if (index >= end)
 945                                 break;
 946
 947                         if (xa_is_value(page)) {
 948                                 if (unfalloc)
 949                                         continue;
 950                                 if (shmem_free_swap(mapping, index, page)) {
 951                                         /* Swap was replaced by page: retry */
 952                                         index--;
 953                                         break;
 954                                 }
 955                                 nr_swaps_freed++;
 956                                 continue;
 957                         }
 958
 959                         lock_page(page);
 960
 961                         if (!unfalloc || !PageUptodate(page)) {
 962                                 if (page_mapping(page) != mapping) {
 963                                         /* Page was replaced by swap: retry */
 964                                         unlock_page(page);
 965                                         index--;
 966                                         break;
 967                                 }
 968                                 VM_BUG_ON_PAGE(PageWriteback(page), page);
 969                                 if (shmem_punch_compound(page, start, end))
 970                                         truncate_inode_page(mapping, page);
 971                                 else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
 972                                         /* Wipe the page and don't get stuck */
 973                                         clear_highpage(page);
 974                                         flush_dcache_page(page);
 975                                         set_page_dirty(page);
 976                                         if (index <
 977                                             round_up(start, HPAGE_PMD_NR))
 978                                                 start = index + 1;
 979                                 }
 980                         }
 981                         unlock_page(page);
 982                 }
 983                 pagevec_remove_exceptionals(&pvec);
 984                 pagevec_release(&pvec);
 985                 index++;
 986         }
 987
 988         spin_lock_irq(&info->lock);
 989         info->swapped -= nr_swaps_freed;
 990         shmem_recalc_inode(inode);
 991         spin_unlock_irq(&info->lock);
 992 }
 993
 994 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
 995 {
 996         shmem_undo_range(inode, lstart, lend, false);
 997         inode->i_ctime = inode->i_mtime = current_time(inode);
 998 }
 999 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1000
1001 static int shmem_getattr(const struct path *path, struct kstat *stat,
1002                          u32 request_mask, unsigned int query_flags)
1003 {
1004         struct inode *inode = path->dentry->d_inode;
1005         struct shmem_inode_info *info = SHMEM_I(inode);
1006         struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
1007
1008         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1009                 spin_lock_irq(&info->lock);
1010                 shmem_recalc_inode(inode);
1011                 spin_unlock_irq(&info->lock);
1012         }
1013         generic_fillattr(inode, stat);
1014
1015         if (is_huge_enabled(sb_info))
1016                 stat->blksize = HPAGE_PMD_SIZE;
1017
1018         return 0;
1019 }
1020
1021 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1022 {
1023         struct inode *inode = d_inode(dentry);
1024         struct shmem_inode_info *info = SHMEM_I(inode);
1025         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1026         int error;
1027
1028         error = setattr_prepare(dentry, attr);
1029         if (error)
1030                 return error;
1031
1032         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1033                 loff_t oldsize = inode->i_size;
1034                 loff_t newsize = attr->ia_size;
1035
1036                 /* protected by i_mutex */
1037                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1038                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1039                         return -EPERM;
1040
1041                 if (newsize != oldsize) {
1042                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1043                                         oldsize, newsize);
1044                         if (error)
1045                                 return error;
1046                         i_size_write(inode, newsize);
1047                         inode->i_ctime = inode->i_mtime = current_time(inode);
1048                 }
1049                 if (newsize <= oldsize) {
1050                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1051                         if (oldsize > holebegin)
1052                                 unmap_mapping_range(inode->i_mapping,
1053                                                         holebegin, 0, 1);
1054                         if (info->alloced)
1055                                 shmem_truncate_range(inode,
1056                                                         newsize, (loff_t)-1);
1057                         /* unmap again to remove racily COWed private pages */
1058                         if (oldsize > holebegin)
1059                                 unmap_mapping_range(inode->i_mapping,
1060                                                         holebegin, 0, 1);
1061
1062                         /*
1063                          * Part of the huge page can be beyond i_size: subject
1064                          * to shrink under memory pressure.
1065                          */
1066                         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1067                                 spin_lock(&sbinfo->shrinklist_lock);
1068                                 /*
1069                                  * _careful to defend against unlocked access to
1070                                  * ->shrink_list in shmem_unused_huge_shrink()
1071                                  */
1072                                 if (list_empty_careful(&info->shrinklist)) {
1073                                         list_add_tail(&info->shrinklist,
1074                                                         &sbinfo->shrinklist);
1075                                         sbinfo->shrinklist_len++;
1076                                 }
1077                                 spin_unlock(&sbinfo->shrinklist_lock);
1078                         }
1079                 }
1080         }
1081
1082         setattr_copy(inode, attr);
1083         if (attr->ia_valid & ATTR_MODE)
1084                 error = posix_acl_chmod(inode, inode->i_mode);
1085         return error;
1086 }
1087
1088 static void shmem_evict_inode(struct inode *inode)
1089 {
1090         struct shmem_inode_info *info = SHMEM_I(inode);
1091         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1092
1093         if (inode->i_mapping->a_ops == &shmem_aops) {
1094                 shmem_unacct_size(info->flags, inode->i_size);
1095                 inode->i_size = 0;
1096                 shmem_truncate_range(inode, 0, (loff_t)-1);
1097                 if (!list_empty(&info->shrinklist)) {
1098                         spin_lock(&sbinfo->shrinklist_lock);
1099                         if (!list_empty(&info->shrinklist)) {
1100                                 list_del_init(&info->shrinklist);
1101                                 sbinfo->shrinklist_len--;
1102                         }
1103                         spin_unlock(&sbinfo->shrinklist_lock);
1104                 }
1105                 while (!list_empty(&info->swaplist)) {
1106                         /* Wait while shmem_unuse() is scanning this inode... */
1107                         wait_var_event(&info->stop_eviction,
1108                                        !atomic_read(&info->stop_eviction));
1109                         mutex_lock(&shmem_swaplist_mutex);
1110                         /* ...but beware of the race if we peeked too early */
1111                         if (!atomic_read(&info->stop_eviction))
1112                                 list_del_init(&info->swaplist);
1113                         mutex_unlock(&shmem_swaplist_mutex);
1114                 }
1115         }
1116
1117         simple_xattrs_free(&info->xattrs);
1118         WARN_ON(inode->i_blocks);
1119         shmem_free_inode(inode->i_sb);
1120         clear_inode(inode);
1121 }
1122
1123 extern struct swap_info_struct *swap_info[];
1124
1125 static int shmem_find_swap_entries(struct address_space *mapping,
1126                                    pgoff_t start, unsigned int nr_entries,
1127                                    struct page **entries, pgoff_t *indices,
1128                                    unsigned int type, bool frontswap)
1129 {
1130         XA_STATE(xas, &mapping->i_pages, start);
1131         struct page *page;
1132         swp_entry_t entry;
1133         unsigned int ret = 0;
1134
1135         if (!nr_entries)
1136                 return 0;
1137
1138         rcu_read_lock();
1139         xas_for_each(&xas, page, ULONG_MAX) {
1140                 if (xas_retry(&xas, page))
1141                         continue;
1142
1143                 if (!xa_is_value(page))
1144                         continue;
1145
1146                 entry = radix_to_swp_entry(page);
1147                 if (swp_type(entry) != type)
1148                         continue;
1149                 if (frontswap &&
1150                     !frontswap_test(swap_info[type], swp_offset(entry)))
1151                         continue;
1152
1153                 indices[ret] = xas.xa_index;
1154                 entries[ret] = page;
1155
1156                 if (need_resched()) {
1157                         xas_pause(&xas);
1158                         cond_resched_rcu();
1159                 }
1160                 if (++ret == nr_entries)
1161                         break;
1162         }
1163         rcu_read_unlock();
1164
1165         return ret;
1166 }
1167
1168 /*
1169  * Move the swapped pages for an inode to page cache. Returns the count
1170  * of pages swapped in, or the error in case of failure.
1171  */
1172 static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
1173                                     pgoff_t *indices)
1174 {
1175         int i = 0;
1176         int ret = 0;
1177         int error = 0;
1178         struct address_space *mapping = inode->i_mapping;
1179
1180         for (i = 0; i < pvec.nr; i++) {
1181                 struct page *page = pvec.pages[i];
1182
1183                 if (!xa_is_value(page))
1184                         continue;
1185                 error = shmem_swapin_page(inode, indices[i],
1186                                           &page, SGP_CACHE,
1187                                           mapping_gfp_mask(mapping),
1188                                           NULL, NULL);
1189                 if (error == 0) {
1190                         unlock_page(page);
1191                         put_page(page);
1192                         ret++;
1193                 }
1194                 if (error == -ENOMEM)
1195                         break;
1196                 error = 0;
1197         }
1198         return error ? error : ret;
1199 }
1200
1201 /*
1202  * If swap found in inode, free it and move page from swapcache to filecache.
1203  */
1204 static int shmem_unuse_inode(struct inode *inode, unsigned int type,
1205                              bool frontswap, unsigned long *fs_pages_to_unuse)
1206 {
1207         struct address_space *mapping = inode->i_mapping;
1208         pgoff_t start = 0;
1209         struct pagevec pvec;
1210         pgoff_t indices[PAGEVEC_SIZE];
1211         bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0);
1212         int ret = 0;
1213
1214         pagevec_init(&pvec);
1215         do {
1216                 unsigned int nr_entries = PAGEVEC_SIZE;
1217
1218                 if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE)
1219                         nr_entries = *fs_pages_to_unuse;
1220
1221                 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
1222                                                   pvec.pages, indices,
1223                                                   type, frontswap);
1224                 if (pvec.nr == 0) {
1225                         ret = 0;
1226                         break;
1227                 }
1228
1229                 ret = shmem_unuse_swap_entries(inode, pvec, indices);
1230                 if (ret < 0)
1231                         break;
1232
1233                 if (frontswap_partial) {
1234                         *fs_pages_to_unuse -= ret;
1235                         if (*fs_pages_to_unuse == 0) {
1236                                 ret = FRONTSWAP_PAGES_UNUSED;
1237                                 break;
1238                         }
1239                 }
1240
1241                 start = indices[pvec.nr - 1];
1242         } while (true);
1243
1244         return ret;
1245 }
1246
1247 /*
1248  * Read all the shared memory data that resides in the swap
1249  * device 'type' back into memory, so the swap device can be
1250  * unused.
1251  */
1252 int shmem_unuse(unsigned int type, bool frontswap,
1253                 unsigned long *fs_pages_to_unuse)
1254 {
1255         struct shmem_inode_info *info, *next;
1256         int error = 0;
1257
1258         if (list_empty(&shmem_swaplist))
1259                 return 0;
1260
1261         mutex_lock(&shmem_swaplist_mutex);
1262         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1263                 if (!info->swapped) {
1264                         list_del_init(&info->swaplist);
1265                         continue;
1266                 }
1267                 /*
1268                  * Drop the swaplist mutex while searching the inode for swap;
1269                  * but before doing so, make sure shmem_evict_inode() will not
1270                  * remove placeholder inode from swaplist, nor let it be freed
1271                  * (igrab() would protect from unlink, but not from unmount).
1272                  */
1273                 atomic_inc(&info->stop_eviction);
1274                 mutex_unlock(&shmem_swaplist_mutex);
1275
1276                 error = shmem_unuse_inode(&info->vfs_inode, type, frontswap,
1277                                           fs_pages_to_unuse);
1278                 cond_resched();
1279
1280                 mutex_lock(&shmem_swaplist_mutex);
1281                 next = list_next_entry(info, swaplist);
1282                 if (!info->swapped)
1283                         list_del_init(&info->swaplist);
1284                 if (atomic_dec_and_test(&info->stop_eviction))
1285                         wake_up_var(&info->stop_eviction);
1286                 if (error)
1287                         break;
1288         }
1289         mutex_unlock(&shmem_swaplist_mutex);
1290
1291         return error;
1292 }
1293
1294 /*
1295  * Move the page from the page cache to the swap cache.
1296  */
1297 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1298 {
1299         struct shmem_inode_info *info;
1300         struct address_space *mapping;
1301         struct inode *inode;
1302         swp_entry_t swap;
1303         pgoff_t index;
1304
1305         VM_BUG_ON_PAGE(PageCompound(page), page);
1306         BUG_ON(!PageLocked(page));
1307         mapping = page->mapping;
1308         index = page->index;
1309         inode = mapping->host;
1310         info = SHMEM_I(inode);
1311         if (info->flags & VM_LOCKED)
1312                 goto redirty;
1313         if (!total_swap_pages)
1314                 goto redirty;
1315
1316         /*
1317          * Our capabilities prevent regular writeback or sync from ever calling
1318          * shmem_writepage; but a stacking filesystem might use ->writepage of
1319          * its underlying filesystem, in which case tmpfs should write out to
1320          * swap only in response to memory pressure, and not for the writeback
1321          * threads or sync.
1322          */
1323         if (!wbc->for_reclaim) {
1324                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1325                 goto redirty;
1326         }
1327
1328         /*
1329          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1330          * value into swapfile.c, the only way we can correctly account for a
1331          * fallocated page arriving here is now to initialize it and write it.
1332          *
1333          * That's okay for a page already fallocated earlier, but if we have
1334          * not yet completed the fallocation, then (a) we want to keep track
1335          * of this page in case we have to undo it, and (b) it may not be a
1336          * good idea to continue anyway, once we're pushing into swap.  So
1337          * reactivate the page, and let shmem_fallocate() quit when too many.
1338          */
1339         if (!PageUptodate(page)) {
1340                 if (inode->i_private) {
1341                         struct shmem_falloc *shmem_falloc;
1342                         spin_lock(&inode->i_lock);
1343                         shmem_falloc = inode->i_private;
1344                         if (shmem_falloc &&
1345                             !shmem_falloc->waitq &&
1346                             index >= shmem_falloc->start &&
1347                             index < shmem_falloc->next)
1348                                 shmem_falloc->nr_unswapped++;
1349                         else
1350                                 shmem_falloc = NULL;
1351                         spin_unlock(&inode->i_lock);
1352                         if (shmem_falloc)
1353                                 goto redirty;
1354                 }
1355                 clear_highpage(page);
1356                 flush_dcache_page(page);
1357                 SetPageUptodate(page);
1358         }
1359
1360         swap = get_swap_page(page);
1361         if (!swap.val)
1362                 goto redirty;
1363
1364         /*
1365          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1366          * if it's not already there.  Do it now before the page is
1367          * moved to swap cache, when its pagelock no longer protects
1368          * the inode from eviction.  But don't unlock the mutex until
1369          * we've incremented swapped, because shmem_unuse_inode() will
1370          * prune a !swapped inode from the swaplist under this mutex.
1371          */
1372         mutex_lock(&shmem_swaplist_mutex);
1373         if (list_empty(&info->swaplist))
1374                 list_add(&info->swaplist, &shmem_swaplist);
1375
1376         if (add_to_swap_cache(page, swap,
1377                         __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN) == 0) {
1378                 spin_lock_irq(&info->lock);
1379                 shmem_recalc_inode(inode);
1380                 info->swapped++;
1381                 spin_unlock_irq(&info->lock);
1382
1383                 swap_shmem_alloc(swap);
1384                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1385
1386                 mutex_unlock(&shmem_swaplist_mutex);
1387                 BUG_ON(page_mapped(page));
1388                 swap_writepage(page, wbc);
1389                 return 0;
1390         }
1391
1392         mutex_unlock(&shmem_swaplist_mutex);
1393         put_swap_page(page, swap);
1394 redirty:
1395         set_page_dirty(page);
1396         if (wbc->for_reclaim)
1397                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1398         unlock_page(page);
1399         return 0;
1400 }
1401
1402 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1403 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1404 {
1405         char buffer[64];
1406
1407         if (!mpol || mpol->mode == MPOL_DEFAULT)
1408                 return;         /* show nothing */
1409
1410         mpol_to_str(buffer, sizeof(buffer), mpol);
1411
1412         seq_printf(seq, ",mpol=%s", buffer);
1413 }
1414
1415 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1416 {
1417         struct mempolicy *mpol = NULL;
1418         if (sbinfo->mpol) {
1419                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1420                 mpol = sbinfo->mpol;
1421                 mpol_get(mpol);
1422                 spin_unlock(&sbinfo->stat_lock);
1423         }
1424         return mpol;
1425 }
1426 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1427 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1428 {
1429 }
1430 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1431 {
1432         return NULL;
1433 }
1434 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1435 #ifndef CONFIG_NUMA
1436 #define vm_policy vm_private_data
1437 #endif
1438
1439 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1440                 struct shmem_inode_info *info, pgoff_t index)
1441 {
1442         /* Create a pseudo vma that just contains the policy */
1443         vma_init(vma, NULL);
1444         /* Bias interleave by inode number to distribute better across nodes */
1445         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1446         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1447 }
1448
1449 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1450 {
1451         /* Drop reference taken by mpol_shared_policy_lookup() */
1452         mpol_cond_put(vma->vm_policy);
1453 }
1454
1455 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1456                         struct shmem_inode_info *info, pgoff_t index)
1457 {
1458         struct vm_area_struct pvma;
1459         struct page *page;
1460         struct vm_fault vmf;
1461
1462         shmem_pseudo_vma_init(&pvma, info, index);
1463         vmf.vma = &pvma;
1464         vmf.address = 0;
1465         page = swap_cluster_readahead(swap, gfp, &vmf);
1466         shmem_pseudo_vma_destroy(&pvma);
1467
1468         return page;
1469 }
1470
1471 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1472                 struct shmem_inode_info *info, pgoff_t index)
1473 {
1474         struct vm_area_struct pvma;
1475         struct address_space *mapping = info->vfs_inode.i_mapping;
1476         pgoff_t hindex;
1477         struct page *page;
1478
1479         hindex = round_down(index, HPAGE_PMD_NR);
1480         if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1481                                                                 XA_PRESENT))
1482                 return NULL;
1483
1484         shmem_pseudo_vma_init(&pvma, info, hindex);
1485         page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1486                         HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1487         shmem_pseudo_vma_destroy(&pvma);
1488         if (page)
1489                 prep_transhuge_page(page);
1490         else
1491                 count_vm_event(THP_FILE_FALLBACK);
1492         return page;
1493 }
1494
1495 static struct page *shmem_alloc_page(gfp_t gfp,
1496                         struct shmem_inode_info *info, pgoff_t index)
1497 {
1498         struct vm_area_struct pvma;
1499         struct page *page;
1500
1501         shmem_pseudo_vma_init(&pvma, info, index);
1502         page = alloc_page_vma(gfp, &pvma, 0);
1503         shmem_pseudo_vma_destroy(&pvma);
1504
1505         return page;
1506 }
1507
1508 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1509                 struct inode *inode,
1510                 pgoff_t index, bool huge)
1511 {
1512         struct shmem_inode_info *info = SHMEM_I(inode);
1513         struct page *page;
1514         int nr;
1515         int err = -ENOSPC;
1516
1517         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1518                 huge = false;
1519         nr = huge ? HPAGE_PMD_NR : 1;
1520
1521         if (!shmem_inode_acct_block(inode, nr))
1522                 goto failed;
1523
1524         if (huge)
1525                 page = shmem_alloc_hugepage(gfp, info, index);
1526         else
1527                 page = shmem_alloc_page(gfp, info, index);
1528         if (page) {
1529                 __SetPageLocked(page);
1530                 __SetPageSwapBacked(page);
1531                 return page;
1532         }
1533
1534         err = -ENOMEM;
1535         shmem_inode_unacct_blocks(inode, nr);
1536 failed:
1537         return ERR_PTR(err);
1538 }
1539
1540 /*
1541  * When a page is moved from swapcache to shmem filecache (either by the
1542  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1543  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1544  * ignorance of the mapping it belongs to.  If that mapping has special
1545  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1546  * we may need to copy to a suitable page before moving to filecache.
1547  *
1548  * In a future release, this may well be extended to respect cpuset and
1549  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1550  * but for now it is a simple matter of zone.
1551  */
1552 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1553 {
1554         return page_zonenum(page) > gfp_zone(gfp);
1555 }
1556
1557 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1558                                 struct shmem_inode_info *info, pgoff_t index)
1559 {
1560         struct page *oldpage, *newpage;
1561         struct address_space *swap_mapping;
1562         swp_entry_t entry;
1563         pgoff_t swap_index;
1564         int error;
1565
1566         oldpage = *pagep;
1567         entry.val = page_private(oldpage);
1568         swap_index = swp_offset(entry);
1569         swap_mapping = page_mapping(oldpage);
1570
1571         /*
1572          * We have arrived here because our zones are constrained, so don't
1573          * limit chance of success by further cpuset and node constraints.
1574          */
1575         gfp &= ~GFP_CONSTRAINT_MASK;
1576         newpage = shmem_alloc_page(gfp, info, index);
1577         if (!newpage)
1578                 return -ENOMEM;
1579
1580         get_page(newpage);
1581         copy_highpage(newpage, oldpage);
1582         flush_dcache_page(newpage);
1583
1584         __SetPageLocked(newpage);
1585         __SetPageSwapBacked(newpage);
1586         SetPageUptodate(newpage);
1587         set_page_private(newpage, entry.val);
1588         SetPageSwapCache(newpage);
1589
1590         /*
1591          * Our caller will very soon move newpage out of swapcache, but it's
1592          * a nice clean interface for us to replace oldpage by newpage there.
1593          */
1594         xa_lock_irq(&swap_mapping->i_pages);
1595         error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
1596         if (!error) {
1597                 mem_cgroup_migrate(oldpage, newpage);
1598                 __inc_lruvec_page_state(newpage, NR_FILE_PAGES);
1599                 __dec_lruvec_page_state(oldpage, NR_FILE_PAGES);
1600         }
1601         xa_unlock_irq(&swap_mapping->i_pages);
1602
1603         if (unlikely(error)) {
1604                 /*
1605                  * Is this possible?  I think not, now that our callers check
1606                  * both PageSwapCache and page_private after getting page lock;
1607                  * but be defensive.  Reverse old to newpage for clear and free.
1608                  */
1609                 oldpage = newpage;
1610         } else {
1611                 lru_cache_add(newpage);
1612                 *pagep = newpage;
1613         }
1614
1615         ClearPageSwapCache(oldpage);
1616         set_page_private(oldpage, 0);
1617
1618         unlock_page(oldpage);
1619         put_page(oldpage);
1620         put_page(oldpage);
1621         return error;
1622 }
1623
1624 /*
1625  * Swap in the page pointed to by *pagep.
1626  * Caller has to make sure that *pagep contains a valid swapped page.
1627  * Returns 0 and the page in pagep if success. On failure, returns the
1628  * the error code and NULL in *pagep.
1629  */
1630 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
1631                              struct page **pagep, enum sgp_type sgp,
1632                              gfp_t gfp, struct vm_area_struct *vma,
1633                              vm_fault_t *fault_type)
1634 {
1635         struct address_space *mapping = inode->i_mapping;
1636         struct shmem_inode_info *info = SHMEM_I(inode);
1637         struct mm_struct *charge_mm = vma ? vma->vm_mm : current->mm;
1638         struct page *page;
1639         swp_entry_t swap;
1640         int error;
1641
1642         VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
1643         swap = radix_to_swp_entry(*pagep);
1644         *pagep = NULL;
1645
1646         /* Look it up and read it in.. */
1647         page = lookup_swap_cache(swap, NULL, 0);
1648         if (!page) {
1649                 /* Or update major stats only when swapin succeeds?? */
1650                 if (fault_type) {
1651                         *fault_type |= VM_FAULT_MAJOR;
1652                         count_vm_event(PGMAJFAULT);
1653                         count_memcg_event_mm(charge_mm, PGMAJFAULT);
1654                 }
1655                 /* Here we actually start the io */
1656                 page = shmem_swapin(swap, gfp, info, index);
1657                 if (!page) {
1658                         error = -ENOMEM;
1659                         goto failed;
1660                 }
1661         }
1662
1663         /* We have to do this with page locked to prevent races */
1664         lock_page(page);
1665         if (!PageSwapCache(page) || page_private(page) != swap.val ||
1666             !shmem_confirm_swap(mapping, index, swap)) {
1667                 error = -EEXIST;
1668                 goto unlock;
1669         }
1670         if (!PageUptodate(page)) {
1671                 error = -EIO;
1672                 goto failed;
1673         }
1674         wait_on_page_writeback(page);
1675
1676         if (shmem_should_replace_page(page, gfp)) {
1677                 error = shmem_replace_page(&page, gfp, info, index);
1678                 if (error)
1679                         goto failed;
1680         }
1681
1682         error = shmem_add_to_page_cache(page, mapping, index,
1683                                         swp_to_radix_entry(swap), gfp,
1684                                         charge_mm);
1685         if (error)
1686                 goto failed;
1687
1688         spin_lock_irq(&info->lock);
1689         info->swapped--;
1690         shmem_recalc_inode(inode);
1691         spin_unlock_irq(&info->lock);
1692
1693         if (sgp == SGP_WRITE)
1694                 mark_page_accessed(page);
1695
1696         delete_from_swap_cache(page);
1697         set_page_dirty(page);
1698         swap_free(swap);
1699
1700         *pagep = page;
1701         return 0;
1702 failed:
1703         if (!shmem_confirm_swap(mapping, index, swap))
1704                 error = -EEXIST;
1705 unlock:
1706         if (page) {
1707                 unlock_page(page);
1708                 put_page(page);
1709         }
1710
1711         return error;
1712 }
1713
1714 /*
1715  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1716  *
1717  * If we allocate a new one we do not mark it dirty. That's up to the
1718  * vm. If we swap it in we mark it dirty since we also free the swap
1719  * entry since a page cannot live in both the swap and page cache.
1720  *
1721  * vmf and fault_type are only supplied by shmem_fault:
1722  * otherwise they are NULL.
1723  */
1724 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1725         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1726         struct vm_area_struct *vma, struct vm_fault *vmf,
1727                         vm_fault_t *fault_type)
1728 {
1729         struct address_space *mapping = inode->i_mapping;
1730         struct shmem_inode_info *info = SHMEM_I(inode);
1731         struct shmem_sb_info *sbinfo;
1732         struct mm_struct *charge_mm;
1733         struct page *page;
1734         enum sgp_type sgp_huge = sgp;
1735         pgoff_t hindex = index;
1736         int error;
1737         int once = 0;
1738         int alloced = 0;
1739
1740         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1741                 return -EFBIG;
1742         if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1743                 sgp = SGP_CACHE;
1744 repeat:
1745         if (sgp <= SGP_CACHE &&
1746             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1747                 return -EINVAL;
1748         }
1749
1750         sbinfo = SHMEM_SB(inode->i_sb);
1751         charge_mm = vma ? vma->vm_mm : current->mm;
1752
1753         page = find_lock_entry(mapping, index);
1754         if (xa_is_value(page)) {
1755                 error = shmem_swapin_page(inode, index, &page,
1756                                           sgp, gfp, vma, fault_type);
1757                 if (error == -EEXIST)
1758                         goto repeat;
1759
1760                 *pagep = page;
1761                 return error;
1762         }
1763
1764         if (page && sgp == SGP_WRITE)
1765                 mark_page_accessed(page);
1766
1767         /* fallocated page? */
1768         if (page && !PageUptodate(page)) {
1769                 if (sgp != SGP_READ)
1770                         goto clear;
1771                 unlock_page(page);
1772                 put_page(page);
1773                 page = NULL;
1774         }
1775         if (page || sgp == SGP_READ) {
1776                 *pagep = page;
1777                 return 0;
1778         }
1779
1780         /*
1781          * Fast cache lookup did not find it:
1782          * bring it back from swap or allocate.
1783          */
1784
1785         if (vma && userfaultfd_missing(vma)) {
1786                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1787                 return 0;
1788         }
1789
1790         /* shmem_symlink() */
1791         if (mapping->a_ops != &shmem_aops)
1792                 goto alloc_nohuge;
1793         if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1794                 goto alloc_nohuge;
1795         if (shmem_huge == SHMEM_HUGE_FORCE)
1796                 goto alloc_huge;
1797         switch (sbinfo->huge) {
1798         case SHMEM_HUGE_NEVER:
1799                 goto alloc_nohuge;
1800         case SHMEM_HUGE_WITHIN_SIZE: {
1801                 loff_t i_size;
1802                 pgoff_t off;
1803
1804                 off = round_up(index, HPAGE_PMD_NR);
1805                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
1806                 if (i_size >= HPAGE_PMD_SIZE &&
1807                     i_size >> PAGE_SHIFT >= off)
1808                         goto alloc_huge;
1809
1810                 fallthrough;
1811         }
1812         case SHMEM_HUGE_ADVISE:
1813                 if (sgp_huge == SGP_HUGE)
1814                         goto alloc_huge;
1815                 /* TODO: implement fadvise() hints */
1816                 goto alloc_nohuge;
1817         }
1818
1819 alloc_huge:
1820         page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1821         if (IS_ERR(page)) {
1822 alloc_nohuge:
1823                 page = shmem_alloc_and_acct_page(gfp, inode,
1824                                                  index, false);
1825         }
1826         if (IS_ERR(page)) {
1827                 int retry = 5;
1828
1829                 error = PTR_ERR(page);
1830                 page = NULL;
1831                 if (error != -ENOSPC)
1832                         goto unlock;
1833                 /*
1834                  * Try to reclaim some space by splitting a huge page
1835                  * beyond i_size on the filesystem.
1836                  */
1837                 while (retry--) {
1838                         int ret;
1839
1840                         ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1841                         if (ret == SHRINK_STOP)
1842                                 break;
1843                         if (ret)
1844                                 goto alloc_nohuge;
1845                 }
1846                 goto unlock;
1847         }
1848
1849         if (PageTransHuge(page))
1850                 hindex = round_down(index, HPAGE_PMD_NR);
1851         else
1852                 hindex = index;
1853
1854         if (sgp == SGP_WRITE)
1855                 __SetPageReferenced(page);
1856
1857         error = shmem_add_to_page_cache(page, mapping, hindex,
1858                                         NULL, gfp & GFP_RECLAIM_MASK,
1859                                         charge_mm);
1860         if (error)
1861                 goto unacct;
1862         lru_cache_add(page);
1863
1864         spin_lock_irq(&info->lock);
1865         info->alloced += compound_nr(page);
1866         inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1867         shmem_recalc_inode(inode);
1868         spin_unlock_irq(&info->lock);
1869         alloced = true;
1870
1871         if (PageTransHuge(page) &&
1872             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1873                         hindex + HPAGE_PMD_NR - 1) {
1874                 /*
1875                  * Part of the huge page is beyond i_size: subject
1876                  * to shrink under memory pressure.
1877                  */
1878                 spin_lock(&sbinfo->shrinklist_lock);
1879                 /*
1880                  * _careful to defend against unlocked access to
1881                  * ->shrink_list in shmem_unused_huge_shrink()
1882                  */
1883                 if (list_empty_careful(&info->shrinklist)) {
1884                         list_add_tail(&info->shrinklist,
1885                                       &sbinfo->shrinklist);
1886                         sbinfo->shrinklist_len++;
1887                 }
1888                 spin_unlock(&sbinfo->shrinklist_lock);
1889         }
1890
1891         /*
1892          * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1893          */
1894         if (sgp == SGP_FALLOC)
1895                 sgp = SGP_WRITE;
1896 clear:
1897         /*
1898          * Let SGP_WRITE caller clear ends if write does not fill page;
1899          * but SGP_FALLOC on a page fallocated earlier must initialize
1900          * it now, lest undo on failure cancel our earlier guarantee.
1901          */
1902         if (sgp != SGP_WRITE && !PageUptodate(page)) {
1903                 struct page *head = compound_head(page);
1904                 int i;
1905
1906                 for (i = 0; i < compound_nr(head); i++) {
1907                         clear_highpage(head + i);
1908                         flush_dcache_page(head + i);
1909                 }
1910                 SetPageUptodate(head);
1911         }
1912
1913         /* Perhaps the file has been truncated since we checked */
1914         if (sgp <= SGP_CACHE &&
1915             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1916                 if (alloced) {
1917                         ClearPageDirty(page);
1918                         delete_from_page_cache(page);
1919                         spin_lock_irq(&info->lock);
1920                         shmem_recalc_inode(inode);
1921                         spin_unlock_irq(&info->lock);
1922                 }
1923                 error = -EINVAL;
1924                 goto unlock;
1925         }
1926         *pagep = page + index - hindex;
1927         return 0;
1928
1929         /*
1930          * Error recovery.
1931          */
1932 unacct:
1933         shmem_inode_unacct_blocks(inode, compound_nr(page));
1934
1935         if (PageTransHuge(page)) {
1936                 unlock_page(page);
1937                 put_page(page);
1938                 goto alloc_nohuge;
1939         }
1940 unlock:
1941         if (page) {
1942                 unlock_page(page);
1943                 put_page(page);
1944         }
1945         if (error == -ENOSPC && !once++) {
1946                 spin_lock_irq(&info->lock);
1947                 shmem_recalc_inode(inode);
1948                 spin_unlock_irq(&info->lock);
1949                 goto repeat;
1950         }
1951         if (error == -EEXIST)
1952                 goto repeat;
1953         return error;
1954 }
1955
1956 /*
1957  * This is like autoremove_wake_function, but it removes the wait queue
1958  * entry unconditionally - even if something else had already woken the
1959  * target.
1960  */
1961 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1962 {
1963         int ret = default_wake_function(wait, mode, sync, key);
1964         list_del_init(&wait->entry);
1965         return ret;
1966 }
1967
1968 static vm_fault_t shmem_fault(struct vm_fault *vmf)
1969 {
1970         struct vm_area_struct *vma = vmf->vma;
1971         struct inode *inode = file_inode(vma->vm_file);
1972         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1973         enum sgp_type sgp;
1974         int err;
1975         vm_fault_t ret = VM_FAULT_LOCKED;
1976
1977         /*
1978          * Trinity finds that probing a hole which tmpfs is punching can
1979          * prevent the hole-punch from ever completing: which in turn
1980          * locks writers out with its hold on i_mutex.  So refrain from
1981          * faulting pages into the hole while it's being punched.  Although
1982          * shmem_undo_range() does remove the additions, it may be unable to
1983          * keep up, as each new page needs its own unmap_mapping_range() call,
1984          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1985          *
1986          * It does not matter if we sometimes reach this check just before the
1987          * hole-punch begins, so that one fault then races with the punch:
1988          * we just need to make racing faults a rare case.
1989          *
1990          * The implementation below would be much simpler if we just used a
1991          * standard mutex or completion: but we cannot take i_mutex in fault,
1992          * and bloating every shmem inode for this unlikely case would be sad.
1993          */
1994         if (unlikely(inode->i_private)) {
1995                 struct shmem_falloc *shmem_falloc;
1996
1997                 spin_lock(&inode->i_lock);
1998                 shmem_falloc = inode->i_private;
1999                 if (shmem_falloc &&
2000                     shmem_falloc->waitq &&
2001                     vmf->pgoff >= shmem_falloc->start &&
2002                     vmf->pgoff < shmem_falloc->next) {
2003                         struct file *fpin;
2004                         wait_queue_head_t *shmem_falloc_waitq;
2005                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2006
2007                         ret = VM_FAULT_NOPAGE;
2008                         fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2009                         if (fpin)
2010                                 ret = VM_FAULT_RETRY;
2011
2012                         shmem_falloc_waitq = shmem_falloc->waitq;
2013                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2014                                         TASK_UNINTERRUPTIBLE);
2015                         spin_unlock(&inode->i_lock);
2016                         schedule();
2017
2018                         /*
2019                          * shmem_falloc_waitq points into the shmem_fallocate()
2020                          * stack of the hole-punching task: shmem_falloc_waitq
2021                          * is usually invalid by the time we reach here, but
2022                          * finish_wait() does not dereference it in that case;
2023                          * though i_lock needed lest racing with wake_up_all().
2024                          */
2025                         spin_lock(&inode->i_lock);
2026                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2027                         spin_unlock(&inode->i_lock);
2028
2029                         if (fpin)
2030                                 fput(fpin);
2031                         return ret;
2032                 }
2033                 spin_unlock(&inode->i_lock);
2034         }
2035
2036         sgp = SGP_CACHE;
2037
2038         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2039             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2040                 sgp = SGP_NOHUGE;
2041         else if (vma->vm_flags & VM_HUGEPAGE)
2042                 sgp = SGP_HUGE;
2043
2044         err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2045                                   gfp, vma, vmf, &ret);
2046         if (err)
2047                 return vmf_error(err);
2048         return ret;
2049 }
2050
2051 unsigned long shmem_get_unmapped_area(struct file *file,
2052                                       unsigned long uaddr, unsigned long len,
2053                                       unsigned long pgoff, unsigned long flags)
2054 {
2055         unsigned long (*get_area)(struct file *,
2056                 unsigned long, unsigned long, unsigned long, unsigned long);
2057         unsigned long addr;
2058         unsigned long offset;
2059         unsigned long inflated_len;
2060         unsigned long inflated_addr;
2061         unsigned long inflated_offset;
2062
2063         if (len > TASK_SIZE)
2064                 return -ENOMEM;
2065
2066         get_area = current->mm->get_unmapped_area;
2067         addr = get_area(file, uaddr, len, pgoff, flags);
2068
2069         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2070                 return addr;
2071         if (IS_ERR_VALUE(addr))
2072                 return addr;
2073         if (addr & ~PAGE_MASK)
2074                 return addr;
2075         if (addr > TASK_SIZE - len)
2076                 return addr;
2077
2078         if (shmem_huge == SHMEM_HUGE_DENY)
2079                 return addr;
2080         if (len < HPAGE_PMD_SIZE)
2081                 return addr;
2082         if (flags & MAP_FIXED)
2083                 return addr;
2084         /*
2085          * Our priority is to support MAP_SHARED mapped hugely;
2086          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2087          * But if caller specified an address hint and we allocated area there
2088          * successfully, respect that as before.
2089          */
2090         if (uaddr == addr)
2091                 return addr;
2092
2093         if (shmem_huge != SHMEM_HUGE_FORCE) {
2094                 struct super_block *sb;
2095
2096                 if (file) {
2097                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2098                         sb = file_inode(file)->i_sb;
2099                 } else {
2100                         /*
2101                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2102                          * for "/dev/zero", to create a shared anonymous object.
2103                          */
2104                         if (IS_ERR(shm_mnt))
2105                                 return addr;
2106                         sb = shm_mnt->mnt_sb;
2107                 }
2108                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2109                         return addr;
2110         }
2111
2112         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2113         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2114                 return addr;
2115         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2116                 return addr;
2117
2118         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2119         if (inflated_len > TASK_SIZE)
2120                 return addr;
2121         if (inflated_len < len)
2122                 return addr;
2123
2124         inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2125         if (IS_ERR_VALUE(inflated_addr))
2126                 return addr;
2127         if (inflated_addr & ~PAGE_MASK)
2128                 return addr;
2129
2130         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2131         inflated_addr += offset - inflated_offset;
2132         if (inflated_offset > offset)
2133                 inflated_addr += HPAGE_PMD_SIZE;
2134
2135         if (inflated_addr > TASK_SIZE - len)
2136                 return addr;
2137         return inflated_addr;
2138 }
2139
2140 #ifdef CONFIG_NUMA
2141 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2142 {
2143         struct inode *inode = file_inode(vma->vm_file);
2144         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2145 }
2146
2147 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2148                                           unsigned long addr)
2149 {
2150         struct inode *inode = file_inode(vma->vm_file);
2151         pgoff_t index;
2152
2153         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2154         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2155 }
2156 #endif
2157
2158 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2159 {
2160         struct inode *inode = file_inode(file);
2161         struct shmem_inode_info *info = SHMEM_I(inode);
2162         int retval = -ENOMEM;
2163
2164         /*
2165          * What serializes the accesses to info->flags?
2166          * ipc_lock_object() when called from shmctl_do_lock(),
2167          * no serialization needed when called from shm_destroy().
2168          */
2169         if (lock && !(info->flags & VM_LOCKED)) {
2170                 if (!user_shm_lock(inode->i_size, user))
2171                         goto out_nomem;
2172                 info->flags |= VM_LOCKED;
2173                 mapping_set_unevictable(file->f_mapping);
2174         }
2175         if (!lock && (info->flags & VM_LOCKED) && user) {
2176                 user_shm_unlock(inode->i_size, user);
2177                 info->flags &= ~VM_LOCKED;
2178                 mapping_clear_unevictable(file->f_mapping);
2179         }
2180         retval = 0;
2181
2182 out_nomem:
2183         return retval;
2184 }
2185
2186 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2187 {
2188         struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2189
2190         if (info->seals & F_SEAL_FUTURE_WRITE) {
2191                 /*
2192                  * New PROT_WRITE and MAP_SHARED mmaps are not allowed when
2193                  * "future write" seal active.
2194                  */
2195                 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
2196                         return -EPERM;
2197
2198                 /*
2199                  * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as
2200                  * MAP_SHARED and read-only, take care to not allow mprotect to
2201                  * revert protections on such mappings. Do this only for shared
2202                  * mappings. For private mappings, don't need to mask
2203                  * VM_MAYWRITE as we still want them to be COW-writable.
2204                  */
2205                 if (vma->vm_flags & VM_SHARED)
2206                         vma->vm_flags &= ~(VM_MAYWRITE);
2207         }
2208
2209         file_accessed(file);
2210         vma->vm_ops = &shmem_vm_ops;
2211         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2212                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2213                         (vma->vm_end & HPAGE_PMD_MASK)) {
2214                 khugepaged_enter(vma, vma->vm_flags);
2215         }
2216         return 0;
2217 }
2218
2219 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2220                                      umode_t mode, dev_t dev, unsigned long flags)
2221 {
2222         struct inode *inode;
2223         struct shmem_inode_info *info;
2224         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2225
2226         if (shmem_reserve_inode(sb))
2227                 return NULL;
2228
2229         inode = new_inode(sb);
2230         if (inode) {
2231                 inode->i_ino = get_next_ino();
2232                 inode_init_owner(inode, dir, mode);
2233                 inode->i_blocks = 0;
2234                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2235                 inode->i_generation = prandom_u32();
2236                 info = SHMEM_I(inode);
2237                 memset(info, 0, (char *)inode - (char *)info);
2238                 spin_lock_init(&info->lock);
2239                 atomic_set(&info->stop_eviction, 0);
2240                 info->seals = F_SEAL_SEAL;
2241                 info->flags = flags & VM_NORESERVE;
2242                 INIT_LIST_HEAD(&info->shrinklist);
2243                 INIT_LIST_HEAD(&info->swaplist);
2244                 simple_xattrs_init(&info->xattrs);
2245                 cache_no_acl(inode);
2246
2247                 switch (mode & S_IFMT) {
2248                 default:
2249                         inode->i_op = &shmem_special_inode_operations;
2250                         init_special_inode(inode, mode, dev);
2251                         break;
2252                 case S_IFREG:
2253                         inode->i_mapping->a_ops = &shmem_aops;
2254                         inode->i_op = &shmem_inode_operations;
2255                         inode->i_fop = &shmem_file_operations;
2256                         mpol_shared_policy_init(&info->policy,
2257                                                  shmem_get_sbmpol(sbinfo));
2258                         break;
2259                 case S_IFDIR:
2260                         inc_nlink(inode);
2261                         /* Some things misbehave if size == 0 on a directory */
2262                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2263                         inode->i_op = &shmem_dir_inode_operations;
2264                         inode->i_fop = &simple_dir_operations;
2265                         break;
2266                 case S_IFLNK:
2267                         /*
2268                          * Must not load anything in the rbtree,
2269                          * mpol_free_shared_policy will not be called.
2270                          */
2271                         mpol_shared_policy_init(&info->policy, NULL);
2272                         break;
2273                 }
2274
2275                 lockdep_annotate_inode_mutex_key(inode);
2276         } else
2277                 shmem_free_inode(sb);
2278         return inode;
2279 }
2280
2281 bool shmem_mapping(struct address_space *mapping)
2282 {
2283         return mapping->a_ops == &shmem_aops;
2284 }
2285
2286 static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2287                                   pmd_t *dst_pmd,
2288                                   struct vm_area_struct *dst_vma,
2289                                   unsigned long dst_addr,
2290                                   unsigned long src_addr,
2291                                   bool zeropage,
2292                                   struct page **pagep)
2293 {
2294         struct inode *inode = file_inode(dst_vma->vm_file);
2295         struct shmem_inode_info *info = SHMEM_I(inode);
2296         struct address_space *mapping = inode->i_mapping;
2297         gfp_t gfp = mapping_gfp_mask(mapping);
2298         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2299         spinlock_t *ptl;
2300         void *page_kaddr;
2301         struct page *page;
2302         pte_t _dst_pte, *dst_pte;
2303         int ret;
2304         pgoff_t offset, max_off;
2305
2306         ret = -ENOMEM;
2307         if (!shmem_inode_acct_block(inode, 1))
2308                 goto out;
2309
2310         if (!*pagep) {
2311                 page = shmem_alloc_page(gfp, info, pgoff);
2312                 if (!page)
2313                         goto out_unacct_blocks;
2314
2315                 if (!zeropage) {        /* mcopy_atomic */
2316                         page_kaddr = kmap_atomic(page);
2317                         ret = copy_from_user(page_kaddr,
2318                                              (const void __user *)src_addr,
2319                                              PAGE_SIZE);
2320                         kunmap_atomic(page_kaddr);
2321
2322                         /* fallback to copy_from_user outside mmap_lock */
2323                         if (unlikely(ret)) {
2324                                 *pagep = page;
2325                                 shmem_inode_unacct_blocks(inode, 1);
2326                                 /* don't free the page */
2327                                 return -ENOENT;
2328                         }
2329                 } else {                /* mfill_zeropage_atomic */
2330                         clear_highpage(page);
2331                 }
2332         } else {
2333                 page = *pagep;
2334                 *pagep = NULL;
2335         }
2336
2337         VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2338         __SetPageLocked(page);
2339         __SetPageSwapBacked(page);
2340         __SetPageUptodate(page);
2341
2342         ret = -EFAULT;
2343         offset = linear_page_index(dst_vma, dst_addr);
2344         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2345         if (unlikely(offset >= max_off))
2346                 goto out_release;
2347
2348         ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2349                                       gfp & GFP_RECLAIM_MASK, dst_mm);
2350         if (ret)
2351                 goto out_release;
2352
2353         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2354         if (dst_vma->vm_flags & VM_WRITE)
2355                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2356         else {
2357                 /*
2358                  * We don't set the pte dirty if the vma has no
2359                  * VM_WRITE permission, so mark the page dirty or it
2360                  * could be freed from under us. We could do it
2361                  * unconditionally before unlock_page(), but doing it
2362                  * only if VM_WRITE is not set is faster.
2363                  */
2364                 set_page_dirty(page);
2365         }
2366
2367         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2368
2369         ret = -EFAULT;
2370         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2371         if (unlikely(offset >= max_off))
2372                 goto out_release_unlock;
2373
2374         ret = -EEXIST;
2375         if (!pte_none(*dst_pte))
2376                 goto out_release_unlock;
2377
2378         lru_cache_add(page);
2379
2380         spin_lock_irq(&info->lock);
2381         info->alloced++;
2382         inode->i_blocks += BLOCKS_PER_PAGE;
2383         shmem_recalc_inode(inode);
2384         spin_unlock_irq(&info->lock);
2385
2386         inc_mm_counter(dst_mm, mm_counter_file(page));
2387         page_add_file_rmap(page, false);
2388         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2389
2390         /* No need to invalidate - it was non-present before */
2391         update_mmu_cache(dst_vma, dst_addr, dst_pte);
2392         pte_unmap_unlock(dst_pte, ptl);
2393         unlock_page(page);
2394         ret = 0;
2395 out:
2396         return ret;
2397 out_release_unlock:
2398         pte_unmap_unlock(dst_pte, ptl);
2399         ClearPageDirty(page);
2400         delete_from_page_cache(page);
2401 out_release:
2402         unlock_page(page);
2403         put_page(page);
2404 out_unacct_blocks:
2405         shmem_inode_unacct_blocks(inode, 1);
2406         goto out;
2407 }
2408
2409 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2410                            pmd_t *dst_pmd,
2411                            struct vm_area_struct *dst_vma,
2412                            unsigned long dst_addr,
2413                            unsigned long src_addr,
2414                            struct page **pagep)
2415 {
2416         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2417                                       dst_addr, src_addr, false, pagep);
2418 }
2419
2420 int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2421                              pmd_t *dst_pmd,
2422                              struct vm_area_struct *dst_vma,
2423                              unsigned long dst_addr)
2424 {
2425         struct page *page = NULL;
2426
2427         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2428                                       dst_addr, 0, true, &page);
2429 }
2430
2431 #ifdef CONFIG_TMPFS
2432 static const struct inode_operations shmem_symlink_inode_operations;
2433 static const struct inode_operations shmem_short_symlink_operations;
2434
2435 #ifdef CONFIG_TMPFS_XATTR
2436 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2437 #else
2438 #define shmem_initxattrs NULL
2439 #endif
2440
2441 static int
2442 shmem_write_begin(struct file *file, struct address_space *mapping,
2443                         loff_t pos, unsigned len, unsigned flags,
2444                         struct page **pagep, void **fsdata)
2445 {
2446         struct inode *inode = mapping->host;
2447         struct shmem_inode_info *info = SHMEM_I(inode);
2448         pgoff_t index = pos >> PAGE_SHIFT;
2449
2450         /* i_mutex is held by caller */
2451         if (unlikely(info->seals & (F_SEAL_GROW |
2452                                    F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2453                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2454                         return -EPERM;
2455                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2456                         return -EPERM;
2457         }
2458
2459         return shmem_getpage(inode, index, pagep, SGP_WRITE);
2460 }
2461
2462 static int
2463 shmem_write_end(struct file *file, struct address_space *mapping,
2464                         loff_t pos, unsigned len, unsigned copied,
2465                         struct page *page, void *fsdata)
2466 {
2467         struct inode *inode = mapping->host;
2468
2469         if (pos + copied > inode->i_size)
2470                 i_size_write(inode, pos + copied);
2471
2472         if (!PageUptodate(page)) {
2473                 struct page *head = compound_head(page);
2474                 if (PageTransCompound(page)) {
2475                         int i;
2476
2477                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2478                                 if (head + i == page)
2479                                         continue;
2480                                 clear_highpage(head + i);
2481                                 flush_dcache_page(head + i);
2482                         }
2483                 }
2484                 if (copied < PAGE_SIZE) {
2485                         unsigned from = pos & (PAGE_SIZE - 1);
2486                         zero_user_segments(page, 0, from,
2487                                         from + copied, PAGE_SIZE);
2488                 }
2489                 SetPageUptodate(head);
2490         }
2491         set_page_dirty(page);
2492         unlock_page(page);
2493         put_page(page);
2494
2495         return copied;
2496 }
2497
2498 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2499 {
2500         struct file *file = iocb->ki_filp;
2501         struct inode *inode = file_inode(file);
2502         struct address_space *mapping = inode->i_mapping;
2503         pgoff_t index;
2504         unsigned long offset;
2505         enum sgp_type sgp = SGP_READ;
2506         int error = 0;
2507         ssize_t retval = 0;
2508         loff_t *ppos = &iocb->ki_pos;
2509
2510         /*
2511          * Might this read be for a stacking filesystem?  Then when reading
2512          * holes of a sparse file, we actually need to allocate those pages,
2513          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2514          */
2515         if (!iter_is_iovec(to))
2516                 sgp = SGP_CACHE;
2517
2518         index = *ppos >> PAGE_SHIFT;
2519         offset = *ppos & ~PAGE_MASK;
2520
2521         for (;;) {
2522                 struct page *page = NULL;
2523                 pgoff_t end_index;
2524                 unsigned long nr, ret;
2525                 loff_t i_size = i_size_read(inode);
2526
2527                 end_index = i_size >> PAGE_SHIFT;
2528                 if (index > end_index)
2529                         break;
2530                 if (index == end_index) {
2531                         nr = i_size & ~PAGE_MASK;
2532                         if (nr <= offset)
2533                                 break;
2534                 }
2535
2536                 error = shmem_getpage(inode, index, &page, sgp);
2537                 if (error) {
2538                         if (error == -EINVAL)
2539                                 error = 0;
2540                         break;
2541                 }
2542                 if (page) {
2543                         if (sgp == SGP_CACHE)
2544                                 set_page_dirty(page);
2545                         unlock_page(page);
2546                 }
2547
2548                 /*
2549                  * We must evaluate after, since reads (unlike writes)
2550                  * are called without i_mutex protection against truncate
2551                  */
2552                 nr = PAGE_SIZE;
2553                 i_size = i_size_read(inode);
2554                 end_index = i_size >> PAGE_SHIFT;
2555                 if (index == end_index) {
2556                         nr = i_size & ~PAGE_MASK;
2557                         if (nr <= offset) {
2558                                 if (page)
2559                                         put_page(page);
2560                                 break;
2561                         }
2562                 }
2563                 nr -= offset;
2564
2565                 if (page) {
2566                         /*
2567                          * If users can be writing to this page using arbitrary
2568                          * virtual addresses, take care about potential aliasing
2569                          * before reading the page on the kernel side.
2570                          */
2571                         if (mapping_writably_mapped(mapping))
2572                                 flush_dcache_page(page);
2573                         /*
2574                          * Mark the page accessed if we read the beginning.
2575                          */
2576                         if (!offset)
2577                                 mark_page_accessed(page);
2578                 } else {
2579                         page = ZERO_PAGE(0);
2580                         get_page(page);
2581                 }
2582
2583                 /*
2584                  * Ok, we have the page, and it's up-to-date, so
2585                  * now we can copy it to user space...
2586                  */
2587                 ret = copy_page_to_iter(page, offset, nr, to);
2588                 retval += ret;
2589                 offset += ret;
2590                 index += offset >> PAGE_SHIFT;
2591                 offset &= ~PAGE_MASK;
2592
2593                 put_page(page);
2594                 if (!iov_iter_count(to))
2595                         break;
2596                 if (ret < nr) {
2597                         error = -EFAULT;
2598                         break;
2599                 }
2600                 cond_resched();
2601         }
2602
2603         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2604         file_accessed(file);
2605         return retval ? retval : error;
2606 }
2607
2608 /*
2609  * llseek SEEK_DATA or SEEK_HOLE through the page cache.
2610  */
2611 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2612                                     pgoff_t index, pgoff_t end, int whence)
2613 {
2614         struct page *page;
2615         struct pagevec pvec;
2616         pgoff_t indices[PAGEVEC_SIZE];
2617         bool done = false;
2618         int i;
2619
2620         pagevec_init(&pvec);
2621         pvec.nr = 1;            /* start small: we may be there already */
2622         while (!done) {
2623                 pvec.nr = find_get_entries(mapping, index,
2624                                         pvec.nr, pvec.pages, indices);
2625                 if (!pvec.nr) {
2626                         if (whence == SEEK_DATA)
2627                                 index = end;
2628                         break;
2629                 }
2630                 for (i = 0; i < pvec.nr; i++, index++) {
2631                         if (index < indices[i]) {
2632                                 if (whence == SEEK_HOLE) {
2633                                         done = true;
2634                                         break;
2635                                 }
2636                                 index = indices[i];
2637                         }
2638                         page = pvec.pages[i];
2639                         if (page && !xa_is_value(page)) {
2640                                 if (!PageUptodate(page))
2641                                         page = NULL;
2642                         }
2643                         if (index >= end ||
2644                             (page && whence == SEEK_DATA) ||
2645                             (!page && whence == SEEK_HOLE)) {
2646                                 done = true;
2647                                 break;
2648                         }
2649                 }
2650                 pagevec_remove_exceptionals(&pvec);
2651                 pagevec_release(&pvec);
2652                 pvec.nr = PAGEVEC_SIZE;
2653                 cond_resched();
2654         }
2655         return index;
2656 }
2657
2658 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2659 {
2660         struct address_space *mapping = file->f_mapping;
2661         struct inode *inode = mapping->host;
2662         pgoff_t start, end;
2663         loff_t new_offset;
2664
2665         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2666                 return generic_file_llseek_size(file, offset, whence,
2667                                         MAX_LFS_FILESIZE, i_size_read(inode));
2668         inode_lock(inode);
2669         /* We're holding i_mutex so we can access i_size directly */
2670
2671         if (offset < 0 || offset >= inode->i_size)
2672                 offset = -ENXIO;
2673         else {
2674                 start = offset >> PAGE_SHIFT;
2675                 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2676                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2677                 new_offset <<= PAGE_SHIFT;
2678                 if (new_offset > offset) {
2679                         if (new_offset < inode->i_size)
2680                                 offset = new_offset;
2681                         else if (whence == SEEK_DATA)
2682                                 offset = -ENXIO;
2683                         else
2684                                 offset = inode->i_size;
2685                 }
2686         }
2687
2688         if (offset >= 0)
2689                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2690         inode_unlock(inode);
2691         return offset;
2692 }
2693
2694 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2695                                                          loff_t len)
2696 {
2697         struct inode *inode = file_inode(file);
2698         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2699         struct shmem_inode_info *info = SHMEM_I(inode);
2700         struct shmem_falloc shmem_falloc;
2701         pgoff_t start, index, end;
2702         int error;
2703
2704         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2705                 return -EOPNOTSUPP;
2706
2707         inode_lock(inode);
2708
2709         if (mode & FALLOC_FL_PUNCH_HOLE) {
2710                 struct address_space *mapping = file->f_mapping;
2711                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2712                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2713                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2714
2715                 /* protected by i_mutex */
2716                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2717                         error = -EPERM;
2718                         goto out;
2719                 }
2720
2721                 shmem_falloc.waitq = &shmem_falloc_waitq;
2722                 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2723                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2724                 spin_lock(&inode->i_lock);
2725                 inode->i_private = &shmem_falloc;
2726                 spin_unlock(&inode->i_lock);
2727
2728                 if ((u64)unmap_end > (u64)unmap_start)
2729                         unmap_mapping_range(mapping, unmap_start,
2730                                             1 + unmap_end - unmap_start, 0);
2731                 shmem_truncate_range(inode, offset, offset + len - 1);
2732                 /* No need to unmap again: hole-punching leaves COWed pages */
2733
2734                 spin_lock(&inode->i_lock);
2735                 inode->i_private = NULL;
2736                 wake_up_all(&shmem_falloc_waitq);
2737                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2738                 spin_unlock(&inode->i_lock);
2739                 error = 0;
2740                 goto out;
2741         }
2742
2743         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2744         error = inode_newsize_ok(inode, offset + len);
2745         if (error)
2746                 goto out;
2747
2748         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2749                 error = -EPERM;
2750                 goto out;
2751         }
2752
2753         start = offset >> PAGE_SHIFT;
2754         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2755         /* Try to avoid a swapstorm if len is impossible to satisfy */
2756         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2757                 error = -ENOSPC;
2758                 goto out;
2759         }
2760
2761         shmem_falloc.waitq = NULL;
2762         shmem_falloc.start = start;
2763         shmem_falloc.next  = start;
2764         shmem_falloc.nr_falloced = 0;
2765         shmem_falloc.nr_unswapped = 0;
2766         spin_lock(&inode->i_lock);
2767         inode->i_private = &shmem_falloc;
2768         spin_unlock(&inode->i_lock);
2769
2770         for (index = start; index < end; index++) {
2771                 struct page *page;
2772
2773                 /*
2774                  * Good, the fallocate(2) manpage permits EINTR: we may have
2775                  * been interrupted because we are using up too much memory.
2776                  */
2777                 if (signal_pending(current))
2778                         error = -EINTR;
2779                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2780                         error = -ENOMEM;
2781                 else
2782                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2783                 if (error) {
2784                         /* Remove the !PageUptodate pages we added */
2785                         if (index > start) {
2786                                 shmem_undo_range(inode,
2787                                     (loff_t)start << PAGE_SHIFT,
2788                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2789                         }
2790                         goto undone;
2791                 }
2792
2793                 /*
2794                  * Inform shmem_writepage() how far we have reached.
2795                  * No need for lock or barrier: we have the page lock.
2796                  */
2797                 shmem_falloc.next++;
2798                 if (!PageUptodate(page))
2799                         shmem_falloc.nr_falloced++;
2800
2801                 /*
2802                  * If !PageUptodate, leave it that way so that freeable pages
2803                  * can be recognized if we need to rollback on error later.
2804                  * But set_page_dirty so that memory pressure will swap rather
2805                  * than free the pages we are allocating (and SGP_CACHE pages
2806                  * might still be clean: we now need to mark those dirty too).
2807                  */
2808                 set_page_dirty(page);
2809                 unlock_page(page);
2810                 put_page(page);
2811                 cond_resched();
2812         }
2813
2814         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2815                 i_size_write(inode, offset + len);
2816         inode->i_ctime = current_time(inode);
2817 undone:
2818         spin_lock(&inode->i_lock);
2819         inode->i_private = NULL;
2820         spin_unlock(&inode->i_lock);
2821 out:
2822         inode_unlock(inode);
2823         return error;
2824 }
2825
2826 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2827 {
2828         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2829
2830         buf->f_type = TMPFS_MAGIC;
2831         buf->f_bsize = PAGE_SIZE;
2832         buf->f_namelen = NAME_MAX;
2833         if (sbinfo->max_blocks) {
2834                 buf->f_blocks = sbinfo->max_blocks;
2835                 buf->f_bavail =
2836                 buf->f_bfree  = sbinfo->max_blocks -
2837                                 percpu_counter_sum(&sbinfo->used_blocks);
2838         }
2839         if (sbinfo->max_inodes) {
2840                 buf->f_files = sbinfo->max_inodes;
2841                 buf->f_ffree = sbinfo->free_inodes;
2842         }
2843         /* else leave those fields 0 like simple_statfs */
2844         return 0;
2845 }
2846
2847 /*
2848  * File creation. Allocate an inode, and we're done..
2849  */
2850 static int
2851 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2852 {
2853         struct inode *inode;
2854         int error = -ENOSPC;
2855
2856         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2857         if (inode) {
2858                 error = simple_acl_create(dir, inode);
2859                 if (error)
2860                         goto out_iput;
2861                 error = security_inode_init_security(inode, dir,
2862                                                      &dentry->d_name,
2863                                                      shmem_initxattrs, NULL);
2864                 if (error && error != -EOPNOTSUPP)
2865                         goto out_iput;
2866
2867                 error = 0;
2868                 dir->i_size += BOGO_DIRENT_SIZE;
2869                 dir->i_ctime = dir->i_mtime = current_time(dir);
2870                 d_instantiate(dentry, inode);
2871                 dget(dentry); /* Extra count - pin the dentry in core */
2872         }
2873         return error;
2874 out_iput:
2875         iput(inode);
2876         return error;
2877 }
2878
2879 static int
2880 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2881 {
2882         struct inode *inode;
2883         int error = -ENOSPC;
2884
2885         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2886         if (inode) {
2887                 error = security_inode_init_security(inode, dir,
2888                                                      NULL,
2889                                                      shmem_initxattrs, NULL);
2890                 if (error && error != -EOPNOTSUPP)
2891                         goto out_iput;
2892                 error = simple_acl_create(dir, inode);
2893                 if (error)
2894                         goto out_iput;
2895                 d_tmpfile(dentry, inode);
2896         }
2897         return error;
2898 out_iput:
2899         iput(inode);
2900         return error;
2901 }
2902
2903 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2904 {
2905         int error;
2906
2907         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2908                 return error;
2909         inc_nlink(dir);
2910         return 0;
2911 }
2912
2913 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2914                 bool excl)
2915 {
2916         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2917 }
2918
2919 /*
2920  * Link a file..
2921  */
2922 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2923 {
2924         struct inode *inode = d_inode(old_dentry);
2925         int ret = 0;
2926
2927         /*
2928          * No ordinary (disk based) filesystem counts links as inodes;
2929          * but each new link needs a new dentry, pinning lowmem, and
2930          * tmpfs dentries cannot be pruned until they are unlinked.
2931          * But if an O_TMPFILE file is linked into the tmpfs, the
2932          * first link must skip that, to get the accounting right.
2933          */
2934         if (inode->i_nlink) {
2935                 ret = shmem_reserve_inode(inode->i_sb);
2936                 if (ret)
2937                         goto out;
2938         }
2939
2940         dir->i_size += BOGO_DIRENT_SIZE;
2941         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2942         inc_nlink(inode);
2943         ihold(inode);   /* New dentry reference */
2944         dget(dentry);           /* Extra pinning count for the created dentry */
2945         d_instantiate(dentry, inode);
2946 out:
2947         return ret;
2948 }
2949
2950 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2951 {
2952         struct inode *inode = d_inode(dentry);
2953
2954         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2955                 shmem_free_inode(inode->i_sb);
2956
2957         dir->i_size -= BOGO_DIRENT_SIZE;
2958         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2959         drop_nlink(inode);
2960         dput(dentry);   /* Undo the count from "create" - this does all the work */
2961         return 0;
2962 }
2963
2964 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2965 {
2966         if (!simple_empty(dentry))
2967                 return -ENOTEMPTY;
2968
2969         drop_nlink(d_inode(dentry));
2970         drop_nlink(dir);
2971         return shmem_unlink(dir, dentry);
2972 }
2973
2974 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2975 {
2976         bool old_is_dir = d_is_dir(old_dentry);
2977         bool new_is_dir = d_is_dir(new_dentry);
2978
2979         if (old_dir != new_dir && old_is_dir != new_is_dir) {
2980                 if (old_is_dir) {
2981                         drop_nlink(old_dir);
2982                         inc_nlink(new_dir);
2983                 } else {
2984                         drop_nlink(new_dir);
2985                         inc_nlink(old_dir);
2986                 }
2987         }
2988         old_dir->i_ctime = old_dir->i_mtime =
2989         new_dir->i_ctime = new_dir->i_mtime =
2990         d_inode(old_dentry)->i_ctime =
2991         d_inode(new_dentry)->i_ctime = current_time(old_dir);
2992
2993         return 0;
2994 }
2995
2996 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
2997 {
2998         struct dentry *whiteout;
2999         int error;
3000
3001         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3002         if (!whiteout)
3003                 return -ENOMEM;
3004
3005         error = shmem_mknod(old_dir, whiteout,
3006                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3007         dput(whiteout);
3008         if (error)
3009                 return error;
3010
3011         /*
3012          * Cheat and hash the whiteout while the old dentry is still in
3013          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3014          *
3015          * d_lookup() will consistently find one of them at this point,
3016          * not sure which one, but that isn't even important.
3017          */
3018         d_rehash(whiteout);
3019         return 0;
3020 }
3021
3022 /*
3023  * The VFS layer already does all the dentry stuff for rename,
3024  * we just have to decrement the usage count for the target if
3025  * it exists so that the VFS layer correctly free's it when it
3026  * gets overwritten.
3027  */
3028 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
3029 {
3030         struct inode *inode = d_inode(old_dentry);
3031         int they_are_dirs = S_ISDIR(inode->i_mode);
3032
3033         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3034                 return -EINVAL;
3035
3036         if (flags & RENAME_EXCHANGE)
3037                 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
3038
3039         if (!simple_empty(new_dentry))
3040                 return -ENOTEMPTY;
3041
3042         if (flags & RENAME_WHITEOUT) {
3043                 int error;
3044
3045                 error = shmem_whiteout(old_dir, old_dentry);
3046                 if (error)
3047                         return error;
3048         }
3049
3050         if (d_really_is_positive(new_dentry)) {
3051                 (void) shmem_unlink(new_dir, new_dentry);
3052                 if (they_are_dirs) {
3053                         drop_nlink(d_inode(new_dentry));
3054                         drop_nlink(old_dir);
3055                 }
3056         } else if (they_are_dirs) {
3057                 drop_nlink(old_dir);
3058                 inc_nlink(new_dir);
3059         }
3060
3061         old_dir->i_size -= BOGO_DIRENT_SIZE;
3062         new_dir->i_size += BOGO_DIRENT_SIZE;
3063         old_dir->i_ctime = old_dir->i_mtime =
3064         new_dir->i_ctime = new_dir->i_mtime =
3065         inode->i_ctime = current_time(old_dir);
3066         return 0;
3067 }
3068
3069 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3070 {
3071         int error;
3072         int len;
3073         struct inode *inode;
3074         struct page *page;
3075
3076         len = strlen(symname) + 1;
3077         if (len > PAGE_SIZE)
3078                 return -ENAMETOOLONG;
3079
3080         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3081                                 VM_NORESERVE);
3082         if (!inode)
3083                 return -ENOSPC;
3084
3085         error = security_inode_init_security(inode, dir, &dentry->d_name,
3086                                              shmem_initxattrs, NULL);
3087         if (error && error != -EOPNOTSUPP) {
3088                 iput(inode);
3089                 return error;
3090         }
3091
3092         inode->i_size = len-1;
3093         if (len <= SHORT_SYMLINK_LEN) {
3094                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3095                 if (!inode->i_link) {
3096                         iput(inode);
3097                         return -ENOMEM;
3098                 }
3099                 inode->i_op = &shmem_short_symlink_operations;
3100         } else {
3101                 inode_nohighmem(inode);
3102                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3103                 if (error) {
3104                         iput(inode);
3105                         return error;
3106                 }
3107                 inode->i_mapping->a_ops = &shmem_aops;
3108                 inode->i_op = &shmem_symlink_inode_operations;
3109                 memcpy(page_address(page), symname, len);
3110                 SetPageUptodate(page);
3111                 set_page_dirty(page);
3112                 unlock_page(page);
3113                 put_page(page);
3114         }
3115         dir->i_size += BOGO_DIRENT_SIZE;
3116         dir->i_ctime = dir->i_mtime = current_time(dir);
3117         d_instantiate(dentry, inode);
3118         dget(dentry);
3119         return 0;
3120 }
3121
3122 static void shmem_put_link(void *arg)
3123 {
3124         mark_page_accessed(arg);
3125         put_page(arg);
3126 }
3127
3128 static const char *shmem_get_link(struct dentry *dentry,
3129                                   struct inode *inode,
3130                                   struct delayed_call *done)
3131 {
3132         struct page *page = NULL;
3133         int error;
3134         if (!dentry) {
3135                 page = find_get_page(inode->i_mapping, 0);
3136                 if (!page)
3137                         return ERR_PTR(-ECHILD);
3138                 if (!PageUptodate(page)) {
3139                         put_page(page);
3140                         return ERR_PTR(-ECHILD);
3141                 }
3142         } else {
3143                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3144                 if (error)
3145                         return ERR_PTR(error);
3146                 unlock_page(page);
3147         }
3148         set_delayed_call(done, shmem_put_link, page);
3149         return page_address(page);
3150 }
3151
3152 #ifdef CONFIG_TMPFS_XATTR
3153 /*
3154  * Superblocks without xattr inode operations may get some security.* xattr
3155  * support from the LSM "for free". As soon as we have any other xattrs
3156  * like ACLs, we also need to implement the security.* handlers at
3157  * filesystem level, though.
3158  */
3159
3160 /*
3161  * Callback for security_inode_init_security() for acquiring xattrs.
3162  */
3163 static int shmem_initxattrs(struct inode *inode,
3164                             const struct xattr *xattr_array,
3165                             void *fs_info)
3166 {
3167         struct shmem_inode_info *info = SHMEM_I(inode);
3168         const struct xattr *xattr;
3169         struct simple_xattr *new_xattr;
3170         size_t len;
3171
3172         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3173                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3174                 if (!new_xattr)
3175                         return -ENOMEM;
3176
3177                 len = strlen(xattr->name) + 1;
3178                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3179                                           GFP_KERNEL);
3180                 if (!new_xattr->name) {
3181                         kfree(new_xattr);
3182                         return -ENOMEM;
3183                 }
3184
3185                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3186                        XATTR_SECURITY_PREFIX_LEN);
3187                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3188                        xattr->name, len);
3189
3190                 simple_xattr_list_add(&info->xattrs, new_xattr);
3191         }
3192
3193         return 0;
3194 }
3195
3196 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3197                                    struct dentry *unused, struct inode *inode,
3198                                    const char *name, void *buffer, size_t size)
3199 {
3200         struct shmem_inode_info *info = SHMEM_I(inode);
3201
3202         name = xattr_full_name(handler, name);
3203         return simple_xattr_get(&info->xattrs, name, buffer, size);
3204 }
3205
3206 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3207                                    struct dentry *unused, struct inode *inode,
3208                                    const char *name, const void *value,
3209                                    size_t size, int flags)
3210 {
3211         struct shmem_inode_info *info = SHMEM_I(inode);
3212
3213         name = xattr_full_name(handler, name);
3214         return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3215 }
3216
3217 static const struct xattr_handler shmem_security_xattr_handler = {
3218         .prefix = XATTR_SECURITY_PREFIX,
3219         .get = shmem_xattr_handler_get,
3220         .set = shmem_xattr_handler_set,
3221 };
3222
3223 static const struct xattr_handler shmem_trusted_xattr_handler = {
3224         .prefix = XATTR_TRUSTED_PREFIX,
3225         .get = shmem_xattr_handler_get,
3226         .set = shmem_xattr_handler_set,
3227 };
3228
3229 static const struct xattr_handler *shmem_xattr_handlers[] = {
3230 #ifdef CONFIG_TMPFS_POSIX_ACL
3231         &posix_acl_access_xattr_handler,
3232         &posix_acl_default_xattr_handler,
3233 #endif
3234         &shmem_security_xattr_handler,
3235         &shmem_trusted_xattr_handler,
3236         NULL
3237 };
3238
3239 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3240 {
3241         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3242         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3243 }
3244 #endif /* CONFIG_TMPFS_XATTR */
3245
3246 static const struct inode_operations shmem_short_symlink_operations = {
3247         .get_link       = simple_get_link,
3248 #ifdef CONFIG_TMPFS_XATTR
3249         .listxattr      = shmem_listxattr,
3250 #endif
3251 };
3252
3253 static const struct inode_operations shmem_symlink_inode_operations = {
3254         .get_link       = shmem_get_link,
3255 #ifdef CONFIG_TMPFS_XATTR
3256         .listxattr      = shmem_listxattr,
3257 #endif
3258 };
3259
3260 static struct dentry *shmem_get_parent(struct dentry *child)
3261 {
3262         return ERR_PTR(-ESTALE);
3263 }
3264
3265 static int shmem_match(struct inode *ino, void *vfh)
3266 {
3267         __u32 *fh = vfh;
3268         __u64 inum = fh[2];
3269         inum = (inum << 32) | fh[1];
3270         return ino->i_ino == inum && fh[0] == ino->i_generation;
3271 }
3272
3273 /* Find any alias of inode, but prefer a hashed alias */
3274 static struct dentry *shmem_find_alias(struct inode *inode)
3275 {
3276         struct dentry *alias = d_find_alias(inode);
3277
3278         return alias ?: d_find_any_alias(inode);
3279 }
3280
3281
3282 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3283                 struct fid *fid, int fh_len, int fh_type)
3284 {
3285         struct inode *inode;
3286         struct dentry *dentry = NULL;
3287         u64 inum;
3288
3289         if (fh_len < 3)
3290                 return NULL;
3291
3292         inum = fid->raw[2];
3293         inum = (inum << 32) | fid->raw[1];
3294
3295         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3296                         shmem_match, fid->raw);
3297         if (inode) {
3298                 dentry = shmem_find_alias(inode);
3299                 iput(inode);
3300         }
3301
3302         return dentry;
3303 }
3304
3305 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3306                                 struct inode *parent)
3307 {
3308         if (*len < 3) {
3309                 *len = 3;
3310                 return FILEID_INVALID;
3311         }
3312
3313         if (inode_unhashed(inode)) {
3314                 /* Unfortunately insert_inode_hash is not idempotent,
3315                  * so as we hash inodes here rather than at creation
3316                  * time, we need a lock to ensure we only try
3317                  * to do it once
3318                  */
3319                 static DEFINE_SPINLOCK(lock);
3320                 spin_lock(&lock);
3321                 if (inode_unhashed(inode))
3322                         __insert_inode_hash(inode,
3323                                             inode->i_ino + inode->i_generation);
3324                 spin_unlock(&lock);
3325         }
3326
3327         fh[0] = inode->i_generation;
3328         fh[1] = inode->i_ino;
3329         fh[2] = ((__u64)inode->i_ino) >> 32;
3330
3331         *len = 3;
3332         return 1;
3333 }
3334
3335 static const struct export_operations shmem_export_ops = {
3336         .get_parent     = shmem_get_parent,
3337         .encode_fh      = shmem_encode_fh,
3338         .fh_to_dentry   = shmem_fh_to_dentry,
3339 };
3340
3341 enum shmem_param {
3342         Opt_gid,
3343         Opt_huge,
3344         Opt_mode,
3345         Opt_mpol,
3346         Opt_nr_blocks,
3347         Opt_nr_inodes,
3348         Opt_size,
3349         Opt_uid,
3350 };
3351
3352 static const struct constant_table shmem_param_enums_huge[] = {
3353         {"never",       SHMEM_HUGE_NEVER },
3354         {"always",      SHMEM_HUGE_ALWAYS },
3355         {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3356         {"advise",      SHMEM_HUGE_ADVISE },
3357         {}
3358 };
3359
3360 const struct fs_parameter_spec shmem_fs_parameters[] = {
3361         fsparam_u32   ("gid",           Opt_gid),
3362         fsparam_enum  ("huge",          Opt_huge,  shmem_param_enums_huge),
3363         fsparam_u32oct("mode",          Opt_mode),
3364         fsparam_string("mpol",          Opt_mpol),
3365         fsparam_string("nr_blocks",     Opt_nr_blocks),
3366         fsparam_string("nr_inodes",     Opt_nr_inodes),
3367         fsparam_string("size",          Opt_size),
3368         fsparam_u32   ("uid",           Opt_uid),
3369         {}
3370 };
3371
3372 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3373 {
3374         struct shmem_options *ctx = fc->fs_private;
3375         struct fs_parse_result result;
3376         unsigned long long size;
3377         char *rest;
3378         int opt;
3379
3380         opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3381         if (opt < 0)
3382                 return opt;
3383
3384         switch (opt) {
3385         case Opt_size:
3386                 size = memparse(param->string, &rest);
3387                 if (*rest == '%') {
3388                         size <<= PAGE_SHIFT;
3389                         size *= totalram_pages();
3390                         do_div(size, 100);
3391                         rest++;
3392                 }
3393                 if (*rest)
3394                         goto bad_value;
3395                 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3396                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3397                 break;
3398         case Opt_nr_blocks:
3399                 ctx->blocks = memparse(param->string, &rest);
3400                 if (*rest)
3401                         goto bad_value;
3402                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3403                 break;
3404         case Opt_nr_inodes:
3405                 ctx->inodes = memparse(param->string, &rest);
3406                 if (*rest)
3407                         goto bad_value;
3408                 ctx->seen |= SHMEM_SEEN_INODES;
3409                 break;
3410         case Opt_mode:
3411                 ctx->mode = result.uint_32 & 07777;
3412                 break;
3413         case Opt_uid:
3414                 ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3415                 if (!uid_valid(ctx->uid))
3416                         goto bad_value;
3417                 break;
3418         case Opt_gid:
3419                 ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3420                 if (!gid_valid(ctx->gid))
3421                         goto bad_value;
3422                 break;
3423         case Opt_huge:
3424                 ctx->huge = result.uint_32;
3425                 if (ctx->huge != SHMEM_HUGE_NEVER &&
3426                     !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3427                       has_transparent_hugepage()))
3428                         goto unsupported_parameter;
3429                 ctx->seen |= SHMEM_SEEN_HUGE;
3430                 break;
3431         case Opt_mpol:
3432                 if (IS_ENABLED(CONFIG_NUMA)) {
3433                         mpol_put(ctx->mpol);
3434                         ctx->mpol = NULL;
3435                         if (mpol_parse_str(param->string, &ctx->mpol))
3436                                 goto bad_value;
3437                         break;
3438                 }
3439                 goto unsupported_parameter;
3440         }
3441         return 0;
3442
3443 unsupported_parameter:
3444         return invalfc(fc, "Unsupported parameter '%s'", param->key);
3445 bad_value:
3446         return invalfc(fc, "Bad value for '%s'", param->key);
3447 }
3448
3449 static int shmem_parse_options(struct fs_context *fc, void *data)
3450 {
3451         char *options = data;
3452
3453         if (options) {
3454                 int err = security_sb_eat_lsm_opts(options, &fc->security);
3455                 if (err)
3456                         return err;
3457         }
3458
3459         while (options != NULL) {
3460                 char *this_char = options;
3461                 for (;;) {
3462                         /*
3463                          * NUL-terminate this option: unfortunately,
3464                          * mount options form a comma-separated list,
3465                          * but mpol's nodelist may also contain commas.
3466                          */
3467                         options = strchr(options, ',');
3468                         if (options == NULL)
3469                                 break;
3470                         options++;
3471                         if (!isdigit(*options)) {
3472                                 options[-1] = '\0';
3473                                 break;
3474                         }
3475                 }
3476                 if (*this_char) {
3477                         char *value = strchr(this_char,'=');
3478                         size_t len = 0;
3479                         int err;
3480
3481                         if (value) {
3482                                 *value++ = '\0';
3483                                 len = strlen(value);
3484                         }
3485                         err = vfs_parse_fs_string(fc, this_char, value, len);
3486                         if (err < 0)
3487                                 return err;
3488                 }
3489         }
3490         return 0;
3491 }
3492
3493 /*
3494  * Reconfigure a shmem filesystem.
3495  *
3496  * Note that we disallow change from limited->unlimited blocks/inodes while any
3497  * are in use; but we must separately disallow unlimited->limited, because in
3498  * that case we have no record of how much is already in use.
3499  */
3500 static int shmem_reconfigure(struct fs_context *fc)
3501 {
3502         struct shmem_options *ctx = fc->fs_private;
3503         struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3504         unsigned long inodes;
3505         const char *err;
3506
3507         spin_lock(&sbinfo->stat_lock);
3508         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3509         if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3510                 if (!sbinfo->max_blocks) {
3511                         err = "Cannot retroactively limit size";
3512                         goto out;
3513                 }
3514                 if (percpu_counter_compare(&sbinfo->used_blocks,
3515                                            ctx->blocks) > 0) {
3516                         err = "Too small a size for current use";
3517                         goto out;
3518                 }
3519         }
3520         if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3521                 if (!sbinfo->max_inodes) {
3522                         err = "Cannot retroactively limit inodes";
3523                         goto out;
3524                 }
3525                 if (ctx->inodes < inodes) {
3526                         err = "Too few inodes for current use";
3527                         goto out;
3528                 }
3529         }
3530
3531         if (ctx->seen & SHMEM_SEEN_HUGE)
3532                 sbinfo->huge = ctx->huge;
3533         if (ctx->seen & SHMEM_SEEN_BLOCKS)
3534                 sbinfo->max_blocks  = ctx->blocks;
3535         if (ctx->seen & SHMEM_SEEN_INODES) {
3536                 sbinfo->max_inodes  = ctx->inodes;
3537                 sbinfo->free_inodes = ctx->inodes - inodes;
3538         }
3539
3540         /*
3541          * Preserve previous mempolicy unless mpol remount option was specified.
3542          */
3543         if (ctx->mpol) {
3544                 mpol_put(sbinfo->mpol);
3545                 sbinfo->mpol = ctx->mpol;       /* transfers initial ref */
3546                 ctx->mpol = NULL;
3547         }
3548         spin_unlock(&sbinfo->stat_lock);
3549         return 0;
3550 out:
3551         spin_unlock(&sbinfo->stat_lock);
3552         return invalfc(fc, "%s", err);
3553 }
3554
3555 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3556 {
3557         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3558
3559         if (sbinfo->max_blocks != shmem_default_max_blocks())
3560                 seq_printf(seq, ",size=%luk",
3561                         sbinfo->max_blocks << (PAGE_SHIFT - 10));
3562         if (sbinfo->max_inodes != shmem_default_max_inodes())
3563                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3564         if (sbinfo->mode != (0777 | S_ISVTX))
3565                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3566         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3567                 seq_printf(seq, ",uid=%u",
3568                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
3569         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3570                 seq_printf(seq, ",gid=%u",
3571                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
3572 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3573         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3574         if (sbinfo->huge)
3575                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3576 #endif
3577         shmem_show_mpol(seq, sbinfo->mpol);
3578         return 0;
3579 }
3580
3581 #endif /* CONFIG_TMPFS */
3582
3583 static void shmem_put_super(struct super_block *sb)
3584 {
3585         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3586
3587         percpu_counter_destroy(&sbinfo->used_blocks);
3588         mpol_put(sbinfo->mpol);
3589         kfree(sbinfo);
3590         sb->s_fs_info = NULL;
3591 }
3592
3593 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3594 {
3595         struct shmem_options *ctx = fc->fs_private;
3596         struct inode *inode;
3597         struct shmem_sb_info *sbinfo;
3598         int err = -ENOMEM;
3599
3600         /* Round up to L1_CACHE_BYTES to resist false sharing */
3601         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3602                                 L1_CACHE_BYTES), GFP_KERNEL);
3603         if (!sbinfo)
3604                 return -ENOMEM;
3605
3606         sb->s_fs_info = sbinfo;
3607
3608 #ifdef CONFIG_TMPFS
3609         /*
3610          * Per default we only allow half of the physical ram per
3611          * tmpfs instance, limiting inodes to one per page of lowmem;
3612          * but the internal instance is left unlimited.
3613          */
3614         if (!(sb->s_flags & SB_KERNMOUNT)) {
3615                 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3616                         ctx->blocks = shmem_default_max_blocks();
3617                 if (!(ctx->seen & SHMEM_SEEN_INODES))
3618                         ctx->inodes = shmem_default_max_inodes();
3619         } else {
3620                 sb->s_flags |= SB_NOUSER;
3621         }
3622         sb->s_export_op = &shmem_export_ops;
3623         sb->s_flags |= SB_NOSEC;
3624 #else
3625         sb->s_flags |= SB_NOUSER;
3626 #endif
3627         sbinfo->max_blocks = ctx->blocks;
3628         sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3629         sbinfo->uid = ctx->uid;
3630         sbinfo->gid = ctx->gid;
3631         sbinfo->mode = ctx->mode;
3632         sbinfo->huge = ctx->huge;
3633         sbinfo->mpol = ctx->mpol;
3634         ctx->mpol = NULL;
3635
3636         spin_lock_init(&sbinfo->stat_lock);
3637         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3638                 goto failed;
3639         spin_lock_init(&sbinfo->shrinklist_lock);
3640         INIT_LIST_HEAD(&sbinfo->shrinklist);
3641
3642         sb->s_maxbytes = MAX_LFS_FILESIZE;
3643         sb->s_blocksize = PAGE_SIZE;
3644         sb->s_blocksize_bits = PAGE_SHIFT;
3645         sb->s_magic = TMPFS_MAGIC;
3646         sb->s_op = &shmem_ops;
3647         sb->s_time_gran = 1;
3648 #ifdef CONFIG_TMPFS_XATTR
3649         sb->s_xattr = shmem_xattr_handlers;
3650 #endif
3651 #ifdef CONFIG_TMPFS_POSIX_ACL
3652         sb->s_flags |= SB_POSIXACL;
3653 #endif
3654         uuid_gen(&sb->s_uuid);
3655
3656         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3657         if (!inode)
3658                 goto failed;
3659         inode->i_uid = sbinfo->uid;
3660         inode->i_gid = sbinfo->gid;
3661         sb->s_root = d_make_root(inode);
3662         if (!sb->s_root)
3663                 goto failed;
3664         return 0;
3665
3666 failed:
3667         shmem_put_super(sb);
3668         return err;
3669 }
3670
3671 static int shmem_get_tree(struct fs_context *fc)
3672 {
3673         return get_tree_nodev(fc, shmem_fill_super);
3674 }
3675
3676 static void shmem_free_fc(struct fs_context *fc)
3677 {
3678         struct shmem_options *ctx = fc->fs_private;
3679
3680         if (ctx) {
3681                 mpol_put(ctx->mpol);
3682                 kfree(ctx);
3683         }
3684 }
3685
3686 static const struct fs_context_operations shmem_fs_context_ops = {
3687         .free                   = shmem_free_fc,
3688         .get_tree               = shmem_get_tree,
3689 #ifdef CONFIG_TMPFS
3690         .parse_monolithic       = shmem_parse_options,
3691         .parse_param            = shmem_parse_one,
3692         .reconfigure            = shmem_reconfigure,
3693 #endif
3694 };
3695
3696 static struct kmem_cache *shmem_inode_cachep;
3697
3698 static struct inode *shmem_alloc_inode(struct super_block *sb)
3699 {
3700         struct shmem_inode_info *info;
3701         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3702         if (!info)
3703                 return NULL;
3704         return &info->vfs_inode;
3705 }
3706
3707 static void shmem_free_in_core_inode(struct inode *inode)
3708 {
3709         if (S_ISLNK(inode->i_mode))
3710                 kfree(inode->i_link);
3711         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3712 }
3713
3714 static void shmem_destroy_inode(struct inode *inode)
3715 {
3716         if (S_ISREG(inode->i_mode))
3717                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3718 }
3719
3720 static void shmem_init_inode(void *foo)
3721 {
3722         struct shmem_inode_info *info = foo;
3723         inode_init_once(&info->vfs_inode);
3724 }
3725
3726 static void shmem_init_inodecache(void)
3727 {
3728         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3729                                 sizeof(struct shmem_inode_info),
3730                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3731 }
3732
3733 static void shmem_destroy_inodecache(void)
3734 {
3735         kmem_cache_destroy(shmem_inode_cachep);
3736 }
3737
3738 static const struct address_space_operations shmem_aops = {
3739         .writepage      = shmem_writepage,
3740         .set_page_dirty = __set_page_dirty_no_writeback,
3741 #ifdef CONFIG_TMPFS
3742         .write_begin    = shmem_write_begin,
3743         .write_end      = shmem_write_end,
3744 #endif
3745 #ifdef CONFIG_MIGRATION
3746         .migratepage    = migrate_page,
3747 #endif
3748         .error_remove_page = generic_error_remove_page,
3749 };
3750
3751 static const struct file_operations shmem_file_operations = {
3752         .mmap           = shmem_mmap,
3753         .get_unmapped_area = shmem_get_unmapped_area,
3754 #ifdef CONFIG_TMPFS
3755         .llseek         = shmem_file_llseek,
3756         .read_iter      = shmem_file_read_iter,
3757         .write_iter     = generic_file_write_iter,
3758         .fsync          = noop_fsync,
3759         .splice_read    = generic_file_splice_read,
3760         .splice_write   = iter_file_splice_write,
3761         .fallocate      = shmem_fallocate,
3762 #endif
3763 };
3764
3765 static const struct inode_operations shmem_inode_operations = {
3766         .getattr        = shmem_getattr,
3767         .setattr        = shmem_setattr,
3768 #ifdef CONFIG_TMPFS_XATTR
3769         .listxattr      = shmem_listxattr,
3770         .set_acl        = simple_set_acl,
3771 #endif
3772 };
3773
3774 static const struct inode_operations shmem_dir_inode_operations = {
3775 #ifdef CONFIG_TMPFS
3776         .create         = shmem_create,
3777         .lookup         = simple_lookup,
3778         .link           = shmem_link,
3779         .unlink         = shmem_unlink,
3780         .symlink        = shmem_symlink,
3781         .mkdir          = shmem_mkdir,
3782         .rmdir          = shmem_rmdir,
3783         .mknod          = shmem_mknod,
3784         .rename         = shmem_rename2,
3785         .tmpfile        = shmem_tmpfile,
3786 #endif
3787 #ifdef CONFIG_TMPFS_XATTR
3788         .listxattr      = shmem_listxattr,
3789 #endif
3790 #ifdef CONFIG_TMPFS_POSIX_ACL
3791         .setattr        = shmem_setattr,
3792         .set_acl        = simple_set_acl,
3793 #endif
3794 };
3795
3796 static const struct inode_operations shmem_special_inode_operations = {
3797 #ifdef CONFIG_TMPFS_XATTR
3798         .listxattr      = shmem_listxattr,
3799 #endif
3800 #ifdef CONFIG_TMPFS_POSIX_ACL
3801         .setattr        = shmem_setattr,
3802         .set_acl        = simple_set_acl,
3803 #endif
3804 };
3805
3806 static const struct super_operations shmem_ops = {
3807         .alloc_inode    = shmem_alloc_inode,
3808         .free_inode     = shmem_free_in_core_inode,
3809         .destroy_inode  = shmem_destroy_inode,
3810 #ifdef CONFIG_TMPFS
3811         .statfs         = shmem_statfs,
3812         .show_options   = shmem_show_options,
3813 #endif
3814         .evict_inode    = shmem_evict_inode,
3815         .drop_inode     = generic_delete_inode,
3816         .put_super      = shmem_put_super,
3817 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3818         .nr_cached_objects      = shmem_unused_huge_count,
3819         .free_cached_objects    = shmem_unused_huge_scan,
3820 #endif
3821 };
3822
3823 static const struct vm_operations_struct shmem_vm_ops = {
3824         .fault          = shmem_fault,
3825         .map_pages      = filemap_map_pages,
3826 #ifdef CONFIG_NUMA
3827         .set_policy     = shmem_set_policy,
3828         .get_policy     = shmem_get_policy,
3829 #endif
3830 };
3831
3832 int shmem_init_fs_context(struct fs_context *fc)
3833 {
3834         struct shmem_options *ctx;
3835
3836         ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
3837         if (!ctx)
3838                 return -ENOMEM;
3839
3840         ctx->mode = 0777 | S_ISVTX;
3841         ctx->uid = current_fsuid();
3842         ctx->gid = current_fsgid();
3843
3844         fc->fs_private = ctx;
3845         fc->ops = &shmem_fs_context_ops;
3846         return 0;
3847 }
3848
3849 static struct file_system_type shmem_fs_type = {
3850         .owner          = THIS_MODULE,
3851         .name           = "tmpfs",
3852         .init_fs_context = shmem_init_fs_context,
3853 #ifdef CONFIG_TMPFS
3854         .parameters     = shmem_fs_parameters,
3855 #endif
3856         .kill_sb        = kill_litter_super,
3857         .fs_flags       = FS_USERNS_MOUNT,
3858 };
3859
3860 int __init shmem_init(void)
3861 {
3862         int error;
3863
3864         shmem_init_inodecache();
3865
3866         error = register_filesystem(&shmem_fs_type);
3867         if (error) {
3868                 pr_err("Could not register tmpfs\n");
3869                 goto out2;
3870         }
3871
3872         shm_mnt = kern_mount(&shmem_fs_type);
3873         if (IS_ERR(shm_mnt)) {
3874                 error = PTR_ERR(shm_mnt);
3875                 pr_err("Could not kern_mount tmpfs\n");
3876                 goto out1;
3877         }
3878
3879 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3880         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
3881                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3882         else
3883                 shmem_huge = 0; /* just in case it was patched */
3884 #endif
3885         return 0;
3886
3887 out1:
3888         unregister_filesystem(&shmem_fs_type);
3889 out2:
3890         shmem_destroy_inodecache();
3891         shm_mnt = ERR_PTR(error);
3892         return error;
3893 }
3894
3895 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
3896 static ssize_t shmem_enabled_show(struct kobject *kobj,
3897                 struct kobj_attribute *attr, char *buf)
3898 {
3899         static const int values[] = {
3900                 SHMEM_HUGE_ALWAYS,
3901                 SHMEM_HUGE_WITHIN_SIZE,
3902                 SHMEM_HUGE_ADVISE,
3903                 SHMEM_HUGE_NEVER,
3904                 SHMEM_HUGE_DENY,
3905                 SHMEM_HUGE_FORCE,
3906         };
3907         int i, count;
3908
3909         for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) {
3910                 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s ";
3911
3912                 count += sprintf(buf + count, fmt,
3913                                 shmem_format_huge(values[i]));
3914         }
3915         buf[count - 1] = '\n';
3916         return count;
3917 }
3918
3919 static ssize_t shmem_enabled_store(struct kobject *kobj,
3920                 struct kobj_attribute *attr, const char *buf, size_t count)
3921 {
3922         char tmp[16];
3923         int huge;
3924
3925         if (count + 1 > sizeof(tmp))
3926                 return -EINVAL;
3927         memcpy(tmp, buf, count);
3928         tmp[count] = '\0';
3929         if (count && tmp[count - 1] == '\n')
3930                 tmp[count - 1] = '\0';
3931
3932         huge = shmem_parse_huge(tmp);
3933         if (huge == -EINVAL)
3934                 return -EINVAL;
3935         if (!has_transparent_hugepage() &&
3936                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
3937                 return -EINVAL;
3938
3939         shmem_huge = huge;
3940         if (shmem_huge > SHMEM_HUGE_DENY)
3941                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3942         return count;
3943 }
3944
3945 struct kobj_attribute shmem_enabled_attr =
3946         __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
3947 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
3948
3949 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3950 bool shmem_huge_enabled(struct vm_area_struct *vma)
3951 {
3952         struct inode *inode = file_inode(vma->vm_file);
3953         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3954         loff_t i_size;
3955         pgoff_t off;
3956
3957         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
3958             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
3959                 return false;
3960         if (shmem_huge == SHMEM_HUGE_FORCE)
3961                 return true;
3962         if (shmem_huge == SHMEM_HUGE_DENY)
3963                 return false;
3964         switch (sbinfo->huge) {
3965                 case SHMEM_HUGE_NEVER:
3966                         return false;
3967                 case SHMEM_HUGE_ALWAYS:
3968                         return true;
3969                 case SHMEM_HUGE_WITHIN_SIZE:
3970                         off = round_up(vma->vm_pgoff, HPAGE_PMD_NR);
3971                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
3972                         if (i_size >= HPAGE_PMD_SIZE &&
3973                                         i_size >> PAGE_SHIFT >= off)
3974                                 return true;
3975                         fallthrough;
3976                 case SHMEM_HUGE_ADVISE:
3977                         /* TODO: implement fadvise() hints */
3978                         return (vma->vm_flags & VM_HUGEPAGE);
3979                 default:
3980                         VM_BUG_ON(1);
3981                         return false;
3982         }
3983 }
3984 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3985
3986 #else /* !CONFIG_SHMEM */
3987
3988 /*
3989  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3990  *
3991  * This is intended for small system where the benefits of the full
3992  * shmem code (swap-backed and resource-limited) are outweighed by
3993  * their complexity. On systems without swap this code should be
3994  * effectively equivalent, but much lighter weight.
3995  */
3996
3997 static struct file_system_type shmem_fs_type = {
3998         .name           = "tmpfs",
3999         .init_fs_context = ramfs_init_fs_context,
4000         .parameters     = ramfs_fs_parameters,
4001         .kill_sb        = kill_litter_super,
4002         .fs_flags       = FS_USERNS_MOUNT,
4003 };
4004
4005 int __init shmem_init(void)
4006 {
4007         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4008
4009         shm_mnt = kern_mount(&shmem_fs_type);
4010         BUG_ON(IS_ERR(shm_mnt));
4011
4012         return 0;
4013 }
4014
4015 int shmem_unuse(unsigned int type, bool frontswap,
4016                 unsigned long *fs_pages_to_unuse)
4017 {
4018         return 0;
4019 }
4020
4021 int shmem_lock(struct file *file, int lock, struct user_struct *user)
4022 {
4023         return 0;
4024 }
4025
4026 void shmem_unlock_mapping(struct address_space *mapping)
4027 {
4028 }
4029
4030 #ifdef CONFIG_MMU
4031 unsigned long shmem_get_unmapped_area(struct file *file,
4032                                       unsigned long addr, unsigned long len,
4033                                       unsigned long pgoff, unsigned long flags)
4034 {
4035         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4036 }
4037 #endif
4038
4039 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4040 {
4041         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4042 }
4043 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4044
4045 #define shmem_vm_ops                            generic_file_vm_ops
4046 #define shmem_file_operations                   ramfs_file_operations
4047 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
4048 #define shmem_acct_size(flags, size)            0
4049 #define shmem_unacct_size(flags, size)          do {} while (0)
4050
4051 #endif /* CONFIG_SHMEM */
4052
4053 /* common code */
4054
4055 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4056                                        unsigned long flags, unsigned int i_flags)
4057 {
4058         struct inode *inode;
4059         struct file *res;
4060
4061         if (IS_ERR(mnt))
4062                 return ERR_CAST(mnt);
4063
4064         if (size < 0 || size > MAX_LFS_FILESIZE)
4065                 return ERR_PTR(-EINVAL);
4066
4067         if (shmem_acct_size(flags, size))
4068                 return ERR_PTR(-ENOMEM);
4069
4070         inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4071                                 flags);
4072         if (unlikely(!inode)) {
4073                 shmem_unacct_size(flags, size);
4074                 return ERR_PTR(-ENOSPC);
4075         }
4076         inode->i_flags |= i_flags;
4077         inode->i_size = size;
4078         clear_nlink(inode);     /* It is unlinked */
4079         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4080         if (!IS_ERR(res))
4081                 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4082                                 &shmem_file_operations);
4083         if (IS_ERR(res))
4084                 iput(inode);
4085         return res;
4086 }
4087
4088 /**
4089  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4090  *      kernel internal.  There will be NO LSM permission checks against the
4091  *      underlying inode.  So users of this interface must do LSM checks at a
4092  *      higher layer.  The users are the big_key and shm implementations.  LSM
4093  *      checks are provided at the key or shm level rather than the inode.
4094  * @name: name for dentry (to be seen in /proc/<pid>/maps
4095  * @size: size to be set for the file
4096  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4097  */
4098 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4099 {
4100         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4101 }
4102
4103 /**
4104  * shmem_file_setup - get an unlinked file living in tmpfs
4105  * @name: name for dentry (to be seen in /proc/<pid>/maps
4106  * @size: size to be set for the file
4107  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4108  */
4109 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4110 {
4111         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4112 }
4113 EXPORT_SYMBOL_GPL(shmem_file_setup);
4114
4115 /**
4116  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4117  * @mnt: the tmpfs mount where the file will be created
4118  * @name: name for dentry (to be seen in /proc/<pid>/maps
4119  * @size: size to be set for the file
4120  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4121  */
4122 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4123                                        loff_t size, unsigned long flags)
4124 {
4125         return __shmem_file_setup(mnt, name, size, flags, 0);
4126 }
4127 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4128
4129 /**
4130  * shmem_zero_setup - setup a shared anonymous mapping
4131  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
4132  */
4133 int shmem_zero_setup(struct vm_area_struct *vma)
4134 {
4135         struct file *file;
4136         loff_t size = vma->vm_end - vma->vm_start;
4137
4138         /*
4139          * Cloning a new file under mmap_lock leads to a lock ordering conflict
4140          * between XFS directory reading and selinux: since this file is only
4141          * accessible to the user through its mapping, use S_PRIVATE flag to
4142          * bypass file security, in the same way as shmem_kernel_file_setup().
4143          */
4144         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4145         if (IS_ERR(file))
4146                 return PTR_ERR(file);
4147
4148         if (vma->vm_file)
4149                 fput(vma->vm_file);
4150         vma->vm_file = file;
4151         vma->vm_ops = &shmem_vm_ops;
4152
4153         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4154                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4155                         (vma->vm_end & HPAGE_PMD_MASK)) {
4156                 khugepaged_enter(vma, vma->vm_flags);
4157         }
4158
4159         return 0;
4160 }
4161
4162 /**
4163  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4164  * @mapping:    the page's address_space
4165  * @index:      the page index
4166  * @gfp:        the page allocator flags to use if allocating
4167  *
4168  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4169  * with any new page allocations done using the specified allocation flags.
4170  * But read_cache_page_gfp() uses the ->readpage() method: which does not
4171  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4172  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4173  *
4174  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4175  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4176  */
4177 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4178                                          pgoff_t index, gfp_t gfp)
4179 {
4180 #ifdef CONFIG_SHMEM
4181         struct inode *inode = mapping->host;
4182         struct page *page;
4183         int error;
4184
4185         BUG_ON(mapping->a_ops != &shmem_aops);
4186         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4187                                   gfp, NULL, NULL, NULL);
4188         if (error)
4189                 page = ERR_PTR(error);
4190         else
4191                 unlock_page(page);
4192         return page;
4193 #else
4194         /*
4195          * The tiny !SHMEM case uses ramfs without swap
4196          */
4197         return read_cache_page_gfp(mapping, index, gfp);
4198 #endif
4199 }
4200 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);