mm/shmem.c

   1 /*
   2  * Resizable virtual memory filesystem for Linux.
   3  *
   4  * Copyright (C) 2000 Linus Torvalds.
   5  *               2000 Transmeta Corp.
   6  *               2000-2001 Christoph Rohland
   7  *               2000-2001 SAP AG
   8  *               2002 Red Hat Inc.
   9  * Copyright (C) 2002-2011 Hugh Dickins.
  10  * Copyright (C) 2011 Google Inc.
  11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
  12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
  13  *
  14  * Extended attribute support for tmpfs:
  15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  17  *
  18  * tiny-shmem:
  19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  20  *
  21  * This file is released under the GPL.
  22  */
  23
  24 #include <linux/fs.h>
  25 #include <linux/init.h>
  26 #include <linux/vfs.h>
  27 #include <linux/mount.h>
  28 #include <linux/ramfs.h>
  29 #include <linux/pagemap.h>
  30 #include <linux/file.h>
  31 #include <linux/fileattr.h>
  32 #include <linux/mm.h>
  33 #include <linux/random.h>
  34 #include <linux/sched/signal.h>
  35 #include <linux/export.h>
  36 #include <linux/swap.h>
  37 #include <linux/uio.h>
  38 #include <linux/hugetlb.h>
  39 #include <linux/fs_parser.h>
  40 #include <linux/swapfile.h>
  41 #include <linux/iversion.h>
  42 #include "swap.h"
  43
  44 static struct vfsmount *shm_mnt;
  45
  46 #ifdef CONFIG_SHMEM
  47 /*
  48  * This virtual memory filesystem is heavily based on the ramfs. It
  49  * extends ramfs by the ability to use swap and honor resource limits
  50  * which makes it a completely usable filesystem.
  51  */
  52
  53 #include <linux/xattr.h>
  54 #include <linux/exportfs.h>
  55 #include <linux/posix_acl.h>
  56 #include <linux/posix_acl_xattr.h>
  57 #include <linux/mman.h>
  58 #include <linux/string.h>
  59 #include <linux/slab.h>
  60 #include <linux/backing-dev.h>
  61 #include <linux/shmem_fs.h>
  62 #include <linux/writeback.h>
  63 #include <linux/pagevec.h>
  64 #include <linux/percpu_counter.h>
  65 #include <linux/falloc.h>
  66 #include <linux/splice.h>
  67 #include <linux/security.h>
  68 #include <linux/swapops.h>
  69 #include <linux/mempolicy.h>
  70 #include <linux/namei.h>
  71 #include <linux/ctype.h>
  72 #include <linux/migrate.h>
  73 #include <linux/highmem.h>
  74 #include <linux/seq_file.h>
  75 #include <linux/magic.h>
  76 #include <linux/syscalls.h>
  77 #include <linux/fcntl.h>
  78 #include <uapi/linux/memfd.h>
  79 #include <linux/userfaultfd_k.h>
  80 #include <linux/rmap.h>
  81 #include <linux/uuid.h>
  82
  83 #include <linux/uaccess.h>
  84
  85 #include "internal.h"
  86
  87 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
  88 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
  89
  90 /* Pretend that each entry is of this size in directory's i_size */
  91 #define BOGO_DIRENT_SIZE 20
  92
  93 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
  94 #define SHORT_SYMLINK_LEN 128
  95
  96 /*
  97  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
  98  * inode->i_private (with i_rwsem making sure that it has only one user at
  99  * a time): we would prefer not to enlarge the shmem inode just for that.
 100  */
 101 struct shmem_falloc {
 102         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
 103         pgoff_t start;          /* start of range currently being fallocated */
 104         pgoff_t next;           /* the next page offset to be fallocated */
 105         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
 106         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
 107 };
 108
 109 struct shmem_options {
 110         unsigned long long blocks;
 111         unsigned long long inodes;
 112         struct mempolicy *mpol;
 113         kuid_t uid;
 114         kgid_t gid;
 115         umode_t mode;
 116         bool full_inums;
 117         int huge;
 118         int seen;
 119 #define SHMEM_SEEN_BLOCKS 1
 120 #define SHMEM_SEEN_INODES 2
 121 #define SHMEM_SEEN_HUGE 4
 122 #define SHMEM_SEEN_INUMS 8
 123 };
 124
 125 #ifdef CONFIG_TMPFS
 126 static unsigned long shmem_default_max_blocks(void)
 127 {
 128         return totalram_pages() / 2;
 129 }
 130
 131 static unsigned long shmem_default_max_inodes(void)
 132 {
 133         unsigned long nr_pages = totalram_pages();
 134
 135         return min(nr_pages - totalhigh_pages(), nr_pages / 2);
 136 }
 137 #endif
 138
 139 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
 140                              struct folio **foliop, enum sgp_type sgp,
 141                              gfp_t gfp, struct vm_area_struct *vma,
 142                              vm_fault_t *fault_type);
 143
 144 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
 145 {
 146         return sb->s_fs_info;
 147 }
 148
 149 /*
 150  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 151  * for shared memory and for shared anonymous (/dev/zero) mappings
 152  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 153  * consistent with the pre-accounting of private mappings ...
 154  */
 155 static inline int shmem_acct_size(unsigned long flags, loff_t size)
 156 {
 157         return (flags & VM_NORESERVE) ?
 158                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
 159 }
 160
 161 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 162 {
 163         if (!(flags & VM_NORESERVE))
 164                 vm_unacct_memory(VM_ACCT(size));
 165 }
 166
 167 static inline int shmem_reacct_size(unsigned long flags,
 168                 loff_t oldsize, loff_t newsize)
 169 {
 170         if (!(flags & VM_NORESERVE)) {
 171                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
 172                         return security_vm_enough_memory_mm(current->mm,
 173                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
 174                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
 175                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
 176         }
 177         return 0;
 178 }
 179
 180 /*
 181  * ... whereas tmpfs objects are accounted incrementally as
 182  * pages are allocated, in order to allow large sparse files.
 183  * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
 184  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 185  */
 186 static inline int shmem_acct_block(unsigned long flags, long pages)
 187 {
 188         if (!(flags & VM_NORESERVE))
 189                 return 0;
 190
 191         return security_vm_enough_memory_mm(current->mm,
 192                         pages * VM_ACCT(PAGE_SIZE));
 193 }
 194
 195 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
 196 {
 197         if (flags & VM_NORESERVE)
 198                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
 199 }
 200
 201 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
 202 {
 203         struct shmem_inode_info *info = SHMEM_I(inode);
 204         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 205
 206         if (shmem_acct_block(info->flags, pages))
 207                 return false;
 208
 209         if (sbinfo->max_blocks) {
 210                 if (percpu_counter_compare(&sbinfo->used_blocks,
 211                                            sbinfo->max_blocks - pages) > 0)
 212                         goto unacct;
 213                 percpu_counter_add(&sbinfo->used_blocks, pages);
 214         }
 215
 216         return true;
 217
 218 unacct:
 219         shmem_unacct_blocks(info->flags, pages);
 220         return false;
 221 }
 222
 223 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
 224 {
 225         struct shmem_inode_info *info = SHMEM_I(inode);
 226         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 227
 228         if (sbinfo->max_blocks)
 229                 percpu_counter_sub(&sbinfo->used_blocks, pages);
 230         shmem_unacct_blocks(info->flags, pages);
 231 }
 232
 233 static const struct super_operations shmem_ops;
 234 const struct address_space_operations shmem_aops;
 235 static const struct file_operations shmem_file_operations;
 236 static const struct inode_operations shmem_inode_operations;
 237 static const struct inode_operations shmem_dir_inode_operations;
 238 static const struct inode_operations shmem_special_inode_operations;
 239 static const struct vm_operations_struct shmem_vm_ops;
 240 static struct file_system_type shmem_fs_type;
 241
 242 bool vma_is_shmem(struct vm_area_struct *vma)
 243 {
 244         return vma->vm_ops == &shmem_vm_ops;
 245 }
 246
 247 static LIST_HEAD(shmem_swaplist);
 248 static DEFINE_MUTEX(shmem_swaplist_mutex);
 249
 250 /*
 251  * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
 252  * produces a novel ino for the newly allocated inode.
 253  *
 254  * It may also be called when making a hard link to permit the space needed by
 255  * each dentry. However, in that case, no new inode number is needed since that
 256  * internally draws from another pool of inode numbers (currently global
 257  * get_next_ino()). This case is indicated by passing NULL as inop.
 258  */
 259 #define SHMEM_INO_BATCH 1024
 260 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
 261 {
 262         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 263         ino_t ino;
 264
 265         if (!(sb->s_flags & SB_KERNMOUNT)) {
 266                 raw_spin_lock(&sbinfo->stat_lock);
 267                 if (sbinfo->max_inodes) {
 268                         if (!sbinfo->free_inodes) {
 269                                 raw_spin_unlock(&sbinfo->stat_lock);
 270                                 return -ENOSPC;
 271                         }
 272                         sbinfo->free_inodes--;
 273                 }
 274                 if (inop) {
 275                         ino = sbinfo->next_ino++;
 276                         if (unlikely(is_zero_ino(ino)))
 277                                 ino = sbinfo->next_ino++;
 278                         if (unlikely(!sbinfo->full_inums &&
 279                                      ino > UINT_MAX)) {
 280                                 /*
 281                                  * Emulate get_next_ino uint wraparound for
 282                                  * compatibility
 283                                  */
 284                                 if (IS_ENABLED(CONFIG_64BIT))
 285                                         pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
 286                                                 __func__, MINOR(sb->s_dev));
 287                                 sbinfo->next_ino = 1;
 288                                 ino = sbinfo->next_ino++;
 289                         }
 290                         *inop = ino;
 291                 }
 292                 raw_spin_unlock(&sbinfo->stat_lock);
 293         } else if (inop) {
 294                 /*
 295                  * __shmem_file_setup, one of our callers, is lock-free: it
 296                  * doesn't hold stat_lock in shmem_reserve_inode since
 297                  * max_inodes is always 0, and is called from potentially
 298                  * unknown contexts. As such, use a per-cpu batched allocator
 299                  * which doesn't require the per-sb stat_lock unless we are at
 300                  * the batch boundary.
 301                  *
 302                  * We don't need to worry about inode{32,64} since SB_KERNMOUNT
 303                  * shmem mounts are not exposed to userspace, so we don't need
 304                  * to worry about things like glibc compatibility.
 305                  */
 306                 ino_t *next_ino;
 307
 308                 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
 309                 ino = *next_ino;
 310                 if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
 311                         raw_spin_lock(&sbinfo->stat_lock);
 312                         ino = sbinfo->next_ino;
 313                         sbinfo->next_ino += SHMEM_INO_BATCH;
 314                         raw_spin_unlock(&sbinfo->stat_lock);
 315                         if (unlikely(is_zero_ino(ino)))
 316                                 ino++;
 317                 }
 318                 *inop = ino;
 319                 *next_ino = ++ino;
 320                 put_cpu();
 321         }
 322
 323         return 0;
 324 }
 325
 326 static void shmem_free_inode(struct super_block *sb)
 327 {
 328         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 329         if (sbinfo->max_inodes) {
 330                 raw_spin_lock(&sbinfo->stat_lock);
 331                 sbinfo->free_inodes++;
 332                 raw_spin_unlock(&sbinfo->stat_lock);
 333         }
 334 }
 335
 336 /**
 337  * shmem_recalc_inode - recalculate the block usage of an inode
 338  * @inode: inode to recalc
 339  *
 340  * We have to calculate the free blocks since the mm can drop
 341  * undirtied hole pages behind our back.
 342  *
 343  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 344  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 345  *
 346  * It has to be called with the spinlock held.
 347  */
 348 static void shmem_recalc_inode(struct inode *inode)
 349 {
 350         struct shmem_inode_info *info = SHMEM_I(inode);
 351         long freed;
 352
 353         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
 354         if (freed > 0) {
 355                 info->alloced -= freed;
 356                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
 357                 shmem_inode_unacct_blocks(inode, freed);
 358         }
 359 }
 360
 361 bool shmem_charge(struct inode *inode, long pages)
 362 {
 363         struct shmem_inode_info *info = SHMEM_I(inode);
 364         unsigned long flags;
 365
 366         if (!shmem_inode_acct_block(inode, pages))
 367                 return false;
 368
 369         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
 370         inode->i_mapping->nrpages += pages;
 371
 372         spin_lock_irqsave(&info->lock, flags);
 373         info->alloced += pages;
 374         inode->i_blocks += pages * BLOCKS_PER_PAGE;
 375         shmem_recalc_inode(inode);
 376         spin_unlock_irqrestore(&info->lock, flags);
 377
 378         return true;
 379 }
 380
 381 void shmem_uncharge(struct inode *inode, long pages)
 382 {
 383         struct shmem_inode_info *info = SHMEM_I(inode);
 384         unsigned long flags;
 385
 386         /* nrpages adjustment done by __filemap_remove_folio() or caller */
 387
 388         spin_lock_irqsave(&info->lock, flags);
 389         info->alloced -= pages;
 390         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
 391         shmem_recalc_inode(inode);
 392         spin_unlock_irqrestore(&info->lock, flags);
 393
 394         shmem_inode_unacct_blocks(inode, pages);
 395 }
 396
 397 /*
 398  * Replace item expected in xarray by a new item, while holding xa_lock.
 399  */
 400 static int shmem_replace_entry(struct address_space *mapping,
 401                         pgoff_t index, void *expected, void *replacement)
 402 {
 403         XA_STATE(xas, &mapping->i_pages, index);
 404         void *item;
 405
 406         VM_BUG_ON(!expected);
 407         VM_BUG_ON(!replacement);
 408         item = xas_load(&xas);
 409         if (item != expected)
 410                 return -ENOENT;
 411         xas_store(&xas, replacement);
 412         return 0;
 413 }
 414
 415 /*
 416  * Sometimes, before we decide whether to proceed or to fail, we must check
 417  * that an entry was not already brought back from swap by a racing thread.
 418  *
 419  * Checking page is not enough: by the time a SwapCache page is locked, it
 420  * might be reused, and again be SwapCache, using the same swap as before.
 421  */
 422 static bool shmem_confirm_swap(struct address_space *mapping,
 423                                pgoff_t index, swp_entry_t swap)
 424 {
 425         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
 426 }
 427
 428 /*
 429  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
 430  *
 431  * SHMEM_HUGE_NEVER:
 432  *      disables huge pages for the mount;
 433  * SHMEM_HUGE_ALWAYS:
 434  *      enables huge pages for the mount;
 435  * SHMEM_HUGE_WITHIN_SIZE:
 436  *      only allocate huge pages if the page will be fully within i_size,
 437  *      also respect fadvise()/madvise() hints;
 438  * SHMEM_HUGE_ADVISE:
 439  *      only allocate huge pages if requested with fadvise()/madvise();
 440  */
 441
 442 #define SHMEM_HUGE_NEVER        0
 443 #define SHMEM_HUGE_ALWAYS       1
 444 #define SHMEM_HUGE_WITHIN_SIZE  2
 445 #define SHMEM_HUGE_ADVISE       3
 446
 447 /*
 448  * Special values.
 449  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
 450  *
 451  * SHMEM_HUGE_DENY:
 452  *      disables huge on shm_mnt and all mounts, for emergency use;
 453  * SHMEM_HUGE_FORCE:
 454  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
 455  *
 456  */
 457 #define SHMEM_HUGE_DENY         (-1)
 458 #define SHMEM_HUGE_FORCE        (-2)
 459
 460 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 461 /* ifdef here to avoid bloating shmem.o when not necessary */
 462
 463 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
 464
 465 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
 466                    pgoff_t index, bool shmem_huge_force)
 467 {
 468         loff_t i_size;
 469
 470         if (!S_ISREG(inode->i_mode))
 471                 return false;
 472         if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) ||
 473             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)))
 474                 return false;
 475         if (shmem_huge_force)
 476                 return true;
 477         if (shmem_huge == SHMEM_HUGE_FORCE)
 478                 return true;
 479         if (shmem_huge == SHMEM_HUGE_DENY)
 480                 return false;
 481
 482         switch (SHMEM_SB(inode->i_sb)->huge) {
 483         case SHMEM_HUGE_ALWAYS:
 484                 return true;
 485         case SHMEM_HUGE_WITHIN_SIZE:
 486                 index = round_up(index + 1, HPAGE_PMD_NR);
 487                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
 488                 if (i_size >> PAGE_SHIFT >= index)
 489                         return true;
 490                 fallthrough;
 491         case SHMEM_HUGE_ADVISE:
 492                 if (vma && (vma->vm_flags & VM_HUGEPAGE))
 493                         return true;
 494                 fallthrough;
 495         default:
 496                 return false;
 497         }
 498 }
 499
 500 #if defined(CONFIG_SYSFS)
 501 static int shmem_parse_huge(const char *str)
 502 {
 503         if (!strcmp(str, "never"))
 504                 return SHMEM_HUGE_NEVER;
 505         if (!strcmp(str, "always"))
 506                 return SHMEM_HUGE_ALWAYS;
 507         if (!strcmp(str, "within_size"))
 508                 return SHMEM_HUGE_WITHIN_SIZE;
 509         if (!strcmp(str, "advise"))
 510                 return SHMEM_HUGE_ADVISE;
 511         if (!strcmp(str, "deny"))
 512                 return SHMEM_HUGE_DENY;
 513         if (!strcmp(str, "force"))
 514                 return SHMEM_HUGE_FORCE;
 515         return -EINVAL;
 516 }
 517 #endif
 518
 519 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
 520 static const char *shmem_format_huge(int huge)
 521 {
 522         switch (huge) {
 523         case SHMEM_HUGE_NEVER:
 524                 return "never";
 525         case SHMEM_HUGE_ALWAYS:
 526                 return "always";
 527         case SHMEM_HUGE_WITHIN_SIZE:
 528                 return "within_size";
 529         case SHMEM_HUGE_ADVISE:
 530                 return "advise";
 531         case SHMEM_HUGE_DENY:
 532                 return "deny";
 533         case SHMEM_HUGE_FORCE:
 534                 return "force";
 535         default:
 536                 VM_BUG_ON(1);
 537                 return "bad_val";
 538         }
 539 }
 540 #endif
 541
 542 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 543                 struct shrink_control *sc, unsigned long nr_to_split)
 544 {
 545         LIST_HEAD(list), *pos, *next;
 546         LIST_HEAD(to_remove);
 547         struct inode *inode;
 548         struct shmem_inode_info *info;
 549         struct folio *folio;
 550         unsigned long batch = sc ? sc->nr_to_scan : 128;
 551         int split = 0;
 552
 553         if (list_empty(&sbinfo->shrinklist))
 554                 return SHRINK_STOP;
 555
 556         spin_lock(&sbinfo->shrinklist_lock);
 557         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
 558                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 559
 560                 /* pin the inode */
 561                 inode = igrab(&info->vfs_inode);
 562
 563                 /* inode is about to be evicted */
 564                 if (!inode) {
 565                         list_del_init(&info->shrinklist);
 566                         goto next;
 567                 }
 568
 569                 /* Check if there's anything to gain */
 570                 if (round_up(inode->i_size, PAGE_SIZE) ==
 571                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
 572                         list_move(&info->shrinklist, &to_remove);
 573                         goto next;
 574                 }
 575
 576                 list_move(&info->shrinklist, &list);
 577 next:
 578                 sbinfo->shrinklist_len--;
 579                 if (!--batch)
 580                         break;
 581         }
 582         spin_unlock(&sbinfo->shrinklist_lock);
 583
 584         list_for_each_safe(pos, next, &to_remove) {
 585                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 586                 inode = &info->vfs_inode;
 587                 list_del_init(&info->shrinklist);
 588                 iput(inode);
 589         }
 590
 591         list_for_each_safe(pos, next, &list) {
 592                 int ret;
 593                 pgoff_t index;
 594
 595                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 596                 inode = &info->vfs_inode;
 597
 598                 if (nr_to_split && split >= nr_to_split)
 599                         goto move_back;
 600
 601                 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
 602                 folio = filemap_get_folio(inode->i_mapping, index);
 603                 if (!folio)
 604                         goto drop;
 605
 606                 /* No huge page at the end of the file: nothing to split */
 607                 if (!folio_test_large(folio)) {
 608                         folio_put(folio);
 609                         goto drop;
 610                 }
 611
 612                 /*
 613                  * Move the inode on the list back to shrinklist if we failed
 614                  * to lock the page at this time.
 615                  *
 616                  * Waiting for the lock may lead to deadlock in the
 617                  * reclaim path.
 618                  */
 619                 if (!folio_trylock(folio)) {
 620                         folio_put(folio);
 621                         goto move_back;
 622                 }
 623
 624                 ret = split_folio(folio);
 625                 folio_unlock(folio);
 626                 folio_put(folio);
 627
 628                 /* If split failed move the inode on the list back to shrinklist */
 629                 if (ret)
 630                         goto move_back;
 631
 632                 split++;
 633 drop:
 634                 list_del_init(&info->shrinklist);
 635                 goto put;
 636 move_back:
 637                 /*
 638                  * Make sure the inode is either on the global list or deleted
 639                  * from any local list before iput() since it could be deleted
 640                  * in another thread once we put the inode (then the local list
 641                  * is corrupted).
 642                  */
 643                 spin_lock(&sbinfo->shrinklist_lock);
 644                 list_move(&info->shrinklist, &sbinfo->shrinklist);
 645                 sbinfo->shrinklist_len++;
 646                 spin_unlock(&sbinfo->shrinklist_lock);
 647 put:
 648                 iput(inode);
 649         }
 650
 651         return split;
 652 }
 653
 654 static long shmem_unused_huge_scan(struct super_block *sb,
 655                 struct shrink_control *sc)
 656 {
 657         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 658
 659         if (!READ_ONCE(sbinfo->shrinklist_len))
 660                 return SHRINK_STOP;
 661
 662         return shmem_unused_huge_shrink(sbinfo, sc, 0);
 663 }
 664
 665 static long shmem_unused_huge_count(struct super_block *sb,
 666                 struct shrink_control *sc)
 667 {
 668         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 669         return READ_ONCE(sbinfo->shrinklist_len);
 670 }
 671 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
 672
 673 #define shmem_huge SHMEM_HUGE_DENY
 674
 675 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
 676                    pgoff_t index, bool shmem_huge_force)
 677 {
 678         return false;
 679 }
 680
 681 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 682                 struct shrink_control *sc, unsigned long nr_to_split)
 683 {
 684         return 0;
 685 }
 686 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 687
 688 /*
 689  * Like filemap_add_folio, but error if expected item has gone.
 690  */
 691 static int shmem_add_to_page_cache(struct folio *folio,
 692                                    struct address_space *mapping,
 693                                    pgoff_t index, void *expected, gfp_t gfp,
 694                                    struct mm_struct *charge_mm)
 695 {
 696         XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
 697         long nr = folio_nr_pages(folio);
 698         int error;
 699
 700         VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
 701         VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 702         VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
 703         VM_BUG_ON(expected && folio_test_large(folio));
 704
 705         folio_ref_add(folio, nr);
 706         folio->mapping = mapping;
 707         folio->index = index;
 708
 709         if (!folio_test_swapcache(folio)) {
 710                 error = mem_cgroup_charge(folio, charge_mm, gfp);
 711                 if (error) {
 712                         if (folio_test_pmd_mappable(folio)) {
 713                                 count_vm_event(THP_FILE_FALLBACK);
 714                                 count_vm_event(THP_FILE_FALLBACK_CHARGE);
 715                         }
 716                         goto error;
 717                 }
 718         }
 719         folio_throttle_swaprate(folio, gfp);
 720
 721         do {
 722                 xas_lock_irq(&xas);
 723                 if (expected != xas_find_conflict(&xas)) {
 724                         xas_set_err(&xas, -EEXIST);
 725                         goto unlock;
 726                 }
 727                 if (expected && xas_find_conflict(&xas)) {
 728                         xas_set_err(&xas, -EEXIST);
 729                         goto unlock;
 730                 }
 731                 xas_store(&xas, folio);
 732                 if (xas_error(&xas))
 733                         goto unlock;
 734                 if (folio_test_pmd_mappable(folio)) {
 735                         count_vm_event(THP_FILE_ALLOC);
 736                         __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
 737                 }
 738                 mapping->nrpages += nr;
 739                 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
 740                 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
 741 unlock:
 742                 xas_unlock_irq(&xas);
 743         } while (xas_nomem(&xas, gfp));
 744
 745         if (xas_error(&xas)) {
 746                 error = xas_error(&xas);
 747                 goto error;
 748         }
 749
 750         return 0;
 751 error:
 752         folio->mapping = NULL;
 753         folio_ref_sub(folio, nr);
 754         return error;
 755 }
 756
 757 /*
 758  * Like delete_from_page_cache, but substitutes swap for @folio.
 759  */
 760 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
 761 {
 762         struct address_space *mapping = folio->mapping;
 763         long nr = folio_nr_pages(folio);
 764         int error;
 765
 766         xa_lock_irq(&mapping->i_pages);
 767         error = shmem_replace_entry(mapping, folio->index, folio, radswap);
 768         folio->mapping = NULL;
 769         mapping->nrpages -= nr;
 770         __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
 771         __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
 772         xa_unlock_irq(&mapping->i_pages);
 773         folio_put(folio);
 774         BUG_ON(error);
 775 }
 776
 777 /*
 778  * Remove swap entry from page cache, free the swap and its page cache.
 779  */
 780 static int shmem_free_swap(struct address_space *mapping,
 781                            pgoff_t index, void *radswap)
 782 {
 783         void *old;
 784
 785         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
 786         if (old != radswap)
 787                 return -ENOENT;
 788         free_swap_and_cache(radix_to_swp_entry(radswap));
 789         return 0;
 790 }
 791
 792 /*
 793  * Determine (in bytes) how many of the shmem object's pages mapped by the
 794  * given offsets are swapped out.
 795  *
 796  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
 797  * as long as the inode doesn't go away and racy results are not a problem.
 798  */
 799 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 800                                                 pgoff_t start, pgoff_t end)
 801 {
 802         XA_STATE(xas, &mapping->i_pages, start);
 803         struct page *page;
 804         unsigned long swapped = 0;
 805
 806         rcu_read_lock();
 807         xas_for_each(&xas, page, end - 1) {
 808                 if (xas_retry(&xas, page))
 809                         continue;
 810                 if (xa_is_value(page))
 811                         swapped++;
 812
 813                 if (need_resched()) {
 814                         xas_pause(&xas);
 815                         cond_resched_rcu();
 816                 }
 817         }
 818
 819         rcu_read_unlock();
 820
 821         return swapped << PAGE_SHIFT;
 822 }
 823
 824 /*
 825  * Determine (in bytes) how many of the shmem object's pages mapped by the
 826  * given vma is swapped out.
 827  *
 828  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
 829  * as long as the inode doesn't go away and racy results are not a problem.
 830  */
 831 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
 832 {
 833         struct inode *inode = file_inode(vma->vm_file);
 834         struct shmem_inode_info *info = SHMEM_I(inode);
 835         struct address_space *mapping = inode->i_mapping;
 836         unsigned long swapped;
 837
 838         /* Be careful as we don't hold info->lock */
 839         swapped = READ_ONCE(info->swapped);
 840
 841         /*
 842          * The easier cases are when the shmem object has nothing in swap, or
 843          * the vma maps it whole. Then we can simply use the stats that we
 844          * already track.
 845          */
 846         if (!swapped)
 847                 return 0;
 848
 849         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
 850                 return swapped << PAGE_SHIFT;
 851
 852         /* Here comes the more involved part */
 853         return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
 854                                         vma->vm_pgoff + vma_pages(vma));
 855 }
 856
 857 /*
 858  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
 859  */
 860 void shmem_unlock_mapping(struct address_space *mapping)
 861 {
 862         struct folio_batch fbatch;
 863         pgoff_t index = 0;
 864
 865         folio_batch_init(&fbatch);
 866         /*
 867          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
 868          */
 869         while (!mapping_unevictable(mapping) &&
 870                filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
 871                 check_move_unevictable_folios(&fbatch);
 872                 folio_batch_release(&fbatch);
 873                 cond_resched();
 874         }
 875 }
 876
 877 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
 878 {
 879         struct folio *folio;
 880
 881         /*
 882          * At first avoid shmem_get_folio(,,,SGP_READ): that fails
 883          * beyond i_size, and reports fallocated pages as holes.
 884          */
 885         folio = __filemap_get_folio(inode->i_mapping, index,
 886                                         FGP_ENTRY | FGP_LOCK, 0);
 887         if (!xa_is_value(folio))
 888                 return folio;
 889         /*
 890          * But read a page back from swap if any of it is within i_size
 891          * (although in some cases this is just a waste of time).
 892          */
 893         folio = NULL;
 894         shmem_get_folio(inode, index, &folio, SGP_READ);
 895         return folio;
 896 }
 897
 898 /*
 899  * Remove range of pages and swap entries from page cache, and free them.
 900  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
 901  */
 902 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
 903                                                                  bool unfalloc)
 904 {
 905         struct address_space *mapping = inode->i_mapping;
 906         struct shmem_inode_info *info = SHMEM_I(inode);
 907         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
 908         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
 909         struct folio_batch fbatch;
 910         pgoff_t indices[PAGEVEC_SIZE];
 911         struct folio *folio;
 912         bool same_folio;
 913         long nr_swaps_freed = 0;
 914         pgoff_t index;
 915         int i;
 916
 917         if (lend == -1)
 918                 end = -1;       /* unsigned, so actually very big */
 919
 920         if (info->fallocend > start && info->fallocend <= end && !unfalloc)
 921                 info->fallocend = start;
 922
 923         folio_batch_init(&fbatch);
 924         index = start;
 925         while (index < end && find_lock_entries(mapping, index, end - 1,
 926                         &fbatch, indices)) {
 927                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 928                         folio = fbatch.folios[i];
 929
 930                         index = indices[i];
 931
 932                         if (xa_is_value(folio)) {
 933                                 if (unfalloc)
 934                                         continue;
 935                                 nr_swaps_freed += !shmem_free_swap(mapping,
 936                                                                 index, folio);
 937                                 continue;
 938                         }
 939                         index += folio_nr_pages(folio) - 1;
 940
 941                         if (!unfalloc || !folio_test_uptodate(folio))
 942                                 truncate_inode_folio(mapping, folio);
 943                         folio_unlock(folio);
 944                 }
 945                 folio_batch_remove_exceptionals(&fbatch);
 946                 folio_batch_release(&fbatch);
 947                 cond_resched();
 948                 index++;
 949         }
 950
 951         same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
 952         folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
 953         if (folio) {
 954                 same_folio = lend < folio_pos(folio) + folio_size(folio);
 955                 folio_mark_dirty(folio);
 956                 if (!truncate_inode_partial_folio(folio, lstart, lend)) {
 957                         start = folio->index + folio_nr_pages(folio);
 958                         if (same_folio)
 959                                 end = folio->index;
 960                 }
 961                 folio_unlock(folio);
 962                 folio_put(folio);
 963                 folio = NULL;
 964         }
 965
 966         if (!same_folio)
 967                 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
 968         if (folio) {
 969                 folio_mark_dirty(folio);
 970                 if (!truncate_inode_partial_folio(folio, lstart, lend))
 971                         end = folio->index;
 972                 folio_unlock(folio);
 973                 folio_put(folio);
 974         }
 975
 976         index = start;
 977         while (index < end) {
 978                 cond_resched();
 979
 980                 if (!find_get_entries(mapping, index, end - 1, &fbatch,
 981                                 indices)) {
 982                         /* If all gone or hole-punch or unfalloc, we're done */
 983                         if (index == start || end != -1)
 984                                 break;
 985                         /* But if truncating, restart to make sure all gone */
 986                         index = start;
 987                         continue;
 988                 }
 989                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 990                         folio = fbatch.folios[i];
 991
 992                         index = indices[i];
 993                         if (xa_is_value(folio)) {
 994                                 if (unfalloc)
 995                                         continue;
 996                                 if (shmem_free_swap(mapping, index, folio)) {
 997                                         /* Swap was replaced by page: retry */
 998                                         index--;
 999                                         break;
1000                                 }
1001                                 nr_swaps_freed++;
1002                                 continue;
1003                         }
1004
1005                         folio_lock(folio);
1006
1007                         if (!unfalloc || !folio_test_uptodate(folio)) {
1008                                 if (folio_mapping(folio) != mapping) {
1009                                         /* Page was replaced by swap: retry */
1010                                         folio_unlock(folio);
1011                                         index--;
1012                                         break;
1013                                 }
1014                                 VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1015                                                 folio);
1016                                 truncate_inode_folio(mapping, folio);
1017                         }
1018                         index = folio->index + folio_nr_pages(folio) - 1;
1019                         folio_unlock(folio);
1020                 }
1021                 folio_batch_remove_exceptionals(&fbatch);
1022                 folio_batch_release(&fbatch);
1023                 index++;
1024         }
1025
1026         spin_lock_irq(&info->lock);
1027         info->swapped -= nr_swaps_freed;
1028         shmem_recalc_inode(inode);
1029         spin_unlock_irq(&info->lock);
1030 }
1031
1032 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1033 {
1034         shmem_undo_range(inode, lstart, lend, false);
1035         inode->i_ctime = inode->i_mtime = current_time(inode);
1036         inode_inc_iversion(inode);
1037 }
1038 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1039
1040 static int shmem_getattr(struct user_namespace *mnt_userns,
1041                          const struct path *path, struct kstat *stat,
1042                          u32 request_mask, unsigned int query_flags)
1043 {
1044         struct inode *inode = path->dentry->d_inode;
1045         struct shmem_inode_info *info = SHMEM_I(inode);
1046
1047         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1048                 spin_lock_irq(&info->lock);
1049                 shmem_recalc_inode(inode);
1050                 spin_unlock_irq(&info->lock);
1051         }
1052         if (info->fsflags & FS_APPEND_FL)
1053                 stat->attributes |= STATX_ATTR_APPEND;
1054         if (info->fsflags & FS_IMMUTABLE_FL)
1055                 stat->attributes |= STATX_ATTR_IMMUTABLE;
1056         if (info->fsflags & FS_NODUMP_FL)
1057                 stat->attributes |= STATX_ATTR_NODUMP;
1058         stat->attributes_mask |= (STATX_ATTR_APPEND |
1059                         STATX_ATTR_IMMUTABLE |
1060                         STATX_ATTR_NODUMP);
1061         generic_fillattr(&init_user_ns, inode, stat);
1062
1063         if (shmem_is_huge(NULL, inode, 0, false))
1064                 stat->blksize = HPAGE_PMD_SIZE;
1065
1066         if (request_mask & STATX_BTIME) {
1067                 stat->result_mask |= STATX_BTIME;
1068                 stat->btime.tv_sec = info->i_crtime.tv_sec;
1069                 stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1070         }
1071
1072         return 0;
1073 }
1074
1075 static int shmem_setattr(struct user_namespace *mnt_userns,
1076                          struct dentry *dentry, struct iattr *attr)
1077 {
1078         struct inode *inode = d_inode(dentry);
1079         struct shmem_inode_info *info = SHMEM_I(inode);
1080         int error;
1081         bool update_mtime = false;
1082         bool update_ctime = true;
1083
1084         error = setattr_prepare(&init_user_ns, dentry, attr);
1085         if (error)
1086                 return error;
1087
1088         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1089                 loff_t oldsize = inode->i_size;
1090                 loff_t newsize = attr->ia_size;
1091
1092                 /* protected by i_rwsem */
1093                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1094                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1095                         return -EPERM;
1096
1097                 if (newsize != oldsize) {
1098                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1099                                         oldsize, newsize);
1100                         if (error)
1101                                 return error;
1102                         i_size_write(inode, newsize);
1103                         update_mtime = true;
1104                 } else {
1105                         update_ctime = false;
1106                 }
1107                 if (newsize <= oldsize) {
1108                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1109                         if (oldsize > holebegin)
1110                                 unmap_mapping_range(inode->i_mapping,
1111                                                         holebegin, 0, 1);
1112                         if (info->alloced)
1113                                 shmem_truncate_range(inode,
1114                                                         newsize, (loff_t)-1);
1115                         /* unmap again to remove racily COWed private pages */
1116                         if (oldsize > holebegin)
1117                                 unmap_mapping_range(inode->i_mapping,
1118                                                         holebegin, 0, 1);
1119                 }
1120         }
1121
1122         setattr_copy(&init_user_ns, inode, attr);
1123         if (attr->ia_valid & ATTR_MODE)
1124                 error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode);
1125         if (!error && update_ctime) {
1126                 inode->i_ctime = current_time(inode);
1127                 if (update_mtime)
1128                         inode->i_mtime = inode->i_ctime;
1129                 inode_inc_iversion(inode);
1130         }
1131         return error;
1132 }
1133
1134 static void shmem_evict_inode(struct inode *inode)
1135 {
1136         struct shmem_inode_info *info = SHMEM_I(inode);
1137         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1138
1139         if (shmem_mapping(inode->i_mapping)) {
1140                 shmem_unacct_size(info->flags, inode->i_size);
1141                 inode->i_size = 0;
1142                 mapping_set_exiting(inode->i_mapping);
1143                 shmem_truncate_range(inode, 0, (loff_t)-1);
1144                 if (!list_empty(&info->shrinklist)) {
1145                         spin_lock(&sbinfo->shrinklist_lock);
1146                         if (!list_empty(&info->shrinklist)) {
1147                                 list_del_init(&info->shrinklist);
1148                                 sbinfo->shrinklist_len--;
1149                         }
1150                         spin_unlock(&sbinfo->shrinklist_lock);
1151                 }
1152                 while (!list_empty(&info->swaplist)) {
1153                         /* Wait while shmem_unuse() is scanning this inode... */
1154                         wait_var_event(&info->stop_eviction,
1155                                        !atomic_read(&info->stop_eviction));
1156                         mutex_lock(&shmem_swaplist_mutex);
1157                         /* ...but beware of the race if we peeked too early */
1158                         if (!atomic_read(&info->stop_eviction))
1159                                 list_del_init(&info->swaplist);
1160                         mutex_unlock(&shmem_swaplist_mutex);
1161                 }
1162         }
1163
1164         simple_xattrs_free(&info->xattrs);
1165         WARN_ON(inode->i_blocks);
1166         shmem_free_inode(inode->i_sb);
1167         clear_inode(inode);
1168 }
1169
1170 static int shmem_find_swap_entries(struct address_space *mapping,
1171                                    pgoff_t start, struct folio_batch *fbatch,
1172                                    pgoff_t *indices, unsigned int type)
1173 {
1174         XA_STATE(xas, &mapping->i_pages, start);
1175         struct folio *folio;
1176         swp_entry_t entry;
1177
1178         rcu_read_lock();
1179         xas_for_each(&xas, folio, ULONG_MAX) {
1180                 if (xas_retry(&xas, folio))
1181                         continue;
1182
1183                 if (!xa_is_value(folio))
1184                         continue;
1185
1186                 entry = radix_to_swp_entry(folio);
1187                 /*
1188                  * swapin error entries can be found in the mapping. But they're
1189                  * deliberately ignored here as we've done everything we can do.
1190                  */
1191                 if (swp_type(entry) != type)
1192                         continue;
1193
1194                 indices[folio_batch_count(fbatch)] = xas.xa_index;
1195                 if (!folio_batch_add(fbatch, folio))
1196                         break;
1197
1198                 if (need_resched()) {
1199                         xas_pause(&xas);
1200                         cond_resched_rcu();
1201                 }
1202         }
1203         rcu_read_unlock();
1204
1205         return xas.xa_index;
1206 }
1207
1208 /*
1209  * Move the swapped pages for an inode to page cache. Returns the count
1210  * of pages swapped in, or the error in case of failure.
1211  */
1212 static int shmem_unuse_swap_entries(struct inode *inode,
1213                 struct folio_batch *fbatch, pgoff_t *indices)
1214 {
1215         int i = 0;
1216         int ret = 0;
1217         int error = 0;
1218         struct address_space *mapping = inode->i_mapping;
1219
1220         for (i = 0; i < folio_batch_count(fbatch); i++) {
1221                 struct folio *folio = fbatch->folios[i];
1222
1223                 if (!xa_is_value(folio))
1224                         continue;
1225                 error = shmem_swapin_folio(inode, indices[i],
1226                                           &folio, SGP_CACHE,
1227                                           mapping_gfp_mask(mapping),
1228                                           NULL, NULL);
1229                 if (error == 0) {
1230                         folio_unlock(folio);
1231                         folio_put(folio);
1232                         ret++;
1233                 }
1234                 if (error == -ENOMEM)
1235                         break;
1236                 error = 0;
1237         }
1238         return error ? error : ret;
1239 }
1240
1241 /*
1242  * If swap found in inode, free it and move page from swapcache to filecache.
1243  */
1244 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1245 {
1246         struct address_space *mapping = inode->i_mapping;
1247         pgoff_t start = 0;
1248         struct folio_batch fbatch;
1249         pgoff_t indices[PAGEVEC_SIZE];
1250         int ret = 0;
1251
1252         do {
1253                 folio_batch_init(&fbatch);
1254                 shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1255                 if (folio_batch_count(&fbatch) == 0) {
1256                         ret = 0;
1257                         break;
1258                 }
1259
1260                 ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1261                 if (ret < 0)
1262                         break;
1263
1264                 start = indices[folio_batch_count(&fbatch) - 1];
1265         } while (true);
1266
1267         return ret;
1268 }
1269
1270 /*
1271  * Read all the shared memory data that resides in the swap
1272  * device 'type' back into memory, so the swap device can be
1273  * unused.
1274  */
1275 int shmem_unuse(unsigned int type)
1276 {
1277         struct shmem_inode_info *info, *next;
1278         int error = 0;
1279
1280         if (list_empty(&shmem_swaplist))
1281                 return 0;
1282
1283         mutex_lock(&shmem_swaplist_mutex);
1284         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1285                 if (!info->swapped) {
1286                         list_del_init(&info->swaplist);
1287                         continue;
1288                 }
1289                 /*
1290                  * Drop the swaplist mutex while searching the inode for swap;
1291                  * but before doing so, make sure shmem_evict_inode() will not
1292                  * remove placeholder inode from swaplist, nor let it be freed
1293                  * (igrab() would protect from unlink, but not from unmount).
1294                  */
1295                 atomic_inc(&info->stop_eviction);
1296                 mutex_unlock(&shmem_swaplist_mutex);
1297
1298                 error = shmem_unuse_inode(&info->vfs_inode, type);
1299                 cond_resched();
1300
1301                 mutex_lock(&shmem_swaplist_mutex);
1302                 next = list_next_entry(info, swaplist);
1303                 if (!info->swapped)
1304                         list_del_init(&info->swaplist);
1305                 if (atomic_dec_and_test(&info->stop_eviction))
1306                         wake_up_var(&info->stop_eviction);
1307                 if (error)
1308                         break;
1309         }
1310         mutex_unlock(&shmem_swaplist_mutex);
1311
1312         return error;
1313 }
1314
1315 /*
1316  * Move the page from the page cache to the swap cache.
1317  */
1318 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1319 {
1320         struct folio *folio = page_folio(page);
1321         struct shmem_inode_info *info;
1322         struct address_space *mapping;
1323         struct inode *inode;
1324         swp_entry_t swap;
1325         pgoff_t index;
1326
1327         /*
1328          * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1329          * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1330          * and its shmem_writeback() needs them to be split when swapping.
1331          */
1332         if (folio_test_large(folio)) {
1333                 /* Ensure the subpages are still dirty */
1334                 folio_test_set_dirty(folio);
1335                 if (split_huge_page(page) < 0)
1336                         goto redirty;
1337                 folio = page_folio(page);
1338                 folio_clear_dirty(folio);
1339         }
1340
1341         BUG_ON(!folio_test_locked(folio));
1342         mapping = folio->mapping;
1343         index = folio->index;
1344         inode = mapping->host;
1345         info = SHMEM_I(inode);
1346         if (info->flags & VM_LOCKED)
1347                 goto redirty;
1348         if (!total_swap_pages)
1349                 goto redirty;
1350
1351         /*
1352          * Our capabilities prevent regular writeback or sync from ever calling
1353          * shmem_writepage; but a stacking filesystem might use ->writepage of
1354          * its underlying filesystem, in which case tmpfs should write out to
1355          * swap only in response to memory pressure, and not for the writeback
1356          * threads or sync.
1357          */
1358         if (!wbc->for_reclaim) {
1359                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1360                 goto redirty;
1361         }
1362
1363         /*
1364          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1365          * value into swapfile.c, the only way we can correctly account for a
1366          * fallocated folio arriving here is now to initialize it and write it.
1367          *
1368          * That's okay for a folio already fallocated earlier, but if we have
1369          * not yet completed the fallocation, then (a) we want to keep track
1370          * of this folio in case we have to undo it, and (b) it may not be a
1371          * good idea to continue anyway, once we're pushing into swap.  So
1372          * reactivate the folio, and let shmem_fallocate() quit when too many.
1373          */
1374         if (!folio_test_uptodate(folio)) {
1375                 if (inode->i_private) {
1376                         struct shmem_falloc *shmem_falloc;
1377                         spin_lock(&inode->i_lock);
1378                         shmem_falloc = inode->i_private;
1379                         if (shmem_falloc &&
1380                             !shmem_falloc->waitq &&
1381                             index >= shmem_falloc->start &&
1382                             index < shmem_falloc->next)
1383                                 shmem_falloc->nr_unswapped++;
1384                         else
1385                                 shmem_falloc = NULL;
1386                         spin_unlock(&inode->i_lock);
1387                         if (shmem_falloc)
1388                                 goto redirty;
1389                 }
1390                 folio_zero_range(folio, 0, folio_size(folio));
1391                 flush_dcache_folio(folio);
1392                 folio_mark_uptodate(folio);
1393         }
1394
1395         swap = folio_alloc_swap(folio);
1396         if (!swap.val)
1397                 goto redirty;
1398
1399         /*
1400          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1401          * if it's not already there.  Do it now before the folio is
1402          * moved to swap cache, when its pagelock no longer protects
1403          * the inode from eviction.  But don't unlock the mutex until
1404          * we've incremented swapped, because shmem_unuse_inode() will
1405          * prune a !swapped inode from the swaplist under this mutex.
1406          */
1407         mutex_lock(&shmem_swaplist_mutex);
1408         if (list_empty(&info->swaplist))
1409                 list_add(&info->swaplist, &shmem_swaplist);
1410
1411         if (add_to_swap_cache(folio, swap,
1412                         __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1413                         NULL) == 0) {
1414                 spin_lock_irq(&info->lock);
1415                 shmem_recalc_inode(inode);
1416                 info->swapped++;
1417                 spin_unlock_irq(&info->lock);
1418
1419                 swap_shmem_alloc(swap);
1420                 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1421
1422                 mutex_unlock(&shmem_swaplist_mutex);
1423                 BUG_ON(folio_mapped(folio));
1424                 swap_writepage(&folio->page, wbc);
1425                 return 0;
1426         }
1427
1428         mutex_unlock(&shmem_swaplist_mutex);
1429         put_swap_folio(folio, swap);
1430 redirty:
1431         folio_mark_dirty(folio);
1432         if (wbc->for_reclaim)
1433                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with folio locked */
1434         folio_unlock(folio);
1435         return 0;
1436 }
1437
1438 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1439 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1440 {
1441         char buffer[64];
1442
1443         if (!mpol || mpol->mode == MPOL_DEFAULT)
1444                 return;         /* show nothing */
1445
1446         mpol_to_str(buffer, sizeof(buffer), mpol);
1447
1448         seq_printf(seq, ",mpol=%s", buffer);
1449 }
1450
1451 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1452 {
1453         struct mempolicy *mpol = NULL;
1454         if (sbinfo->mpol) {
1455                 raw_spin_lock(&sbinfo->stat_lock);      /* prevent replace/use races */
1456                 mpol = sbinfo->mpol;
1457                 mpol_get(mpol);
1458                 raw_spin_unlock(&sbinfo->stat_lock);
1459         }
1460         return mpol;
1461 }
1462 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1463 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1464 {
1465 }
1466 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1467 {
1468         return NULL;
1469 }
1470 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1471 #ifndef CONFIG_NUMA
1472 #define vm_policy vm_private_data
1473 #endif
1474
1475 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1476                 struct shmem_inode_info *info, pgoff_t index)
1477 {
1478         /* Create a pseudo vma that just contains the policy */
1479         vma_init(vma, NULL);
1480         /* Bias interleave by inode number to distribute better across nodes */
1481         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1482         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1483 }
1484
1485 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1486 {
1487         /* Drop reference taken by mpol_shared_policy_lookup() */
1488         mpol_cond_put(vma->vm_policy);
1489 }
1490
1491 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1492                         struct shmem_inode_info *info, pgoff_t index)
1493 {
1494         struct vm_area_struct pvma;
1495         struct page *page;
1496         struct vm_fault vmf = {
1497                 .vma = &pvma,
1498         };
1499
1500         shmem_pseudo_vma_init(&pvma, info, index);
1501         page = swap_cluster_readahead(swap, gfp, &vmf);
1502         shmem_pseudo_vma_destroy(&pvma);
1503
1504         if (!page)
1505                 return NULL;
1506         return page_folio(page);
1507 }
1508
1509 /*
1510  * Make sure huge_gfp is always more limited than limit_gfp.
1511  * Some of the flags set permissions, while others set limitations.
1512  */
1513 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1514 {
1515         gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1516         gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1517         gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1518         gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1519
1520         /* Allow allocations only from the originally specified zones. */
1521         result |= zoneflags;
1522
1523         /*
1524          * Minimize the result gfp by taking the union with the deny flags,
1525          * and the intersection of the allow flags.
1526          */
1527         result |= (limit_gfp & denyflags);
1528         result |= (huge_gfp & limit_gfp) & allowflags;
1529
1530         return result;
1531 }
1532
1533 static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1534                 struct shmem_inode_info *info, pgoff_t index)
1535 {
1536         struct vm_area_struct pvma;
1537         struct address_space *mapping = info->vfs_inode.i_mapping;
1538         pgoff_t hindex;
1539         struct folio *folio;
1540
1541         hindex = round_down(index, HPAGE_PMD_NR);
1542         if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1543                                                                 XA_PRESENT))
1544                 return NULL;
1545
1546         shmem_pseudo_vma_init(&pvma, info, hindex);
1547         folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1548         shmem_pseudo_vma_destroy(&pvma);
1549         if (!folio)
1550                 count_vm_event(THP_FILE_FALLBACK);
1551         return folio;
1552 }
1553
1554 static struct folio *shmem_alloc_folio(gfp_t gfp,
1555                         struct shmem_inode_info *info, pgoff_t index)
1556 {
1557         struct vm_area_struct pvma;
1558         struct folio *folio;
1559
1560         shmem_pseudo_vma_init(&pvma, info, index);
1561         folio = vma_alloc_folio(gfp, 0, &pvma, 0, false);
1562         shmem_pseudo_vma_destroy(&pvma);
1563
1564         return folio;
1565 }
1566
1567 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
1568                 pgoff_t index, bool huge)
1569 {
1570         struct shmem_inode_info *info = SHMEM_I(inode);
1571         struct folio *folio;
1572         int nr;
1573         int err = -ENOSPC;
1574
1575         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1576                 huge = false;
1577         nr = huge ? HPAGE_PMD_NR : 1;
1578
1579         if (!shmem_inode_acct_block(inode, nr))
1580                 goto failed;
1581
1582         if (huge)
1583                 folio = shmem_alloc_hugefolio(gfp, info, index);
1584         else
1585                 folio = shmem_alloc_folio(gfp, info, index);
1586         if (folio) {
1587                 __folio_set_locked(folio);
1588                 __folio_set_swapbacked(folio);
1589                 return folio;
1590         }
1591
1592         err = -ENOMEM;
1593         shmem_inode_unacct_blocks(inode, nr);
1594 failed:
1595         return ERR_PTR(err);
1596 }
1597
1598 /*
1599  * When a page is moved from swapcache to shmem filecache (either by the
1600  * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1601  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1602  * ignorance of the mapping it belongs to.  If that mapping has special
1603  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1604  * we may need to copy to a suitable page before moving to filecache.
1605  *
1606  * In a future release, this may well be extended to respect cpuset and
1607  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1608  * but for now it is a simple matter of zone.
1609  */
1610 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1611 {
1612         return folio_zonenum(folio) > gfp_zone(gfp);
1613 }
1614
1615 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1616                                 struct shmem_inode_info *info, pgoff_t index)
1617 {
1618         struct folio *old, *new;
1619         struct address_space *swap_mapping;
1620         swp_entry_t entry;
1621         pgoff_t swap_index;
1622         int error;
1623
1624         old = *foliop;
1625         entry = folio_swap_entry(old);
1626         swap_index = swp_offset(entry);
1627         swap_mapping = swap_address_space(entry);
1628
1629         /*
1630          * We have arrived here because our zones are constrained, so don't
1631          * limit chance of success by further cpuset and node constraints.
1632          */
1633         gfp &= ~GFP_CONSTRAINT_MASK;
1634         VM_BUG_ON_FOLIO(folio_test_large(old), old);
1635         new = shmem_alloc_folio(gfp, info, index);
1636         if (!new)
1637                 return -ENOMEM;
1638
1639         folio_get(new);
1640         folio_copy(new, old);
1641         flush_dcache_folio(new);
1642
1643         __folio_set_locked(new);
1644         __folio_set_swapbacked(new);
1645         folio_mark_uptodate(new);
1646         folio_set_swap_entry(new, entry);
1647         folio_set_swapcache(new);
1648
1649         /*
1650          * Our caller will very soon move newpage out of swapcache, but it's
1651          * a nice clean interface for us to replace oldpage by newpage there.
1652          */
1653         xa_lock_irq(&swap_mapping->i_pages);
1654         error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1655         if (!error) {
1656                 mem_cgroup_migrate(old, new);
1657                 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1658                 __lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1659                 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1660                 __lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1661         }
1662         xa_unlock_irq(&swap_mapping->i_pages);
1663
1664         if (unlikely(error)) {
1665                 /*
1666                  * Is this possible?  I think not, now that our callers check
1667                  * both PageSwapCache and page_private after getting page lock;
1668                  * but be defensive.  Reverse old to newpage for clear and free.
1669                  */
1670                 old = new;
1671         } else {
1672                 folio_add_lru(new);
1673                 *foliop = new;
1674         }
1675
1676         folio_clear_swapcache(old);
1677         old->private = NULL;
1678
1679         folio_unlock(old);
1680         folio_put_refs(old, 2);
1681         return error;
1682 }
1683
1684 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1685                                          struct folio *folio, swp_entry_t swap)
1686 {
1687         struct address_space *mapping = inode->i_mapping;
1688         struct shmem_inode_info *info = SHMEM_I(inode);
1689         swp_entry_t swapin_error;
1690         void *old;
1691
1692         swapin_error = make_swapin_error_entry(&folio->page);
1693         old = xa_cmpxchg_irq(&mapping->i_pages, index,
1694                              swp_to_radix_entry(swap),
1695                              swp_to_radix_entry(swapin_error), 0);
1696         if (old != swp_to_radix_entry(swap))
1697                 return;
1698
1699         folio_wait_writeback(folio);
1700         delete_from_swap_cache(folio);
1701         spin_lock_irq(&info->lock);
1702         /*
1703          * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks won't
1704          * be 0 when inode is released and thus trigger WARN_ON(inode->i_blocks) in
1705          * shmem_evict_inode.
1706          */
1707         info->alloced--;
1708         info->swapped--;
1709         shmem_recalc_inode(inode);
1710         spin_unlock_irq(&info->lock);
1711         swap_free(swap);
1712 }
1713
1714 /*
1715  * Swap in the folio pointed to by *foliop.
1716  * Caller has to make sure that *foliop contains a valid swapped folio.
1717  * Returns 0 and the folio in foliop if success. On failure, returns the
1718  * error code and NULL in *foliop.
1719  */
1720 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1721                              struct folio **foliop, enum sgp_type sgp,
1722                              gfp_t gfp, struct vm_area_struct *vma,
1723                              vm_fault_t *fault_type)
1724 {
1725         struct address_space *mapping = inode->i_mapping;
1726         struct shmem_inode_info *info = SHMEM_I(inode);
1727         struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1728         struct folio *folio = NULL;
1729         swp_entry_t swap;
1730         int error;
1731
1732         VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1733         swap = radix_to_swp_entry(*foliop);
1734         *foliop = NULL;
1735
1736         if (is_swapin_error_entry(swap))
1737                 return -EIO;
1738
1739         /* Look it up and read it in.. */
1740         folio = swap_cache_get_folio(swap, NULL, 0);
1741         if (!folio) {
1742                 /* Or update major stats only when swapin succeeds?? */
1743                 if (fault_type) {
1744                         *fault_type |= VM_FAULT_MAJOR;
1745                         count_vm_event(PGMAJFAULT);
1746                         count_memcg_event_mm(charge_mm, PGMAJFAULT);
1747                 }
1748                 /* Here we actually start the io */
1749                 folio = shmem_swapin(swap, gfp, info, index);
1750                 if (!folio) {
1751                         error = -ENOMEM;
1752                         goto failed;
1753                 }
1754         }
1755
1756         /* We have to do this with folio locked to prevent races */
1757         folio_lock(folio);
1758         if (!folio_test_swapcache(folio) ||
1759             folio_swap_entry(folio).val != swap.val ||
1760             !shmem_confirm_swap(mapping, index, swap)) {
1761                 error = -EEXIST;
1762                 goto unlock;
1763         }
1764         if (!folio_test_uptodate(folio)) {
1765                 error = -EIO;
1766                 goto failed;
1767         }
1768         folio_wait_writeback(folio);
1769
1770         /*
1771          * Some architectures may have to restore extra metadata to the
1772          * folio after reading from swap.
1773          */
1774         arch_swap_restore(swap, folio);
1775
1776         if (shmem_should_replace_folio(folio, gfp)) {
1777                 error = shmem_replace_folio(&folio, gfp, info, index);
1778                 if (error)
1779                         goto failed;
1780         }
1781
1782         error = shmem_add_to_page_cache(folio, mapping, index,
1783                                         swp_to_radix_entry(swap), gfp,
1784                                         charge_mm);
1785         if (error)
1786                 goto failed;
1787
1788         spin_lock_irq(&info->lock);
1789         info->swapped--;
1790         shmem_recalc_inode(inode);
1791         spin_unlock_irq(&info->lock);
1792
1793         if (sgp == SGP_WRITE)
1794                 folio_mark_accessed(folio);
1795
1796         delete_from_swap_cache(folio);
1797         folio_mark_dirty(folio);
1798         swap_free(swap);
1799
1800         *foliop = folio;
1801         return 0;
1802 failed:
1803         if (!shmem_confirm_swap(mapping, index, swap))
1804                 error = -EEXIST;
1805         if (error == -EIO)
1806                 shmem_set_folio_swapin_error(inode, index, folio, swap);
1807 unlock:
1808         if (folio) {
1809                 folio_unlock(folio);
1810                 folio_put(folio);
1811         }
1812
1813         return error;
1814 }
1815
1816 /*
1817  * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
1818  *
1819  * If we allocate a new one we do not mark it dirty. That's up to the
1820  * vm. If we swap it in we mark it dirty since we also free the swap
1821  * entry since a page cannot live in both the swap and page cache.
1822  *
1823  * vma, vmf, and fault_type are only supplied by shmem_fault:
1824  * otherwise they are NULL.
1825  */
1826 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
1827                 struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
1828                 struct vm_area_struct *vma, struct vm_fault *vmf,
1829                 vm_fault_t *fault_type)
1830 {
1831         struct address_space *mapping = inode->i_mapping;
1832         struct shmem_inode_info *info = SHMEM_I(inode);
1833         struct shmem_sb_info *sbinfo;
1834         struct mm_struct *charge_mm;
1835         struct folio *folio;
1836         pgoff_t hindex = index;
1837         gfp_t huge_gfp;
1838         int error;
1839         int once = 0;
1840         int alloced = 0;
1841
1842         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1843                 return -EFBIG;
1844 repeat:
1845         if (sgp <= SGP_CACHE &&
1846             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1847                 return -EINVAL;
1848         }
1849
1850         sbinfo = SHMEM_SB(inode->i_sb);
1851         charge_mm = vma ? vma->vm_mm : NULL;
1852
1853         folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0);
1854         if (folio && vma && userfaultfd_minor(vma)) {
1855                 if (!xa_is_value(folio)) {
1856                         folio_unlock(folio);
1857                         folio_put(folio);
1858                 }
1859                 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1860                 return 0;
1861         }
1862
1863         if (xa_is_value(folio)) {
1864                 error = shmem_swapin_folio(inode, index, &folio,
1865                                           sgp, gfp, vma, fault_type);
1866                 if (error == -EEXIST)
1867                         goto repeat;
1868
1869                 *foliop = folio;
1870                 return error;
1871         }
1872
1873         if (folio) {
1874                 hindex = folio->index;
1875                 if (sgp == SGP_WRITE)
1876                         folio_mark_accessed(folio);
1877                 if (folio_test_uptodate(folio))
1878                         goto out;
1879                 /* fallocated folio */
1880                 if (sgp != SGP_READ)
1881                         goto clear;
1882                 folio_unlock(folio);
1883                 folio_put(folio);
1884         }
1885
1886         /*
1887          * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
1888          * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
1889          */
1890         *foliop = NULL;
1891         if (sgp == SGP_READ)
1892                 return 0;
1893         if (sgp == SGP_NOALLOC)
1894                 return -ENOENT;
1895
1896         /*
1897          * Fast cache lookup and swap lookup did not find it: allocate.
1898          */
1899
1900         if (vma && userfaultfd_missing(vma)) {
1901                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1902                 return 0;
1903         }
1904
1905         if (!shmem_is_huge(vma, inode, index, false))
1906                 goto alloc_nohuge;
1907
1908         huge_gfp = vma_thp_gfp_mask(vma);
1909         huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1910         folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true);
1911         if (IS_ERR(folio)) {
1912 alloc_nohuge:
1913                 folio = shmem_alloc_and_acct_folio(gfp, inode, index, false);
1914         }
1915         if (IS_ERR(folio)) {
1916                 int retry = 5;
1917
1918                 error = PTR_ERR(folio);
1919                 folio = NULL;
1920                 if (error != -ENOSPC)
1921                         goto unlock;
1922                 /*
1923                  * Try to reclaim some space by splitting a large folio
1924                  * beyond i_size on the filesystem.
1925                  */
1926                 while (retry--) {
1927                         int ret;
1928
1929                         ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1930                         if (ret == SHRINK_STOP)
1931                                 break;
1932                         if (ret)
1933                                 goto alloc_nohuge;
1934                 }
1935                 goto unlock;
1936         }
1937
1938         hindex = round_down(index, folio_nr_pages(folio));
1939
1940         if (sgp == SGP_WRITE)
1941                 __folio_set_referenced(folio);
1942
1943         error = shmem_add_to_page_cache(folio, mapping, hindex,
1944                                         NULL, gfp & GFP_RECLAIM_MASK,
1945                                         charge_mm);
1946         if (error)
1947                 goto unacct;
1948         folio_add_lru(folio);
1949
1950         spin_lock_irq(&info->lock);
1951         info->alloced += folio_nr_pages(folio);
1952         inode->i_blocks += (blkcnt_t)BLOCKS_PER_PAGE << folio_order(folio);
1953         shmem_recalc_inode(inode);
1954         spin_unlock_irq(&info->lock);
1955         alloced = true;
1956
1957         if (folio_test_pmd_mappable(folio) &&
1958             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1959                                         folio_next_index(folio) - 1) {
1960                 /*
1961                  * Part of the large folio is beyond i_size: subject
1962                  * to shrink under memory pressure.
1963                  */
1964                 spin_lock(&sbinfo->shrinklist_lock);
1965                 /*
1966                  * _careful to defend against unlocked access to
1967                  * ->shrink_list in shmem_unused_huge_shrink()
1968                  */
1969                 if (list_empty_careful(&info->shrinklist)) {
1970                         list_add_tail(&info->shrinklist,
1971                                       &sbinfo->shrinklist);
1972                         sbinfo->shrinklist_len++;
1973                 }
1974                 spin_unlock(&sbinfo->shrinklist_lock);
1975         }
1976
1977         /*
1978          * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
1979          */
1980         if (sgp == SGP_FALLOC)
1981                 sgp = SGP_WRITE;
1982 clear:
1983         /*
1984          * Let SGP_WRITE caller clear ends if write does not fill folio;
1985          * but SGP_FALLOC on a folio fallocated earlier must initialize
1986          * it now, lest undo on failure cancel our earlier guarantee.
1987          */
1988         if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
1989                 long i, n = folio_nr_pages(folio);
1990
1991                 for (i = 0; i < n; i++)
1992                         clear_highpage(folio_page(folio, i));
1993                 flush_dcache_folio(folio);
1994                 folio_mark_uptodate(folio);
1995         }
1996
1997         /* Perhaps the file has been truncated since we checked */
1998         if (sgp <= SGP_CACHE &&
1999             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2000                 if (alloced) {
2001                         folio_clear_dirty(folio);
2002                         filemap_remove_folio(folio);
2003                         spin_lock_irq(&info->lock);
2004                         shmem_recalc_inode(inode);
2005                         spin_unlock_irq(&info->lock);
2006                 }
2007                 error = -EINVAL;
2008                 goto unlock;
2009         }
2010 out:
2011         *foliop = folio;
2012         return 0;
2013
2014         /*
2015          * Error recovery.
2016          */
2017 unacct:
2018         shmem_inode_unacct_blocks(inode, folio_nr_pages(folio));
2019
2020         if (folio_test_large(folio)) {
2021                 folio_unlock(folio);
2022                 folio_put(folio);
2023                 goto alloc_nohuge;
2024         }
2025 unlock:
2026         if (folio) {
2027                 folio_unlock(folio);
2028                 folio_put(folio);
2029         }
2030         if (error == -ENOSPC && !once++) {
2031                 spin_lock_irq(&info->lock);
2032                 shmem_recalc_inode(inode);
2033                 spin_unlock_irq(&info->lock);
2034                 goto repeat;
2035         }
2036         if (error == -EEXIST)
2037                 goto repeat;
2038         return error;
2039 }
2040
2041 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2042                 enum sgp_type sgp)
2043 {
2044         return shmem_get_folio_gfp(inode, index, foliop, sgp,
2045                         mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
2046 }
2047
2048 /*
2049  * This is like autoremove_wake_function, but it removes the wait queue
2050  * entry unconditionally - even if something else had already woken the
2051  * target.
2052  */
2053 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2054 {
2055         int ret = default_wake_function(wait, mode, sync, key);
2056         list_del_init(&wait->entry);
2057         return ret;
2058 }
2059
2060 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2061 {
2062         struct vm_area_struct *vma = vmf->vma;
2063         struct inode *inode = file_inode(vma->vm_file);
2064         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2065         struct folio *folio = NULL;
2066         int err;
2067         vm_fault_t ret = VM_FAULT_LOCKED;
2068
2069         /*
2070          * Trinity finds that probing a hole which tmpfs is punching can
2071          * prevent the hole-punch from ever completing: which in turn
2072          * locks writers out with its hold on i_rwsem.  So refrain from
2073          * faulting pages into the hole while it's being punched.  Although
2074          * shmem_undo_range() does remove the additions, it may be unable to
2075          * keep up, as each new page needs its own unmap_mapping_range() call,
2076          * and the i_mmap tree grows ever slower to scan if new vmas are added.
2077          *
2078          * It does not matter if we sometimes reach this check just before the
2079          * hole-punch begins, so that one fault then races with the punch:
2080          * we just need to make racing faults a rare case.
2081          *
2082          * The implementation below would be much simpler if we just used a
2083          * standard mutex or completion: but we cannot take i_rwsem in fault,
2084          * and bloating every shmem inode for this unlikely case would be sad.
2085          */
2086         if (unlikely(inode->i_private)) {
2087                 struct shmem_falloc *shmem_falloc;
2088
2089                 spin_lock(&inode->i_lock);
2090                 shmem_falloc = inode->i_private;
2091                 if (shmem_falloc &&
2092                     shmem_falloc->waitq &&
2093                     vmf->pgoff >= shmem_falloc->start &&
2094                     vmf->pgoff < shmem_falloc->next) {
2095                         struct file *fpin;
2096                         wait_queue_head_t *shmem_falloc_waitq;
2097                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2098
2099                         ret = VM_FAULT_NOPAGE;
2100                         fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2101                         if (fpin)
2102                                 ret = VM_FAULT_RETRY;
2103
2104                         shmem_falloc_waitq = shmem_falloc->waitq;
2105                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2106                                         TASK_UNINTERRUPTIBLE);
2107                         spin_unlock(&inode->i_lock);
2108                         schedule();
2109
2110                         /*
2111                          * shmem_falloc_waitq points into the shmem_fallocate()
2112                          * stack of the hole-punching task: shmem_falloc_waitq
2113                          * is usually invalid by the time we reach here, but
2114                          * finish_wait() does not dereference it in that case;
2115                          * though i_lock needed lest racing with wake_up_all().
2116                          */
2117                         spin_lock(&inode->i_lock);
2118                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2119                         spin_unlock(&inode->i_lock);
2120
2121                         if (fpin)
2122                                 fput(fpin);
2123                         return ret;
2124                 }
2125                 spin_unlock(&inode->i_lock);
2126         }
2127
2128         err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2129                                   gfp, vma, vmf, &ret);
2130         if (err)
2131                 return vmf_error(err);
2132         if (folio)
2133                 vmf->page = folio_file_page(folio, vmf->pgoff);
2134         return ret;
2135 }
2136
2137 unsigned long shmem_get_unmapped_area(struct file *file,
2138                                       unsigned long uaddr, unsigned long len,
2139                                       unsigned long pgoff, unsigned long flags)
2140 {
2141         unsigned long (*get_area)(struct file *,
2142                 unsigned long, unsigned long, unsigned long, unsigned long);
2143         unsigned long addr;
2144         unsigned long offset;
2145         unsigned long inflated_len;
2146         unsigned long inflated_addr;
2147         unsigned long inflated_offset;
2148
2149         if (len > TASK_SIZE)
2150                 return -ENOMEM;
2151
2152         get_area = current->mm->get_unmapped_area;
2153         addr = get_area(file, uaddr, len, pgoff, flags);
2154
2155         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2156                 return addr;
2157         if (IS_ERR_VALUE(addr))
2158                 return addr;
2159         if (addr & ~PAGE_MASK)
2160                 return addr;
2161         if (addr > TASK_SIZE - len)
2162                 return addr;
2163
2164         if (shmem_huge == SHMEM_HUGE_DENY)
2165                 return addr;
2166         if (len < HPAGE_PMD_SIZE)
2167                 return addr;
2168         if (flags & MAP_FIXED)
2169                 return addr;
2170         /*
2171          * Our priority is to support MAP_SHARED mapped hugely;
2172          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2173          * But if caller specified an address hint and we allocated area there
2174          * successfully, respect that as before.
2175          */
2176         if (uaddr == addr)
2177                 return addr;
2178
2179         if (shmem_huge != SHMEM_HUGE_FORCE) {
2180                 struct super_block *sb;
2181
2182                 if (file) {
2183                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2184                         sb = file_inode(file)->i_sb;
2185                 } else {
2186                         /*
2187                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2188                          * for "/dev/zero", to create a shared anonymous object.
2189                          */
2190                         if (IS_ERR(shm_mnt))
2191                                 return addr;
2192                         sb = shm_mnt->mnt_sb;
2193                 }
2194                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2195                         return addr;
2196         }
2197
2198         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2199         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2200                 return addr;
2201         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2202                 return addr;
2203
2204         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2205         if (inflated_len > TASK_SIZE)
2206                 return addr;
2207         if (inflated_len < len)
2208                 return addr;
2209
2210         inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2211         if (IS_ERR_VALUE(inflated_addr))
2212                 return addr;
2213         if (inflated_addr & ~PAGE_MASK)
2214                 return addr;
2215
2216         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2217         inflated_addr += offset - inflated_offset;
2218         if (inflated_offset > offset)
2219                 inflated_addr += HPAGE_PMD_SIZE;
2220
2221         if (inflated_addr > TASK_SIZE - len)
2222                 return addr;
2223         return inflated_addr;
2224 }
2225
2226 #ifdef CONFIG_NUMA
2227 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2228 {
2229         struct inode *inode = file_inode(vma->vm_file);
2230         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2231 }
2232
2233 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2234                                           unsigned long addr)
2235 {
2236         struct inode *inode = file_inode(vma->vm_file);
2237         pgoff_t index;
2238
2239         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2240         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2241 }
2242 #endif
2243
2244 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2245 {
2246         struct inode *inode = file_inode(file);
2247         struct shmem_inode_info *info = SHMEM_I(inode);
2248         int retval = -ENOMEM;
2249
2250         /*
2251          * What serializes the accesses to info->flags?
2252          * ipc_lock_object() when called from shmctl_do_lock(),
2253          * no serialization needed when called from shm_destroy().
2254          */
2255         if (lock && !(info->flags & VM_LOCKED)) {
2256                 if (!user_shm_lock(inode->i_size, ucounts))
2257                         goto out_nomem;
2258                 info->flags |= VM_LOCKED;
2259                 mapping_set_unevictable(file->f_mapping);
2260         }
2261         if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2262                 user_shm_unlock(inode->i_size, ucounts);
2263                 info->flags &= ~VM_LOCKED;
2264                 mapping_clear_unevictable(file->f_mapping);
2265         }
2266         retval = 0;
2267
2268 out_nomem:
2269         return retval;
2270 }
2271
2272 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2273 {
2274         struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2275         int ret;
2276
2277         ret = seal_check_future_write(info->seals, vma);
2278         if (ret)
2279                 return ret;
2280
2281         /* arm64 - allow memory tagging on RAM-based files */
2282         vma->vm_flags |= VM_MTE_ALLOWED;
2283
2284         file_accessed(file);
2285         vma->vm_ops = &shmem_vm_ops;
2286         return 0;
2287 }
2288
2289 #ifdef CONFIG_TMPFS_XATTR
2290 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2291
2292 /*
2293  * chattr's fsflags are unrelated to extended attributes,
2294  * but tmpfs has chosen to enable them under the same config option.
2295  */
2296 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2297 {
2298         unsigned int i_flags = 0;
2299
2300         if (fsflags & FS_NOATIME_FL)
2301                 i_flags |= S_NOATIME;
2302         if (fsflags & FS_APPEND_FL)
2303                 i_flags |= S_APPEND;
2304         if (fsflags & FS_IMMUTABLE_FL)
2305                 i_flags |= S_IMMUTABLE;
2306         /*
2307          * But FS_NODUMP_FL does not require any action in i_flags.
2308          */
2309         inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2310 }
2311 #else
2312 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2313 {
2314 }
2315 #define shmem_initxattrs NULL
2316 #endif
2317
2318 static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir,
2319                                      umode_t mode, dev_t dev, unsigned long flags)
2320 {
2321         struct inode *inode;
2322         struct shmem_inode_info *info;
2323         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2324         ino_t ino;
2325
2326         if (shmem_reserve_inode(sb, &ino))
2327                 return NULL;
2328
2329         inode = new_inode(sb);
2330         if (inode) {
2331                 inode->i_ino = ino;
2332                 inode_init_owner(&init_user_ns, inode, dir, mode);
2333                 inode->i_blocks = 0;
2334                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2335                 inode->i_generation = get_random_u32();
2336                 info = SHMEM_I(inode);
2337                 memset(info, 0, (char *)inode - (char *)info);
2338                 spin_lock_init(&info->lock);
2339                 atomic_set(&info->stop_eviction, 0);
2340                 info->seals = F_SEAL_SEAL;
2341                 info->flags = flags & VM_NORESERVE;
2342                 info->i_crtime = inode->i_mtime;
2343                 info->fsflags = (dir == NULL) ? 0 :
2344                         SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2345                 if (info->fsflags)
2346                         shmem_set_inode_flags(inode, info->fsflags);
2347                 INIT_LIST_HEAD(&info->shrinklist);
2348                 INIT_LIST_HEAD(&info->swaplist);
2349                 simple_xattrs_init(&info->xattrs);
2350                 cache_no_acl(inode);
2351                 mapping_set_large_folios(inode->i_mapping);
2352
2353                 switch (mode & S_IFMT) {
2354                 default:
2355                         inode->i_op = &shmem_special_inode_operations;
2356                         init_special_inode(inode, mode, dev);
2357                         break;
2358                 case S_IFREG:
2359                         inode->i_mapping->a_ops = &shmem_aops;
2360                         inode->i_op = &shmem_inode_operations;
2361                         inode->i_fop = &shmem_file_operations;
2362                         mpol_shared_policy_init(&info->policy,
2363                                                  shmem_get_sbmpol(sbinfo));
2364                         break;
2365                 case S_IFDIR:
2366                         inc_nlink(inode);
2367                         /* Some things misbehave if size == 0 on a directory */
2368                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2369                         inode->i_op = &shmem_dir_inode_operations;
2370                         inode->i_fop = &simple_dir_operations;
2371                         break;
2372                 case S_IFLNK:
2373                         /*
2374                          * Must not load anything in the rbtree,
2375                          * mpol_free_shared_policy will not be called.
2376                          */
2377                         mpol_shared_policy_init(&info->policy, NULL);
2378                         break;
2379                 }
2380
2381                 lockdep_annotate_inode_mutex_key(inode);
2382         } else
2383                 shmem_free_inode(sb);
2384         return inode;
2385 }
2386
2387 #ifdef CONFIG_USERFAULTFD
2388 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2389                            pmd_t *dst_pmd,
2390                            struct vm_area_struct *dst_vma,
2391                            unsigned long dst_addr,
2392                            unsigned long src_addr,
2393                            bool zeropage, bool wp_copy,
2394                            struct page **pagep)
2395 {
2396         struct inode *inode = file_inode(dst_vma->vm_file);
2397         struct shmem_inode_info *info = SHMEM_I(inode);
2398         struct address_space *mapping = inode->i_mapping;
2399         gfp_t gfp = mapping_gfp_mask(mapping);
2400         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2401         void *page_kaddr;
2402         struct folio *folio;
2403         int ret;
2404         pgoff_t max_off;
2405
2406         if (!shmem_inode_acct_block(inode, 1)) {
2407                 /*
2408                  * We may have got a page, returned -ENOENT triggering a retry,
2409                  * and now we find ourselves with -ENOMEM. Release the page, to
2410                  * avoid a BUG_ON in our caller.
2411                  */
2412                 if (unlikely(*pagep)) {
2413                         put_page(*pagep);
2414                         *pagep = NULL;
2415                 }
2416                 return -ENOMEM;
2417         }
2418
2419         if (!*pagep) {
2420                 ret = -ENOMEM;
2421                 folio = shmem_alloc_folio(gfp, info, pgoff);
2422                 if (!folio)
2423                         goto out_unacct_blocks;
2424
2425                 if (!zeropage) {        /* COPY */
2426                         page_kaddr = kmap_local_folio(folio, 0);
2427                         /*
2428                          * The read mmap_lock is held here.  Despite the
2429                          * mmap_lock being read recursive a deadlock is still
2430                          * possible if a writer has taken a lock.  For example:
2431                          *
2432                          * process A thread 1 takes read lock on own mmap_lock
2433                          * process A thread 2 calls mmap, blocks taking write lock
2434                          * process B thread 1 takes page fault, read lock on own mmap lock
2435                          * process B thread 2 calls mmap, blocks taking write lock
2436                          * process A thread 1 blocks taking read lock on process B
2437                          * process B thread 1 blocks taking read lock on process A
2438                          *
2439                          * Disable page faults to prevent potential deadlock
2440                          * and retry the copy outside the mmap_lock.
2441                          */
2442                         pagefault_disable();
2443                         ret = copy_from_user(page_kaddr,
2444                                              (const void __user *)src_addr,
2445                                              PAGE_SIZE);
2446                         pagefault_enable();
2447                         kunmap_local(page_kaddr);
2448
2449                         /* fallback to copy_from_user outside mmap_lock */
2450                         if (unlikely(ret)) {
2451                                 *pagep = &folio->page;
2452                                 ret = -ENOENT;
2453                                 /* don't free the page */
2454                                 goto out_unacct_blocks;
2455                         }
2456
2457                         flush_dcache_folio(folio);
2458                 } else {                /* ZEROPAGE */
2459                         clear_user_highpage(&folio->page, dst_addr);
2460                 }
2461         } else {
2462                 folio = page_folio(*pagep);
2463                 VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2464                 *pagep = NULL;
2465         }
2466
2467         VM_BUG_ON(folio_test_locked(folio));
2468         VM_BUG_ON(folio_test_swapbacked(folio));
2469         __folio_set_locked(folio);
2470         __folio_set_swapbacked(folio);
2471         __folio_mark_uptodate(folio);
2472
2473         ret = -EFAULT;
2474         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2475         if (unlikely(pgoff >= max_off))
2476                 goto out_release;
2477
2478         ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
2479                                       gfp & GFP_RECLAIM_MASK, dst_mm);
2480         if (ret)
2481                 goto out_release;
2482
2483         ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2484                                        &folio->page, true, wp_copy);
2485         if (ret)
2486                 goto out_delete_from_cache;
2487
2488         spin_lock_irq(&info->lock);
2489         info->alloced++;
2490         inode->i_blocks += BLOCKS_PER_PAGE;
2491         shmem_recalc_inode(inode);
2492         spin_unlock_irq(&info->lock);
2493
2494         folio_unlock(folio);
2495         return 0;
2496 out_delete_from_cache:
2497         filemap_remove_folio(folio);
2498 out_release:
2499         folio_unlock(folio);
2500         folio_put(folio);
2501 out_unacct_blocks:
2502         shmem_inode_unacct_blocks(inode, 1);
2503         return ret;
2504 }
2505 #endif /* CONFIG_USERFAULTFD */
2506
2507 #ifdef CONFIG_TMPFS
2508 static const struct inode_operations shmem_symlink_inode_operations;
2509 static const struct inode_operations shmem_short_symlink_operations;
2510
2511 static int
2512 shmem_write_begin(struct file *file, struct address_space *mapping,
2513                         loff_t pos, unsigned len,
2514                         struct page **pagep, void **fsdata)
2515 {
2516         struct inode *inode = mapping->host;
2517         struct shmem_inode_info *info = SHMEM_I(inode);
2518         pgoff_t index = pos >> PAGE_SHIFT;
2519         struct folio *folio;
2520         int ret = 0;
2521
2522         /* i_rwsem is held by caller */
2523         if (unlikely(info->seals & (F_SEAL_GROW |
2524                                    F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2525                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2526                         return -EPERM;
2527                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2528                         return -EPERM;
2529         }
2530
2531         ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2532
2533         if (ret)
2534                 return ret;
2535
2536         *pagep = folio_file_page(folio, index);
2537         if (PageHWPoison(*pagep)) {
2538                 folio_unlock(folio);
2539                 folio_put(folio);
2540                 *pagep = NULL;
2541                 return -EIO;
2542         }
2543
2544         return 0;
2545 }
2546
2547 static int
2548 shmem_write_end(struct file *file, struct address_space *mapping,
2549                         loff_t pos, unsigned len, unsigned copied,
2550                         struct page *page, void *fsdata)
2551 {
2552         struct inode *inode = mapping->host;
2553
2554         if (pos + copied > inode->i_size)
2555                 i_size_write(inode, pos + copied);
2556
2557         if (!PageUptodate(page)) {
2558                 struct page *head = compound_head(page);
2559                 if (PageTransCompound(page)) {
2560                         int i;
2561
2562                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2563                                 if (head + i == page)
2564                                         continue;
2565                                 clear_highpage(head + i);
2566                                 flush_dcache_page(head + i);
2567                         }
2568                 }
2569                 if (copied < PAGE_SIZE) {
2570                         unsigned from = pos & (PAGE_SIZE - 1);
2571                         zero_user_segments(page, 0, from,
2572                                         from + copied, PAGE_SIZE);
2573                 }
2574                 SetPageUptodate(head);
2575         }
2576         set_page_dirty(page);
2577         unlock_page(page);
2578         put_page(page);
2579
2580         return copied;
2581 }
2582
2583 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2584 {
2585         struct file *file = iocb->ki_filp;
2586         struct inode *inode = file_inode(file);
2587         struct address_space *mapping = inode->i_mapping;
2588         pgoff_t index;
2589         unsigned long offset;
2590         int error = 0;
2591         ssize_t retval = 0;
2592         loff_t *ppos = &iocb->ki_pos;
2593
2594         index = *ppos >> PAGE_SHIFT;
2595         offset = *ppos & ~PAGE_MASK;
2596
2597         for (;;) {
2598                 struct folio *folio = NULL;
2599                 struct page *page = NULL;
2600                 pgoff_t end_index;
2601                 unsigned long nr, ret;
2602                 loff_t i_size = i_size_read(inode);
2603
2604                 end_index = i_size >> PAGE_SHIFT;
2605                 if (index > end_index)
2606                         break;
2607                 if (index == end_index) {
2608                         nr = i_size & ~PAGE_MASK;
2609                         if (nr <= offset)
2610                                 break;
2611                 }
2612
2613                 error = shmem_get_folio(inode, index, &folio, SGP_READ);
2614                 if (error) {
2615                         if (error == -EINVAL)
2616                                 error = 0;
2617                         break;
2618                 }
2619                 if (folio) {
2620                         folio_unlock(folio);
2621
2622                         page = folio_file_page(folio, index);
2623                         if (PageHWPoison(page)) {
2624                                 folio_put(folio);
2625                                 error = -EIO;
2626                                 break;
2627                         }
2628                 }
2629
2630                 /*
2631                  * We must evaluate after, since reads (unlike writes)
2632                  * are called without i_rwsem protection against truncate
2633                  */
2634                 nr = PAGE_SIZE;
2635                 i_size = i_size_read(inode);
2636                 end_index = i_size >> PAGE_SHIFT;
2637                 if (index == end_index) {
2638                         nr = i_size & ~PAGE_MASK;
2639                         if (nr <= offset) {
2640                                 if (folio)
2641                                         folio_put(folio);
2642                                 break;
2643                         }
2644                 }
2645                 nr -= offset;
2646
2647                 if (folio) {
2648                         /*
2649                          * If users can be writing to this page using arbitrary
2650                          * virtual addresses, take care about potential aliasing
2651                          * before reading the page on the kernel side.
2652                          */
2653                         if (mapping_writably_mapped(mapping))
2654                                 flush_dcache_page(page);
2655                         /*
2656                          * Mark the page accessed if we read the beginning.
2657                          */
2658                         if (!offset)
2659                                 folio_mark_accessed(folio);
2660                         /*
2661                          * Ok, we have the page, and it's up-to-date, so
2662                          * now we can copy it to user space...
2663                          */
2664                         ret = copy_page_to_iter(page, offset, nr, to);
2665                         folio_put(folio);
2666
2667                 } else if (user_backed_iter(to)) {
2668                         /*
2669                          * Copy to user tends to be so well optimized, but
2670                          * clear_user() not so much, that it is noticeably
2671                          * faster to copy the zero page instead of clearing.
2672                          */
2673                         ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
2674                 } else {
2675                         /*
2676                          * But submitting the same page twice in a row to
2677                          * splice() - or others? - can result in confusion:
2678                          * so don't attempt that optimization on pipes etc.
2679                          */
2680                         ret = iov_iter_zero(nr, to);
2681                 }
2682
2683                 retval += ret;
2684                 offset += ret;
2685                 index += offset >> PAGE_SHIFT;
2686                 offset &= ~PAGE_MASK;
2687
2688                 if (!iov_iter_count(to))
2689                         break;
2690                 if (ret < nr) {
2691                         error = -EFAULT;
2692                         break;
2693                 }
2694                 cond_resched();
2695         }
2696
2697         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2698         file_accessed(file);
2699         return retval ? retval : error;
2700 }
2701
2702 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2703 {
2704         struct address_space *mapping = file->f_mapping;
2705         struct inode *inode = mapping->host;
2706
2707         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2708                 return generic_file_llseek_size(file, offset, whence,
2709                                         MAX_LFS_FILESIZE, i_size_read(inode));
2710         if (offset < 0)
2711                 return -ENXIO;
2712
2713         inode_lock(inode);
2714         /* We're holding i_rwsem so we can access i_size directly */
2715         offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
2716         if (offset >= 0)
2717                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2718         inode_unlock(inode);
2719         return offset;
2720 }
2721
2722 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2723                                                          loff_t len)
2724 {
2725         struct inode *inode = file_inode(file);
2726         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2727         struct shmem_inode_info *info = SHMEM_I(inode);
2728         struct shmem_falloc shmem_falloc;
2729         pgoff_t start, index, end, undo_fallocend;
2730         int error;
2731
2732         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2733                 return -EOPNOTSUPP;
2734
2735         inode_lock(inode);
2736
2737         if (mode & FALLOC_FL_PUNCH_HOLE) {
2738                 struct address_space *mapping = file->f_mapping;
2739                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2740                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2741                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2742
2743                 /* protected by i_rwsem */
2744                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2745                         error = -EPERM;
2746                         goto out;
2747                 }
2748
2749                 shmem_falloc.waitq = &shmem_falloc_waitq;
2750                 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2751                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2752                 spin_lock(&inode->i_lock);
2753                 inode->i_private = &shmem_falloc;
2754                 spin_unlock(&inode->i_lock);
2755
2756                 if ((u64)unmap_end > (u64)unmap_start)
2757                         unmap_mapping_range(mapping, unmap_start,
2758                                             1 + unmap_end - unmap_start, 0);
2759                 shmem_truncate_range(inode, offset, offset + len - 1);
2760                 /* No need to unmap again: hole-punching leaves COWed pages */
2761
2762                 spin_lock(&inode->i_lock);
2763                 inode->i_private = NULL;
2764                 wake_up_all(&shmem_falloc_waitq);
2765                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2766                 spin_unlock(&inode->i_lock);
2767                 error = 0;
2768                 goto out;
2769         }
2770
2771         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2772         error = inode_newsize_ok(inode, offset + len);
2773         if (error)
2774                 goto out;
2775
2776         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2777                 error = -EPERM;
2778                 goto out;
2779         }
2780
2781         start = offset >> PAGE_SHIFT;
2782         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2783         /* Try to avoid a swapstorm if len is impossible to satisfy */
2784         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2785                 error = -ENOSPC;
2786                 goto out;
2787         }
2788
2789         shmem_falloc.waitq = NULL;
2790         shmem_falloc.start = start;
2791         shmem_falloc.next  = start;
2792         shmem_falloc.nr_falloced = 0;
2793         shmem_falloc.nr_unswapped = 0;
2794         spin_lock(&inode->i_lock);
2795         inode->i_private = &shmem_falloc;
2796         spin_unlock(&inode->i_lock);
2797
2798         /*
2799          * info->fallocend is only relevant when huge pages might be
2800          * involved: to prevent split_huge_page() freeing fallocated
2801          * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
2802          */
2803         undo_fallocend = info->fallocend;
2804         if (info->fallocend < end)
2805                 info->fallocend = end;
2806
2807         for (index = start; index < end; ) {
2808                 struct folio *folio;
2809
2810                 /*
2811                  * Good, the fallocate(2) manpage permits EINTR: we may have
2812                  * been interrupted because we are using up too much memory.
2813                  */
2814                 if (signal_pending(current))
2815                         error = -EINTR;
2816                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2817                         error = -ENOMEM;
2818                 else
2819                         error = shmem_get_folio(inode, index, &folio,
2820                                                 SGP_FALLOC);
2821                 if (error) {
2822                         info->fallocend = undo_fallocend;
2823                         /* Remove the !uptodate folios we added */
2824                         if (index > start) {
2825                                 shmem_undo_range(inode,
2826                                     (loff_t)start << PAGE_SHIFT,
2827                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2828                         }
2829                         goto undone;
2830                 }
2831
2832                 /*
2833                  * Here is a more important optimization than it appears:
2834                  * a second SGP_FALLOC on the same large folio will clear it,
2835                  * making it uptodate and un-undoable if we fail later.
2836                  */
2837                 index = folio_next_index(folio);
2838                 /* Beware 32-bit wraparound */
2839                 if (!index)
2840                         index--;
2841
2842                 /*
2843                  * Inform shmem_writepage() how far we have reached.
2844                  * No need for lock or barrier: we have the page lock.
2845                  */
2846                 if (!folio_test_uptodate(folio))
2847                         shmem_falloc.nr_falloced += index - shmem_falloc.next;
2848                 shmem_falloc.next = index;
2849
2850                 /*
2851                  * If !uptodate, leave it that way so that freeable folios
2852                  * can be recognized if we need to rollback on error later.
2853                  * But mark it dirty so that memory pressure will swap rather
2854                  * than free the folios we are allocating (and SGP_CACHE folios
2855                  * might still be clean: we now need to mark those dirty too).
2856                  */
2857                 folio_mark_dirty(folio);
2858                 folio_unlock(folio);
2859                 folio_put(folio);
2860                 cond_resched();
2861         }
2862
2863         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2864                 i_size_write(inode, offset + len);
2865 undone:
2866         spin_lock(&inode->i_lock);
2867         inode->i_private = NULL;
2868         spin_unlock(&inode->i_lock);
2869 out:
2870         if (!error)
2871                 file_modified(file);
2872         inode_unlock(inode);
2873         return error;
2874 }
2875
2876 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2877 {
2878         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2879
2880         buf->f_type = TMPFS_MAGIC;
2881         buf->f_bsize = PAGE_SIZE;
2882         buf->f_namelen = NAME_MAX;
2883         if (sbinfo->max_blocks) {
2884                 buf->f_blocks = sbinfo->max_blocks;
2885                 buf->f_bavail =
2886                 buf->f_bfree  = sbinfo->max_blocks -
2887                                 percpu_counter_sum(&sbinfo->used_blocks);
2888         }
2889         if (sbinfo->max_inodes) {
2890                 buf->f_files = sbinfo->max_inodes;
2891                 buf->f_ffree = sbinfo->free_inodes;
2892         }
2893         /* else leave those fields 0 like simple_statfs */
2894
2895         buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
2896
2897         return 0;
2898 }
2899
2900 /*
2901  * File creation. Allocate an inode, and we're done..
2902  */
2903 static int
2904 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir,
2905             struct dentry *dentry, umode_t mode, dev_t dev)
2906 {
2907         struct inode *inode;
2908         int error = -ENOSPC;
2909
2910         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2911         if (inode) {
2912                 error = simple_acl_create(dir, inode);
2913                 if (error)
2914                         goto out_iput;
2915                 error = security_inode_init_security(inode, dir,
2916                                                      &dentry->d_name,
2917                                                      shmem_initxattrs, NULL);
2918                 if (error && error != -EOPNOTSUPP)
2919                         goto out_iput;
2920
2921                 error = 0;
2922                 dir->i_size += BOGO_DIRENT_SIZE;
2923                 dir->i_ctime = dir->i_mtime = current_time(dir);
2924                 inode_inc_iversion(dir);
2925                 d_instantiate(dentry, inode);
2926                 dget(dentry); /* Extra count - pin the dentry in core */
2927         }
2928         return error;
2929 out_iput:
2930         iput(inode);
2931         return error;
2932 }
2933
2934 static int
2935 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
2936               struct file *file, umode_t mode)
2937 {
2938         struct inode *inode;
2939         int error = -ENOSPC;
2940
2941         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2942         if (inode) {
2943                 error = security_inode_init_security(inode, dir,
2944                                                      NULL,
2945                                                      shmem_initxattrs, NULL);
2946                 if (error && error != -EOPNOTSUPP)
2947                         goto out_iput;
2948                 error = simple_acl_create(dir, inode);
2949                 if (error)
2950                         goto out_iput;
2951                 d_tmpfile(file, inode);
2952         }
2953         return finish_open_simple(file, error);
2954 out_iput:
2955         iput(inode);
2956         return error;
2957 }
2958
2959 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
2960                        struct dentry *dentry, umode_t mode)
2961 {
2962         int error;
2963
2964         if ((error = shmem_mknod(&init_user_ns, dir, dentry,
2965                                  mode | S_IFDIR, 0)))
2966                 return error;
2967         inc_nlink(dir);
2968         return 0;
2969 }
2970
2971 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir,
2972                         struct dentry *dentry, umode_t mode, bool excl)
2973 {
2974         return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
2975 }
2976
2977 /*
2978  * Link a file..
2979  */
2980 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2981 {
2982         struct inode *inode = d_inode(old_dentry);
2983         int ret = 0;
2984
2985         /*
2986          * No ordinary (disk based) filesystem counts links as inodes;
2987          * but each new link needs a new dentry, pinning lowmem, and
2988          * tmpfs dentries cannot be pruned until they are unlinked.
2989          * But if an O_TMPFILE file is linked into the tmpfs, the
2990          * first link must skip that, to get the accounting right.
2991          */
2992         if (inode->i_nlink) {
2993                 ret = shmem_reserve_inode(inode->i_sb, NULL);
2994                 if (ret)
2995                         goto out;
2996         }
2997
2998         dir->i_size += BOGO_DIRENT_SIZE;
2999         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3000         inode_inc_iversion(dir);
3001         inc_nlink(inode);
3002         ihold(inode);   /* New dentry reference */
3003         dget(dentry);           /* Extra pinning count for the created dentry */
3004         d_instantiate(dentry, inode);
3005 out:
3006         return ret;
3007 }
3008
3009 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3010 {
3011         struct inode *inode = d_inode(dentry);
3012
3013         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3014                 shmem_free_inode(inode->i_sb);
3015
3016         dir->i_size -= BOGO_DIRENT_SIZE;
3017         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3018         inode_inc_iversion(dir);
3019         drop_nlink(inode);
3020         dput(dentry);   /* Undo the count from "create" - this does all the work */
3021         return 0;
3022 }
3023
3024 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3025 {
3026         if (!simple_empty(dentry))
3027                 return -ENOTEMPTY;
3028
3029         drop_nlink(d_inode(dentry));
3030         drop_nlink(dir);
3031         return shmem_unlink(dir, dentry);
3032 }
3033
3034 static int shmem_whiteout(struct user_namespace *mnt_userns,
3035                           struct inode *old_dir, struct dentry *old_dentry)
3036 {
3037         struct dentry *whiteout;
3038         int error;
3039
3040         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3041         if (!whiteout)
3042                 return -ENOMEM;
3043
3044         error = shmem_mknod(&init_user_ns, old_dir, whiteout,
3045                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3046         dput(whiteout);
3047         if (error)
3048                 return error;
3049
3050         /*
3051          * Cheat and hash the whiteout while the old dentry is still in
3052          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3053          *
3054          * d_lookup() will consistently find one of them at this point,
3055          * not sure which one, but that isn't even important.
3056          */
3057         d_rehash(whiteout);
3058         return 0;
3059 }
3060
3061 /*
3062  * The VFS layer already does all the dentry stuff for rename,
3063  * we just have to decrement the usage count for the target if
3064  * it exists so that the VFS layer correctly free's it when it
3065  * gets overwritten.
3066  */
3067 static int shmem_rename2(struct user_namespace *mnt_userns,
3068                          struct inode *old_dir, struct dentry *old_dentry,
3069                          struct inode *new_dir, struct dentry *new_dentry,
3070                          unsigned int flags)
3071 {
3072         struct inode *inode = d_inode(old_dentry);
3073         int they_are_dirs = S_ISDIR(inode->i_mode);
3074
3075         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3076                 return -EINVAL;
3077
3078         if (flags & RENAME_EXCHANGE)
3079                 return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
3080
3081         if (!simple_empty(new_dentry))
3082                 return -ENOTEMPTY;
3083
3084         if (flags & RENAME_WHITEOUT) {
3085                 int error;
3086
3087                 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry);
3088                 if (error)
3089                         return error;
3090         }
3091
3092         if (d_really_is_positive(new_dentry)) {
3093                 (void) shmem_unlink(new_dir, new_dentry);
3094                 if (they_are_dirs) {
3095                         drop_nlink(d_inode(new_dentry));
3096                         drop_nlink(old_dir);
3097                 }
3098         } else if (they_are_dirs) {
3099                 drop_nlink(old_dir);
3100                 inc_nlink(new_dir);
3101         }
3102
3103         old_dir->i_size -= BOGO_DIRENT_SIZE;
3104         new_dir->i_size += BOGO_DIRENT_SIZE;
3105         old_dir->i_ctime = old_dir->i_mtime =
3106         new_dir->i_ctime = new_dir->i_mtime =
3107         inode->i_ctime = current_time(old_dir);
3108         inode_inc_iversion(old_dir);
3109         inode_inc_iversion(new_dir);
3110         return 0;
3111 }
3112
3113 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir,
3114                          struct dentry *dentry, const char *symname)
3115 {
3116         int error;
3117         int len;
3118         struct inode *inode;
3119         struct folio *folio;
3120
3121         len = strlen(symname) + 1;
3122         if (len > PAGE_SIZE)
3123                 return -ENAMETOOLONG;
3124
3125         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3126                                 VM_NORESERVE);
3127         if (!inode)
3128                 return -ENOSPC;
3129
3130         error = security_inode_init_security(inode, dir, &dentry->d_name,
3131                                              shmem_initxattrs, NULL);
3132         if (error && error != -EOPNOTSUPP) {
3133                 iput(inode);
3134                 return error;
3135         }
3136
3137         inode->i_size = len-1;
3138         if (len <= SHORT_SYMLINK_LEN) {
3139                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3140                 if (!inode->i_link) {
3141                         iput(inode);
3142                         return -ENOMEM;
3143                 }
3144                 inode->i_op = &shmem_short_symlink_operations;
3145         } else {
3146                 inode_nohighmem(inode);
3147                 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3148                 if (error) {
3149                         iput(inode);
3150                         return error;
3151                 }
3152                 inode->i_mapping->a_ops = &shmem_aops;
3153                 inode->i_op = &shmem_symlink_inode_operations;
3154                 memcpy(folio_address(folio), symname, len);
3155                 folio_mark_uptodate(folio);
3156                 folio_mark_dirty(folio);
3157                 folio_unlock(folio);
3158                 folio_put(folio);
3159         }
3160         dir->i_size += BOGO_DIRENT_SIZE;
3161         dir->i_ctime = dir->i_mtime = current_time(dir);
3162         inode_inc_iversion(dir);
3163         d_instantiate(dentry, inode);
3164         dget(dentry);
3165         return 0;
3166 }
3167
3168 static void shmem_put_link(void *arg)
3169 {
3170         folio_mark_accessed(arg);
3171         folio_put(arg);
3172 }
3173
3174 static const char *shmem_get_link(struct dentry *dentry,
3175                                   struct inode *inode,
3176                                   struct delayed_call *done)
3177 {
3178         struct folio *folio = NULL;
3179         int error;
3180
3181         if (!dentry) {
3182                 folio = filemap_get_folio(inode->i_mapping, 0);
3183                 if (!folio)
3184                         return ERR_PTR(-ECHILD);
3185                 if (PageHWPoison(folio_page(folio, 0)) ||
3186                     !folio_test_uptodate(folio)) {
3187                         folio_put(folio);
3188                         return ERR_PTR(-ECHILD);
3189                 }
3190         } else {
3191                 error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3192                 if (error)
3193                         return ERR_PTR(error);
3194                 if (!folio)
3195                         return ERR_PTR(-ECHILD);
3196                 if (PageHWPoison(folio_page(folio, 0))) {
3197                         folio_unlock(folio);
3198                         folio_put(folio);
3199                         return ERR_PTR(-ECHILD);
3200                 }
3201                 folio_unlock(folio);
3202         }
3203         set_delayed_call(done, shmem_put_link, folio);
3204         return folio_address(folio);
3205 }
3206
3207 #ifdef CONFIG_TMPFS_XATTR
3208
3209 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3210 {
3211         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3212
3213         fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3214
3215         return 0;
3216 }
3217
3218 static int shmem_fileattr_set(struct user_namespace *mnt_userns,
3219                               struct dentry *dentry, struct fileattr *fa)
3220 {
3221         struct inode *inode = d_inode(dentry);
3222         struct shmem_inode_info *info = SHMEM_I(inode);
3223
3224         if (fileattr_has_fsx(fa))
3225                 return -EOPNOTSUPP;
3226         if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3227                 return -EOPNOTSUPP;
3228
3229         info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3230                 (fa->flags & SHMEM_FL_USER_MODIFIABLE);
3231
3232         shmem_set_inode_flags(inode, info->fsflags);
3233         inode->i_ctime = current_time(inode);
3234         inode_inc_iversion(inode);
3235         return 0;
3236 }
3237
3238 /*
3239  * Superblocks without xattr inode operations may get some security.* xattr
3240  * support from the LSM "for free". As soon as we have any other xattrs
3241  * like ACLs, we also need to implement the security.* handlers at
3242  * filesystem level, though.
3243  */
3244
3245 /*
3246  * Callback for security_inode_init_security() for acquiring xattrs.
3247  */
3248 static int shmem_initxattrs(struct inode *inode,
3249                             const struct xattr *xattr_array,
3250                             void *fs_info)
3251 {
3252         struct shmem_inode_info *info = SHMEM_I(inode);
3253         const struct xattr *xattr;
3254         struct simple_xattr *new_xattr;
3255         size_t len;
3256
3257         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3258                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3259                 if (!new_xattr)
3260                         return -ENOMEM;
3261
3262                 len = strlen(xattr->name) + 1;
3263                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3264                                           GFP_KERNEL);
3265                 if (!new_xattr->name) {
3266                         kvfree(new_xattr);
3267                         return -ENOMEM;
3268                 }
3269
3270                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3271                        XATTR_SECURITY_PREFIX_LEN);
3272                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3273                        xattr->name, len);
3274
3275                 simple_xattr_list_add(&info->xattrs, new_xattr);
3276         }
3277
3278         return 0;
3279 }
3280
3281 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3282                                    struct dentry *unused, struct inode *inode,
3283                                    const char *name, void *buffer, size_t size)
3284 {
3285         struct shmem_inode_info *info = SHMEM_I(inode);
3286
3287         name = xattr_full_name(handler, name);
3288         return simple_xattr_get(&info->xattrs, name, buffer, size);
3289 }
3290
3291 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3292                                    struct user_namespace *mnt_userns,
3293                                    struct dentry *unused, struct inode *inode,
3294                                    const char *name, const void *value,
3295                                    size_t size, int flags)
3296 {
3297         struct shmem_inode_info *info = SHMEM_I(inode);
3298         int err;
3299
3300         name = xattr_full_name(handler, name);
3301         err = simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3302         if (!err) {
3303                 inode->i_ctime = current_time(inode);
3304                 inode_inc_iversion(inode);
3305         }
3306         return err;
3307 }
3308
3309 static const struct xattr_handler shmem_security_xattr_handler = {
3310         .prefix = XATTR_SECURITY_PREFIX,
3311         .get = shmem_xattr_handler_get,
3312         .set = shmem_xattr_handler_set,
3313 };
3314
3315 static const struct xattr_handler shmem_trusted_xattr_handler = {
3316         .prefix = XATTR_TRUSTED_PREFIX,
3317         .get = shmem_xattr_handler_get,
3318         .set = shmem_xattr_handler_set,
3319 };
3320
3321 static const struct xattr_handler *shmem_xattr_handlers[] = {
3322 #ifdef CONFIG_TMPFS_POSIX_ACL
3323         &posix_acl_access_xattr_handler,
3324         &posix_acl_default_xattr_handler,
3325 #endif
3326         &shmem_security_xattr_handler,
3327         &shmem_trusted_xattr_handler,
3328         NULL
3329 };
3330
3331 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3332 {
3333         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3334         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3335 }
3336 #endif /* CONFIG_TMPFS_XATTR */
3337
3338 static const struct inode_operations shmem_short_symlink_operations = {
3339         .getattr        = shmem_getattr,
3340         .get_link       = simple_get_link,
3341 #ifdef CONFIG_TMPFS_XATTR
3342         .listxattr      = shmem_listxattr,
3343 #endif
3344 };
3345
3346 static const struct inode_operations shmem_symlink_inode_operations = {
3347         .getattr        = shmem_getattr,
3348         .get_link       = shmem_get_link,
3349 #ifdef CONFIG_TMPFS_XATTR
3350         .listxattr      = shmem_listxattr,
3351 #endif
3352 };
3353
3354 static struct dentry *shmem_get_parent(struct dentry *child)
3355 {
3356         return ERR_PTR(-ESTALE);
3357 }
3358
3359 static int shmem_match(struct inode *ino, void *vfh)
3360 {
3361         __u32 *fh = vfh;
3362         __u64 inum = fh[2];
3363         inum = (inum << 32) | fh[1];
3364         return ino->i_ino == inum && fh[0] == ino->i_generation;
3365 }
3366
3367 /* Find any alias of inode, but prefer a hashed alias */
3368 static struct dentry *shmem_find_alias(struct inode *inode)
3369 {
3370         struct dentry *alias = d_find_alias(inode);
3371
3372         return alias ?: d_find_any_alias(inode);
3373 }
3374
3375
3376 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3377                 struct fid *fid, int fh_len, int fh_type)
3378 {
3379         struct inode *inode;
3380         struct dentry *dentry = NULL;
3381         u64 inum;
3382
3383         if (fh_len < 3)
3384                 return NULL;
3385
3386         inum = fid->raw[2];
3387         inum = (inum << 32) | fid->raw[1];
3388
3389         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3390                         shmem_match, fid->raw);
3391         if (inode) {
3392                 dentry = shmem_find_alias(inode);
3393                 iput(inode);
3394         }
3395
3396         return dentry;
3397 }
3398
3399 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3400                                 struct inode *parent)
3401 {
3402         if (*len < 3) {
3403                 *len = 3;
3404                 return FILEID_INVALID;
3405         }
3406
3407         if (inode_unhashed(inode)) {
3408                 /* Unfortunately insert_inode_hash is not idempotent,
3409                  * so as we hash inodes here rather than at creation
3410                  * time, we need a lock to ensure we only try
3411                  * to do it once
3412                  */
3413                 static DEFINE_SPINLOCK(lock);
3414                 spin_lock(&lock);
3415                 if (inode_unhashed(inode))
3416                         __insert_inode_hash(inode,
3417                                             inode->i_ino + inode->i_generation);
3418                 spin_unlock(&lock);
3419         }
3420
3421         fh[0] = inode->i_generation;
3422         fh[1] = inode->i_ino;
3423         fh[2] = ((__u64)inode->i_ino) >> 32;
3424
3425         *len = 3;
3426         return 1;
3427 }
3428
3429 static const struct export_operations shmem_export_ops = {
3430         .get_parent     = shmem_get_parent,
3431         .encode_fh      = shmem_encode_fh,
3432         .fh_to_dentry   = shmem_fh_to_dentry,
3433 };
3434
3435 enum shmem_param {
3436         Opt_gid,
3437         Opt_huge,
3438         Opt_mode,
3439         Opt_mpol,
3440         Opt_nr_blocks,
3441         Opt_nr_inodes,
3442         Opt_size,
3443         Opt_uid,
3444         Opt_inode32,
3445         Opt_inode64,
3446 };
3447
3448 static const struct constant_table shmem_param_enums_huge[] = {
3449         {"never",       SHMEM_HUGE_NEVER },
3450         {"always",      SHMEM_HUGE_ALWAYS },
3451         {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3452         {"advise",      SHMEM_HUGE_ADVISE },
3453         {}
3454 };
3455
3456 const struct fs_parameter_spec shmem_fs_parameters[] = {
3457         fsparam_u32   ("gid",           Opt_gid),
3458         fsparam_enum  ("huge",          Opt_huge,  shmem_param_enums_huge),
3459         fsparam_u32oct("mode",          Opt_mode),
3460         fsparam_string("mpol",          Opt_mpol),
3461         fsparam_string("nr_blocks",     Opt_nr_blocks),
3462         fsparam_string("nr_inodes",     Opt_nr_inodes),
3463         fsparam_string("size",          Opt_size),
3464         fsparam_u32   ("uid",           Opt_uid),
3465         fsparam_flag  ("inode32",       Opt_inode32),
3466         fsparam_flag  ("inode64",       Opt_inode64),
3467         {}
3468 };
3469
3470 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3471 {
3472         struct shmem_options *ctx = fc->fs_private;
3473         struct fs_parse_result result;
3474         unsigned long long size;
3475         char *rest;
3476         int opt;
3477
3478         opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3479         if (opt < 0)
3480                 return opt;
3481
3482         switch (opt) {
3483         case Opt_size:
3484                 size = memparse(param->string, &rest);
3485                 if (*rest == '%') {
3486                         size <<= PAGE_SHIFT;
3487                         size *= totalram_pages();
3488                         do_div(size, 100);
3489                         rest++;
3490                 }
3491                 if (*rest)
3492                         goto bad_value;
3493                 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3494                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3495                 break;
3496         case Opt_nr_blocks:
3497                 ctx->blocks = memparse(param->string, &rest);
3498                 if (*rest || ctx->blocks > S64_MAX)
3499                         goto bad_value;
3500                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3501                 break;
3502         case Opt_nr_inodes:
3503                 ctx->inodes = memparse(param->string, &rest);
3504                 if (*rest)
3505                         goto bad_value;
3506                 ctx->seen |= SHMEM_SEEN_INODES;
3507                 break;
3508         case Opt_mode:
3509                 ctx->mode = result.uint_32 & 07777;
3510                 break;
3511         case Opt_uid:
3512                 ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3513                 if (!uid_valid(ctx->uid))
3514                         goto bad_value;
3515                 break;
3516         case Opt_gid:
3517                 ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3518                 if (!gid_valid(ctx->gid))
3519                         goto bad_value;
3520                 break;
3521         case Opt_huge:
3522                 ctx->huge = result.uint_32;
3523                 if (ctx->huge != SHMEM_HUGE_NEVER &&
3524                     !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3525                       has_transparent_hugepage()))
3526                         goto unsupported_parameter;
3527                 ctx->seen |= SHMEM_SEEN_HUGE;
3528                 break;
3529         case Opt_mpol:
3530                 if (IS_ENABLED(CONFIG_NUMA)) {
3531                         mpol_put(ctx->mpol);
3532                         ctx->mpol = NULL;
3533                         if (mpol_parse_str(param->string, &ctx->mpol))
3534                                 goto bad_value;
3535                         break;
3536                 }
3537                 goto unsupported_parameter;
3538         case Opt_inode32:
3539                 ctx->full_inums = false;
3540                 ctx->seen |= SHMEM_SEEN_INUMS;
3541                 break;
3542         case Opt_inode64:
3543                 if (sizeof(ino_t) < 8) {
3544                         return invalfc(fc,
3545                                        "Cannot use inode64 with <64bit inums in kernel\n");
3546                 }
3547                 ctx->full_inums = true;
3548                 ctx->seen |= SHMEM_SEEN_INUMS;
3549                 break;
3550         }
3551         return 0;
3552
3553 unsupported_parameter:
3554         return invalfc(fc, "Unsupported parameter '%s'", param->key);
3555 bad_value:
3556         return invalfc(fc, "Bad value for '%s'", param->key);
3557 }
3558
3559 static int shmem_parse_options(struct fs_context *fc, void *data)
3560 {
3561         char *options = data;
3562
3563         if (options) {
3564                 int err = security_sb_eat_lsm_opts(options, &fc->security);
3565                 if (err)
3566                         return err;
3567         }
3568
3569         while (options != NULL) {
3570                 char *this_char = options;
3571                 for (;;) {
3572                         /*
3573                          * NUL-terminate this option: unfortunately,
3574                          * mount options form a comma-separated list,
3575                          * but mpol's nodelist may also contain commas.
3576                          */
3577                         options = strchr(options, ',');
3578                         if (options == NULL)
3579                                 break;
3580                         options++;
3581                         if (!isdigit(*options)) {
3582                                 options[-1] = '\0';
3583                                 break;
3584                         }
3585                 }
3586                 if (*this_char) {
3587                         char *value = strchr(this_char, '=');
3588                         size_t len = 0;
3589                         int err;
3590
3591                         if (value) {
3592                                 *value++ = '\0';
3593                                 len = strlen(value);
3594                         }
3595                         err = vfs_parse_fs_string(fc, this_char, value, len);
3596                         if (err < 0)
3597                                 return err;
3598                 }
3599         }
3600         return 0;
3601 }
3602
3603 /*
3604  * Reconfigure a shmem filesystem.
3605  *
3606  * Note that we disallow change from limited->unlimited blocks/inodes while any
3607  * are in use; but we must separately disallow unlimited->limited, because in
3608  * that case we have no record of how much is already in use.
3609  */
3610 static int shmem_reconfigure(struct fs_context *fc)
3611 {
3612         struct shmem_options *ctx = fc->fs_private;
3613         struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3614         unsigned long inodes;
3615         struct mempolicy *mpol = NULL;
3616         const char *err;
3617
3618         raw_spin_lock(&sbinfo->stat_lock);
3619         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3620
3621         if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3622                 if (!sbinfo->max_blocks) {
3623                         err = "Cannot retroactively limit size";
3624                         goto out;
3625                 }
3626                 if (percpu_counter_compare(&sbinfo->used_blocks,
3627                                            ctx->blocks) > 0) {
3628                         err = "Too small a size for current use";
3629                         goto out;
3630                 }
3631         }
3632         if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3633                 if (!sbinfo->max_inodes) {
3634                         err = "Cannot retroactively limit inodes";
3635                         goto out;
3636                 }
3637                 if (ctx->inodes < inodes) {
3638                         err = "Too few inodes for current use";
3639                         goto out;
3640                 }
3641         }
3642
3643         if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
3644             sbinfo->next_ino > UINT_MAX) {
3645                 err = "Current inum too high to switch to 32-bit inums";
3646                 goto out;
3647         }
3648
3649         if (ctx->seen & SHMEM_SEEN_HUGE)
3650                 sbinfo->huge = ctx->huge;
3651         if (ctx->seen & SHMEM_SEEN_INUMS)
3652                 sbinfo->full_inums = ctx->full_inums;
3653         if (ctx->seen & SHMEM_SEEN_BLOCKS)
3654                 sbinfo->max_blocks  = ctx->blocks;
3655         if (ctx->seen & SHMEM_SEEN_INODES) {
3656                 sbinfo->max_inodes  = ctx->inodes;
3657                 sbinfo->free_inodes = ctx->inodes - inodes;
3658         }
3659
3660         /*
3661          * Preserve previous mempolicy unless mpol remount option was specified.
3662          */
3663         if (ctx->mpol) {
3664                 mpol = sbinfo->mpol;
3665                 sbinfo->mpol = ctx->mpol;       /* transfers initial ref */
3666                 ctx->mpol = NULL;
3667         }
3668         raw_spin_unlock(&sbinfo->stat_lock);
3669         mpol_put(mpol);
3670         return 0;
3671 out:
3672         raw_spin_unlock(&sbinfo->stat_lock);
3673         return invalfc(fc, "%s", err);
3674 }
3675
3676 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3677 {
3678         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3679
3680         if (sbinfo->max_blocks != shmem_default_max_blocks())
3681                 seq_printf(seq, ",size=%luk",
3682                         sbinfo->max_blocks << (PAGE_SHIFT - 10));
3683         if (sbinfo->max_inodes != shmem_default_max_inodes())
3684                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3685         if (sbinfo->mode != (0777 | S_ISVTX))
3686                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3687         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3688                 seq_printf(seq, ",uid=%u",
3689                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
3690         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3691                 seq_printf(seq, ",gid=%u",
3692                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
3693
3694         /*
3695          * Showing inode{64,32} might be useful even if it's the system default,
3696          * since then people don't have to resort to checking both here and
3697          * /proc/config.gz to confirm 64-bit inums were successfully applied
3698          * (which may not even exist if IKCONFIG_PROC isn't enabled).
3699          *
3700          * We hide it when inode64 isn't the default and we are using 32-bit
3701          * inodes, since that probably just means the feature isn't even under
3702          * consideration.
3703          *
3704          * As such:
3705          *
3706          *                     +-----------------+-----------------+
3707          *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
3708          *  +------------------+-----------------+-----------------+
3709          *  | full_inums=true  | show            | show            |
3710          *  | full_inums=false | show            | hide            |
3711          *  +------------------+-----------------+-----------------+
3712          *
3713          */
3714         if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
3715                 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
3716 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3717         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3718         if (sbinfo->huge)
3719                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3720 #endif
3721         shmem_show_mpol(seq, sbinfo->mpol);
3722         return 0;
3723 }
3724
3725 #endif /* CONFIG_TMPFS */
3726
3727 static void shmem_put_super(struct super_block *sb)
3728 {
3729         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3730
3731         free_percpu(sbinfo->ino_batch);
3732         percpu_counter_destroy(&sbinfo->used_blocks);
3733         mpol_put(sbinfo->mpol);
3734         kfree(sbinfo);
3735         sb->s_fs_info = NULL;
3736 }
3737
3738 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3739 {
3740         struct shmem_options *ctx = fc->fs_private;
3741         struct inode *inode;
3742         struct shmem_sb_info *sbinfo;
3743
3744         /* Round up to L1_CACHE_BYTES to resist false sharing */
3745         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3746                                 L1_CACHE_BYTES), GFP_KERNEL);
3747         if (!sbinfo)
3748                 return -ENOMEM;
3749
3750         sb->s_fs_info = sbinfo;
3751
3752 #ifdef CONFIG_TMPFS
3753         /*
3754          * Per default we only allow half of the physical ram per
3755          * tmpfs instance, limiting inodes to one per page of lowmem;
3756          * but the internal instance is left unlimited.
3757          */
3758         if (!(sb->s_flags & SB_KERNMOUNT)) {
3759                 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3760                         ctx->blocks = shmem_default_max_blocks();
3761                 if (!(ctx->seen & SHMEM_SEEN_INODES))
3762                         ctx->inodes = shmem_default_max_inodes();
3763                 if (!(ctx->seen & SHMEM_SEEN_INUMS))
3764                         ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
3765         } else {
3766                 sb->s_flags |= SB_NOUSER;
3767         }
3768         sb->s_export_op = &shmem_export_ops;
3769         sb->s_flags |= SB_NOSEC | SB_I_VERSION;
3770 #else
3771         sb->s_flags |= SB_NOUSER;
3772 #endif
3773         sbinfo->max_blocks = ctx->blocks;
3774         sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3775         if (sb->s_flags & SB_KERNMOUNT) {
3776                 sbinfo->ino_batch = alloc_percpu(ino_t);
3777                 if (!sbinfo->ino_batch)
3778                         goto failed;
3779         }
3780         sbinfo->uid = ctx->uid;
3781         sbinfo->gid = ctx->gid;
3782         sbinfo->full_inums = ctx->full_inums;
3783         sbinfo->mode = ctx->mode;
3784         sbinfo->huge = ctx->huge;
3785         sbinfo->mpol = ctx->mpol;
3786         ctx->mpol = NULL;
3787
3788         raw_spin_lock_init(&sbinfo->stat_lock);
3789         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3790                 goto failed;
3791         spin_lock_init(&sbinfo->shrinklist_lock);
3792         INIT_LIST_HEAD(&sbinfo->shrinklist);
3793
3794         sb->s_maxbytes = MAX_LFS_FILESIZE;
3795         sb->s_blocksize = PAGE_SIZE;
3796         sb->s_blocksize_bits = PAGE_SHIFT;
3797         sb->s_magic = TMPFS_MAGIC;
3798         sb->s_op = &shmem_ops;
3799         sb->s_time_gran = 1;
3800 #ifdef CONFIG_TMPFS_XATTR
3801         sb->s_xattr = shmem_xattr_handlers;
3802 #endif
3803 #ifdef CONFIG_TMPFS_POSIX_ACL
3804         sb->s_flags |= SB_POSIXACL;
3805 #endif
3806         uuid_gen(&sb->s_uuid);
3807
3808         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3809         if (!inode)
3810                 goto failed;
3811         inode->i_uid = sbinfo->uid;
3812         inode->i_gid = sbinfo->gid;
3813         sb->s_root = d_make_root(inode);
3814         if (!sb->s_root)
3815                 goto failed;
3816         return 0;
3817
3818 failed:
3819         shmem_put_super(sb);
3820         return -ENOMEM;
3821 }
3822
3823 static int shmem_get_tree(struct fs_context *fc)
3824 {
3825         return get_tree_nodev(fc, shmem_fill_super);
3826 }
3827
3828 static void shmem_free_fc(struct fs_context *fc)
3829 {
3830         struct shmem_options *ctx = fc->fs_private;
3831
3832         if (ctx) {
3833                 mpol_put(ctx->mpol);
3834                 kfree(ctx);
3835         }
3836 }
3837
3838 static const struct fs_context_operations shmem_fs_context_ops = {
3839         .free                   = shmem_free_fc,
3840         .get_tree               = shmem_get_tree,
3841 #ifdef CONFIG_TMPFS
3842         .parse_monolithic       = shmem_parse_options,
3843         .parse_param            = shmem_parse_one,
3844         .reconfigure            = shmem_reconfigure,
3845 #endif
3846 };
3847
3848 static struct kmem_cache *shmem_inode_cachep;
3849
3850 static struct inode *shmem_alloc_inode(struct super_block *sb)
3851 {
3852         struct shmem_inode_info *info;
3853         info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
3854         if (!info)
3855                 return NULL;
3856         return &info->vfs_inode;
3857 }
3858
3859 static void shmem_free_in_core_inode(struct inode *inode)
3860 {
3861         if (S_ISLNK(inode->i_mode))
3862                 kfree(inode->i_link);
3863         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3864 }
3865
3866 static void shmem_destroy_inode(struct inode *inode)
3867 {
3868         if (S_ISREG(inode->i_mode))
3869                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3870 }
3871
3872 static void shmem_init_inode(void *foo)
3873 {
3874         struct shmem_inode_info *info = foo;
3875         inode_init_once(&info->vfs_inode);
3876 }
3877
3878 static void shmem_init_inodecache(void)
3879 {
3880         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3881                                 sizeof(struct shmem_inode_info),
3882                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3883 }
3884
3885 static void shmem_destroy_inodecache(void)
3886 {
3887         kmem_cache_destroy(shmem_inode_cachep);
3888 }
3889
3890 /* Keep the page in page cache instead of truncating it */
3891 static int shmem_error_remove_page(struct address_space *mapping,
3892                                    struct page *page)
3893 {
3894         return 0;
3895 }
3896
3897 const struct address_space_operations shmem_aops = {
3898         .writepage      = shmem_writepage,
3899         .dirty_folio    = noop_dirty_folio,
3900 #ifdef CONFIG_TMPFS
3901         .write_begin    = shmem_write_begin,
3902         .write_end      = shmem_write_end,
3903 #endif
3904 #ifdef CONFIG_MIGRATION
3905         .migrate_folio  = migrate_folio,
3906 #endif
3907         .error_remove_page = shmem_error_remove_page,
3908 };
3909 EXPORT_SYMBOL(shmem_aops);
3910
3911 static const struct file_operations shmem_file_operations = {
3912         .mmap           = shmem_mmap,
3913         .get_unmapped_area = shmem_get_unmapped_area,
3914 #ifdef CONFIG_TMPFS
3915         .llseek         = shmem_file_llseek,
3916         .read_iter      = shmem_file_read_iter,
3917         .write_iter     = generic_file_write_iter,
3918         .fsync          = noop_fsync,
3919         .splice_read    = generic_file_splice_read,
3920         .splice_write   = iter_file_splice_write,
3921         .fallocate      = shmem_fallocate,
3922 #endif
3923 };
3924
3925 static const struct inode_operations shmem_inode_operations = {
3926         .getattr        = shmem_getattr,
3927         .setattr        = shmem_setattr,
3928 #ifdef CONFIG_TMPFS_XATTR
3929         .listxattr      = shmem_listxattr,
3930         .set_acl        = simple_set_acl,
3931         .fileattr_get   = shmem_fileattr_get,
3932         .fileattr_set   = shmem_fileattr_set,
3933 #endif
3934 };
3935
3936 static const struct inode_operations shmem_dir_inode_operations = {
3937 #ifdef CONFIG_TMPFS
3938         .getattr        = shmem_getattr,
3939         .create         = shmem_create,
3940         .lookup         = simple_lookup,
3941         .link           = shmem_link,
3942         .unlink         = shmem_unlink,
3943         .symlink        = shmem_symlink,
3944         .mkdir          = shmem_mkdir,
3945         .rmdir          = shmem_rmdir,
3946         .mknod          = shmem_mknod,
3947         .rename         = shmem_rename2,
3948         .tmpfile        = shmem_tmpfile,
3949 #endif
3950 #ifdef CONFIG_TMPFS_XATTR
3951         .listxattr      = shmem_listxattr,
3952         .fileattr_get   = shmem_fileattr_get,
3953         .fileattr_set   = shmem_fileattr_set,
3954 #endif
3955 #ifdef CONFIG_TMPFS_POSIX_ACL
3956         .setattr        = shmem_setattr,
3957         .set_acl        = simple_set_acl,
3958 #endif
3959 };
3960
3961 static const struct inode_operations shmem_special_inode_operations = {
3962         .getattr        = shmem_getattr,
3963 #ifdef CONFIG_TMPFS_XATTR
3964         .listxattr      = shmem_listxattr,
3965 #endif
3966 #ifdef CONFIG_TMPFS_POSIX_ACL
3967         .setattr        = shmem_setattr,
3968         .set_acl        = simple_set_acl,
3969 #endif
3970 };
3971
3972 static const struct super_operations shmem_ops = {
3973         .alloc_inode    = shmem_alloc_inode,
3974         .free_inode     = shmem_free_in_core_inode,
3975         .destroy_inode  = shmem_destroy_inode,
3976 #ifdef CONFIG_TMPFS
3977         .statfs         = shmem_statfs,
3978         .show_options   = shmem_show_options,
3979 #endif
3980         .evict_inode    = shmem_evict_inode,
3981         .drop_inode     = generic_delete_inode,
3982         .put_super      = shmem_put_super,
3983 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3984         .nr_cached_objects      = shmem_unused_huge_count,
3985         .free_cached_objects    = shmem_unused_huge_scan,
3986 #endif
3987 };
3988
3989 static const struct vm_operations_struct shmem_vm_ops = {
3990         .fault          = shmem_fault,
3991         .map_pages      = filemap_map_pages,
3992 #ifdef CONFIG_NUMA
3993         .set_policy     = shmem_set_policy,
3994         .get_policy     = shmem_get_policy,
3995 #endif
3996 };
3997
3998 int shmem_init_fs_context(struct fs_context *fc)
3999 {
4000         struct shmem_options *ctx;
4001
4002         ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
4003         if (!ctx)
4004                 return -ENOMEM;
4005
4006         ctx->mode = 0777 | S_ISVTX;
4007         ctx->uid = current_fsuid();
4008         ctx->gid = current_fsgid();
4009
4010         fc->fs_private = ctx;
4011         fc->ops = &shmem_fs_context_ops;
4012         return 0;
4013 }
4014
4015 static struct file_system_type shmem_fs_type = {
4016         .owner          = THIS_MODULE,
4017         .name           = "tmpfs",
4018         .init_fs_context = shmem_init_fs_context,
4019 #ifdef CONFIG_TMPFS
4020         .parameters     = shmem_fs_parameters,
4021 #endif
4022         .kill_sb        = kill_litter_super,
4023         .fs_flags       = FS_USERNS_MOUNT,
4024 };
4025
4026 void __init shmem_init(void)
4027 {
4028         int error;
4029
4030         shmem_init_inodecache();
4031
4032         error = register_filesystem(&shmem_fs_type);
4033         if (error) {
4034                 pr_err("Could not register tmpfs\n");
4035                 goto out2;
4036         }
4037
4038         shm_mnt = kern_mount(&shmem_fs_type);
4039         if (IS_ERR(shm_mnt)) {
4040                 error = PTR_ERR(shm_mnt);
4041                 pr_err("Could not kern_mount tmpfs\n");
4042                 goto out1;
4043         }
4044
4045 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4046         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4047                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4048         else
4049                 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4050 #endif
4051         return;
4052
4053 out1:
4054         unregister_filesystem(&shmem_fs_type);
4055 out2:
4056         shmem_destroy_inodecache();
4057         shm_mnt = ERR_PTR(error);
4058 }
4059
4060 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
4061 static ssize_t shmem_enabled_show(struct kobject *kobj,
4062                                   struct kobj_attribute *attr, char *buf)
4063 {
4064         static const int values[] = {
4065                 SHMEM_HUGE_ALWAYS,
4066                 SHMEM_HUGE_WITHIN_SIZE,
4067                 SHMEM_HUGE_ADVISE,
4068                 SHMEM_HUGE_NEVER,
4069                 SHMEM_HUGE_DENY,
4070                 SHMEM_HUGE_FORCE,
4071         };
4072         int len = 0;
4073         int i;
4074
4075         for (i = 0; i < ARRAY_SIZE(values); i++) {
4076                 len += sysfs_emit_at(buf, len,
4077                                      shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4078                                      i ? " " : "",
4079                                      shmem_format_huge(values[i]));
4080         }
4081
4082         len += sysfs_emit_at(buf, len, "\n");
4083
4084         return len;
4085 }
4086
4087 static ssize_t shmem_enabled_store(struct kobject *kobj,
4088                 struct kobj_attribute *attr, const char *buf, size_t count)
4089 {
4090         char tmp[16];
4091         int huge;
4092
4093         if (count + 1 > sizeof(tmp))
4094                 return -EINVAL;
4095         memcpy(tmp, buf, count);
4096         tmp[count] = '\0';
4097         if (count && tmp[count - 1] == '\n')
4098                 tmp[count - 1] = '\0';
4099
4100         huge = shmem_parse_huge(tmp);
4101         if (huge == -EINVAL)
4102                 return -EINVAL;
4103         if (!has_transparent_hugepage() &&
4104                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4105                 return -EINVAL;
4106
4107         shmem_huge = huge;
4108         if (shmem_huge > SHMEM_HUGE_DENY)
4109                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4110         return count;
4111 }
4112
4113 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4114 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4115
4116 #else /* !CONFIG_SHMEM */
4117
4118 /*
4119  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4120  *
4121  * This is intended for small system where the benefits of the full
4122  * shmem code (swap-backed and resource-limited) are outweighed by
4123  * their complexity. On systems without swap this code should be
4124  * effectively equivalent, but much lighter weight.
4125  */
4126
4127 static struct file_system_type shmem_fs_type = {
4128         .name           = "tmpfs",
4129         .init_fs_context = ramfs_init_fs_context,
4130         .parameters     = ramfs_fs_parameters,
4131         .kill_sb        = kill_litter_super,
4132         .fs_flags       = FS_USERNS_MOUNT,
4133 };
4134
4135 void __init shmem_init(void)
4136 {
4137         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4138
4139         shm_mnt = kern_mount(&shmem_fs_type);
4140         BUG_ON(IS_ERR(shm_mnt));
4141 }
4142
4143 int shmem_unuse(unsigned int type)
4144 {
4145         return 0;
4146 }
4147
4148 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4149 {
4150         return 0;
4151 }
4152
4153 void shmem_unlock_mapping(struct address_space *mapping)
4154 {
4155 }
4156
4157 #ifdef CONFIG_MMU
4158 unsigned long shmem_get_unmapped_area(struct file *file,
4159                                       unsigned long addr, unsigned long len,
4160                                       unsigned long pgoff, unsigned long flags)
4161 {
4162         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4163 }
4164 #endif
4165
4166 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4167 {
4168         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4169 }
4170 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4171
4172 #define shmem_vm_ops                            generic_file_vm_ops
4173 #define shmem_file_operations                   ramfs_file_operations
4174 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
4175 #define shmem_acct_size(flags, size)            0
4176 #define shmem_unacct_size(flags, size)          do {} while (0)
4177
4178 #endif /* CONFIG_SHMEM */
4179
4180 /* common code */
4181
4182 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4183                                        unsigned long flags, unsigned int i_flags)
4184 {
4185         struct inode *inode;
4186         struct file *res;
4187
4188         if (IS_ERR(mnt))
4189                 return ERR_CAST(mnt);
4190
4191         if (size < 0 || size > MAX_LFS_FILESIZE)
4192                 return ERR_PTR(-EINVAL);
4193
4194         if (shmem_acct_size(flags, size))
4195                 return ERR_PTR(-ENOMEM);
4196
4197         inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4198                                 flags);
4199         if (unlikely(!inode)) {
4200                 shmem_unacct_size(flags, size);
4201                 return ERR_PTR(-ENOSPC);
4202         }
4203         inode->i_flags |= i_flags;
4204         inode->i_size = size;
4205         clear_nlink(inode);     /* It is unlinked */
4206         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4207         if (!IS_ERR(res))
4208                 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4209                                 &shmem_file_operations);
4210         if (IS_ERR(res))
4211                 iput(inode);
4212         return res;
4213 }
4214
4215 /**
4216  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4217  *      kernel internal.  There will be NO LSM permission checks against the
4218  *      underlying inode.  So users of this interface must do LSM checks at a
4219  *      higher layer.  The users are the big_key and shm implementations.  LSM
4220  *      checks are provided at the key or shm level rather than the inode.
4221  * @name: name for dentry (to be seen in /proc/<pid>/maps
4222  * @size: size to be set for the file
4223  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4224  */
4225 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4226 {
4227         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4228 }
4229
4230 /**
4231  * shmem_file_setup - get an unlinked file living in tmpfs
4232  * @name: name for dentry (to be seen in /proc/<pid>/maps
4233  * @size: size to be set for the file
4234  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4235  */
4236 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4237 {
4238         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4239 }
4240 EXPORT_SYMBOL_GPL(shmem_file_setup);
4241
4242 /**
4243  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4244  * @mnt: the tmpfs mount where the file will be created
4245  * @name: name for dentry (to be seen in /proc/<pid>/maps
4246  * @size: size to be set for the file
4247  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4248  */
4249 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4250                                        loff_t size, unsigned long flags)
4251 {
4252         return __shmem_file_setup(mnt, name, size, flags, 0);
4253 }
4254 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4255
4256 /**
4257  * shmem_zero_setup - setup a shared anonymous mapping
4258  * @vma: the vma to be mmapped is prepared by do_mmap
4259  */
4260 int shmem_zero_setup(struct vm_area_struct *vma)
4261 {
4262         struct file *file;
4263         loff_t size = vma->vm_end - vma->vm_start;
4264
4265         /*
4266          * Cloning a new file under mmap_lock leads to a lock ordering conflict
4267          * between XFS directory reading and selinux: since this file is only
4268          * accessible to the user through its mapping, use S_PRIVATE flag to
4269          * bypass file security, in the same way as shmem_kernel_file_setup().
4270          */
4271         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4272         if (IS_ERR(file))
4273                 return PTR_ERR(file);
4274
4275         if (vma->vm_file)
4276                 fput(vma->vm_file);
4277         vma->vm_file = file;
4278         vma->vm_ops = &shmem_vm_ops;
4279
4280         return 0;
4281 }
4282
4283 /**
4284  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4285  * @mapping:    the page's address_space
4286  * @index:      the page index
4287  * @gfp:        the page allocator flags to use if allocating
4288  *
4289  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4290  * with any new page allocations done using the specified allocation flags.
4291  * But read_cache_page_gfp() uses the ->read_folio() method: which does not
4292  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4293  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4294  *
4295  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4296  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4297  */
4298 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4299                                          pgoff_t index, gfp_t gfp)
4300 {
4301 #ifdef CONFIG_SHMEM
4302         struct inode *inode = mapping->host;
4303         struct folio *folio;
4304         struct page *page;
4305         int error;
4306
4307         BUG_ON(!shmem_mapping(mapping));
4308         error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
4309                                   gfp, NULL, NULL, NULL);
4310         if (error)
4311                 return ERR_PTR(error);
4312
4313         folio_unlock(folio);
4314         page = folio_file_page(folio, index);
4315         if (PageHWPoison(page)) {
4316                 folio_put(folio);
4317                 return ERR_PTR(-EIO);
4318         }
4319
4320         return page;
4321 #else
4322         /*
4323          * The tiny !SHMEM case uses ramfs without swap
4324          */
4325         return read_cache_page_gfp(mapping, index, gfp);
4326 #endif
4327 }
4328 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);