mm/shmem.c

   1 /*
   2  * Resizable virtual memory filesystem for Linux.
   3  *
   4  * Copyright (C) 2000 Linus Torvalds.
   5  *               2000 Transmeta Corp.
   6  *               2000-2001 Christoph Rohland
   7  *               2000-2001 SAP AG
   8  *               2002 Red Hat Inc.
   9  * Copyright (C) 2002-2011 Hugh Dickins.
  10  * Copyright (C) 2011 Google Inc.
  11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
  12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
  13  *
  14  * Extended attribute support for tmpfs:
  15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  17  *
  18  * tiny-shmem:
  19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  20  *
  21  * This file is released under the GPL.
  22  */
  23
  24 #include <linux/fs.h>
  25 #include <linux/init.h>
  26 #include <linux/vfs.h>
  27 #include <linux/mount.h>
  28 #include <linux/ramfs.h>
  29 #include <linux/pagemap.h>
  30 #include <linux/file.h>
  31 #include <linux/fileattr.h>
  32 #include <linux/mm.h>
  33 #include <linux/random.h>
  34 #include <linux/sched/signal.h>
  35 #include <linux/export.h>
  36 #include <linux/shmem_fs.h>
  37 #include <linux/swap.h>
  38 #include <linux/uio.h>
  39 #include <linux/hugetlb.h>
  40 #include <linux/fs_parser.h>
  41 #include <linux/swapfile.h>
  42 #include <linux/iversion.h>
  43 #include "swap.h"
  44
  45 static struct vfsmount *shm_mnt;
  46
  47 #ifdef CONFIG_SHMEM
  48 /*
  49  * This virtual memory filesystem is heavily based on the ramfs. It
  50  * extends ramfs by the ability to use swap and honor resource limits
  51  * which makes it a completely usable filesystem.
  52  */
  53
  54 #include <linux/xattr.h>
  55 #include <linux/exportfs.h>
  56 #include <linux/posix_acl.h>
  57 #include <linux/posix_acl_xattr.h>
  58 #include <linux/mman.h>
  59 #include <linux/string.h>
  60 #include <linux/slab.h>
  61 #include <linux/backing-dev.h>
  62 #include <linux/writeback.h>
  63 #include <linux/pagevec.h>
  64 #include <linux/percpu_counter.h>
  65 #include <linux/falloc.h>
  66 #include <linux/splice.h>
  67 #include <linux/security.h>
  68 #include <linux/swapops.h>
  69 #include <linux/mempolicy.h>
  70 #include <linux/namei.h>
  71 #include <linux/ctype.h>
  72 #include <linux/migrate.h>
  73 #include <linux/highmem.h>
  74 #include <linux/seq_file.h>
  75 #include <linux/magic.h>
  76 #include <linux/syscalls.h>
  77 #include <linux/fcntl.h>
  78 #include <uapi/linux/memfd.h>
  79 #include <linux/rmap.h>
  80 #include <linux/uuid.h>
  81 #include <linux/quotaops.h>
  82
  83 #include <linux/uaccess.h>
  84
  85 #include "internal.h"
  86
  87 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
  88 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
  89
  90 /* Pretend that each entry is of this size in directory's i_size */
  91 #define BOGO_DIRENT_SIZE 20
  92
  93 /* Pretend that one inode + its dentry occupy this much memory */
  94 #define BOGO_INODE_SIZE 1024
  95
  96 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
  97 #define SHORT_SYMLINK_LEN 128
  98
  99 /*
 100  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 101  * inode->i_private (with i_rwsem making sure that it has only one user at
 102  * a time): we would prefer not to enlarge the shmem inode just for that.
 103  */
 104 struct shmem_falloc {
 105         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
 106         pgoff_t start;          /* start of range currently being fallocated */
 107         pgoff_t next;           /* the next page offset to be fallocated */
 108         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
 109         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
 110 };
 111
 112 struct shmem_options {
 113         unsigned long long blocks;
 114         unsigned long long inodes;
 115         struct mempolicy *mpol;
 116         kuid_t uid;
 117         kgid_t gid;
 118         umode_t mode;
 119         bool full_inums;
 120         int huge;
 121         int seen;
 122         bool noswap;
 123         unsigned short quota_types;
 124         struct shmem_quota_limits qlimits;
 125 #define SHMEM_SEEN_BLOCKS 1
 126 #define SHMEM_SEEN_INODES 2
 127 #define SHMEM_SEEN_HUGE 4
 128 #define SHMEM_SEEN_INUMS 8
 129 #define SHMEM_SEEN_NOSWAP 16
 130 #define SHMEM_SEEN_QUOTA 32
 131 };
 132
 133 #ifdef CONFIG_TMPFS
 134 static unsigned long shmem_default_max_blocks(void)
 135 {
 136         return totalram_pages() / 2;
 137 }
 138
 139 static unsigned long shmem_default_max_inodes(void)
 140 {
 141         unsigned long nr_pages = totalram_pages();
 142
 143         return min3(nr_pages - totalhigh_pages(), nr_pages / 2,
 144                         ULONG_MAX / BOGO_INODE_SIZE);
 145 }
 146 #endif
 147
 148 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
 149                              struct folio **foliop, enum sgp_type sgp,
 150                              gfp_t gfp, struct vm_area_struct *vma,
 151                              vm_fault_t *fault_type);
 152
 153 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
 154 {
 155         return sb->s_fs_info;
 156 }
 157
 158 /*
 159  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 160  * for shared memory and for shared anonymous (/dev/zero) mappings
 161  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 162  * consistent with the pre-accounting of private mappings ...
 163  */
 164 static inline int shmem_acct_size(unsigned long flags, loff_t size)
 165 {
 166         return (flags & VM_NORESERVE) ?
 167                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
 168 }
 169
 170 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 171 {
 172         if (!(flags & VM_NORESERVE))
 173                 vm_unacct_memory(VM_ACCT(size));
 174 }
 175
 176 static inline int shmem_reacct_size(unsigned long flags,
 177                 loff_t oldsize, loff_t newsize)
 178 {
 179         if (!(flags & VM_NORESERVE)) {
 180                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
 181                         return security_vm_enough_memory_mm(current->mm,
 182                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
 183                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
 184                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
 185         }
 186         return 0;
 187 }
 188
 189 /*
 190  * ... whereas tmpfs objects are accounted incrementally as
 191  * pages are allocated, in order to allow large sparse files.
 192  * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
 193  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 194  */
 195 static inline int shmem_acct_block(unsigned long flags, long pages)
 196 {
 197         if (!(flags & VM_NORESERVE))
 198                 return 0;
 199
 200         return security_vm_enough_memory_mm(current->mm,
 201                         pages * VM_ACCT(PAGE_SIZE));
 202 }
 203
 204 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
 205 {
 206         if (flags & VM_NORESERVE)
 207                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
 208 }
 209
 210 static int shmem_inode_acct_block(struct inode *inode, long pages)
 211 {
 212         struct shmem_inode_info *info = SHMEM_I(inode);
 213         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 214         int err = -ENOSPC;
 215
 216         if (shmem_acct_block(info->flags, pages))
 217                 return err;
 218
 219         might_sleep();  /* when quotas */
 220         if (sbinfo->max_blocks) {
 221                 if (percpu_counter_compare(&sbinfo->used_blocks,
 222                                            sbinfo->max_blocks - pages) > 0)
 223                         goto unacct;
 224
 225                 err = dquot_alloc_block_nodirty(inode, pages);
 226                 if (err)
 227                         goto unacct;
 228
 229                 percpu_counter_add(&sbinfo->used_blocks, pages);
 230         } else {
 231                 err = dquot_alloc_block_nodirty(inode, pages);
 232                 if (err)
 233                         goto unacct;
 234         }
 235
 236         return 0;
 237
 238 unacct:
 239         shmem_unacct_blocks(info->flags, pages);
 240         return err;
 241 }
 242
 243 static void shmem_inode_unacct_blocks(struct inode *inode, long pages)
 244 {
 245         struct shmem_inode_info *info = SHMEM_I(inode);
 246         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 247
 248         might_sleep();  /* when quotas */
 249         dquot_free_block_nodirty(inode, pages);
 250
 251         if (sbinfo->max_blocks)
 252                 percpu_counter_sub(&sbinfo->used_blocks, pages);
 253         shmem_unacct_blocks(info->flags, pages);
 254 }
 255
 256 static const struct super_operations shmem_ops;
 257 const struct address_space_operations shmem_aops;
 258 static const struct file_operations shmem_file_operations;
 259 static const struct inode_operations shmem_inode_operations;
 260 static const struct inode_operations shmem_dir_inode_operations;
 261 static const struct inode_operations shmem_special_inode_operations;
 262 static const struct vm_operations_struct shmem_vm_ops;
 263 static const struct vm_operations_struct shmem_anon_vm_ops;
 264 static struct file_system_type shmem_fs_type;
 265
 266 bool vma_is_anon_shmem(struct vm_area_struct *vma)
 267 {
 268         return vma->vm_ops == &shmem_anon_vm_ops;
 269 }
 270
 271 bool vma_is_shmem(struct vm_area_struct *vma)
 272 {
 273         return vma_is_anon_shmem(vma) || vma->vm_ops == &shmem_vm_ops;
 274 }
 275
 276 static LIST_HEAD(shmem_swaplist);
 277 static DEFINE_MUTEX(shmem_swaplist_mutex);
 278
 279 #ifdef CONFIG_TMPFS_QUOTA
 280
 281 static int shmem_enable_quotas(struct super_block *sb,
 282                                unsigned short quota_types)
 283 {
 284         int type, err = 0;
 285
 286         sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NOLIST_DIRTY;
 287         for (type = 0; type < SHMEM_MAXQUOTAS; type++) {
 288                 if (!(quota_types & (1 << type)))
 289                         continue;
 290                 err = dquot_load_quota_sb(sb, type, QFMT_SHMEM,
 291                                           DQUOT_USAGE_ENABLED |
 292                                           DQUOT_LIMITS_ENABLED);
 293                 if (err)
 294                         goto out_err;
 295         }
 296         return 0;
 297
 298 out_err:
 299         pr_warn("tmpfs: failed to enable quota tracking (type=%d, err=%d)\n",
 300                 type, err);
 301         for (type--; type >= 0; type--)
 302                 dquot_quota_off(sb, type);
 303         return err;
 304 }
 305
 306 static void shmem_disable_quotas(struct super_block *sb)
 307 {
 308         int type;
 309
 310         for (type = 0; type < SHMEM_MAXQUOTAS; type++)
 311                 dquot_quota_off(sb, type);
 312 }
 313
 314 static struct dquot **shmem_get_dquots(struct inode *inode)
 315 {
 316         return SHMEM_I(inode)->i_dquot;
 317 }
 318 #endif /* CONFIG_TMPFS_QUOTA */
 319
 320 /*
 321  * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
 322  * produces a novel ino for the newly allocated inode.
 323  *
 324  * It may also be called when making a hard link to permit the space needed by
 325  * each dentry. However, in that case, no new inode number is needed since that
 326  * internally draws from another pool of inode numbers (currently global
 327  * get_next_ino()). This case is indicated by passing NULL as inop.
 328  */
 329 #define SHMEM_INO_BATCH 1024
 330 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
 331 {
 332         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 333         ino_t ino;
 334
 335         if (!(sb->s_flags & SB_KERNMOUNT)) {
 336                 raw_spin_lock(&sbinfo->stat_lock);
 337                 if (sbinfo->max_inodes) {
 338                         if (sbinfo->free_ispace < BOGO_INODE_SIZE) {
 339                                 raw_spin_unlock(&sbinfo->stat_lock);
 340                                 return -ENOSPC;
 341                         }
 342                         sbinfo->free_ispace -= BOGO_INODE_SIZE;
 343                 }
 344                 if (inop) {
 345                         ino = sbinfo->next_ino++;
 346                         if (unlikely(is_zero_ino(ino)))
 347                                 ino = sbinfo->next_ino++;
 348                         if (unlikely(!sbinfo->full_inums &&
 349                                      ino > UINT_MAX)) {
 350                                 /*
 351                                  * Emulate get_next_ino uint wraparound for
 352                                  * compatibility
 353                                  */
 354                                 if (IS_ENABLED(CONFIG_64BIT))
 355                                         pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
 356                                                 __func__, MINOR(sb->s_dev));
 357                                 sbinfo->next_ino = 1;
 358                                 ino = sbinfo->next_ino++;
 359                         }
 360                         *inop = ino;
 361                 }
 362                 raw_spin_unlock(&sbinfo->stat_lock);
 363         } else if (inop) {
 364                 /*
 365                  * __shmem_file_setup, one of our callers, is lock-free: it
 366                  * doesn't hold stat_lock in shmem_reserve_inode since
 367                  * max_inodes is always 0, and is called from potentially
 368                  * unknown contexts. As such, use a per-cpu batched allocator
 369                  * which doesn't require the per-sb stat_lock unless we are at
 370                  * the batch boundary.
 371                  *
 372                  * We don't need to worry about inode{32,64} since SB_KERNMOUNT
 373                  * shmem mounts are not exposed to userspace, so we don't need
 374                  * to worry about things like glibc compatibility.
 375                  */
 376                 ino_t *next_ino;
 377
 378                 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
 379                 ino = *next_ino;
 380                 if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
 381                         raw_spin_lock(&sbinfo->stat_lock);
 382                         ino = sbinfo->next_ino;
 383                         sbinfo->next_ino += SHMEM_INO_BATCH;
 384                         raw_spin_unlock(&sbinfo->stat_lock);
 385                         if (unlikely(is_zero_ino(ino)))
 386                                 ino++;
 387                 }
 388                 *inop = ino;
 389                 *next_ino = ++ino;
 390                 put_cpu();
 391         }
 392
 393         return 0;
 394 }
 395
 396 static void shmem_free_inode(struct super_block *sb, size_t freed_ispace)
 397 {
 398         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 399         if (sbinfo->max_inodes) {
 400                 raw_spin_lock(&sbinfo->stat_lock);
 401                 sbinfo->free_ispace += BOGO_INODE_SIZE + freed_ispace;
 402                 raw_spin_unlock(&sbinfo->stat_lock);
 403         }
 404 }
 405
 406 /**
 407  * shmem_recalc_inode - recalculate the block usage of an inode
 408  * @inode: inode to recalc
 409  * @alloced: the change in number of pages allocated to inode
 410  * @swapped: the change in number of pages swapped from inode
 411  *
 412  * We have to calculate the free blocks since the mm can drop
 413  * undirtied hole pages behind our back.
 414  *
 415  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 416  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 417  */
 418 static void shmem_recalc_inode(struct inode *inode, long alloced, long swapped)
 419 {
 420         struct shmem_inode_info *info = SHMEM_I(inode);
 421         long freed;
 422
 423         spin_lock(&info->lock);
 424         info->alloced += alloced;
 425         info->swapped += swapped;
 426         freed = info->alloced - info->swapped -
 427                 READ_ONCE(inode->i_mapping->nrpages);
 428         /*
 429          * Special case: whereas normally shmem_recalc_inode() is called
 430          * after i_mapping->nrpages has already been adjusted (up or down),
 431          * shmem_writepage() has to raise swapped before nrpages is lowered -
 432          * to stop a racing shmem_recalc_inode() from thinking that a page has
 433          * been freed.  Compensate here, to avoid the need for a followup call.
 434          */
 435         if (swapped > 0)
 436                 freed += swapped;
 437         if (freed > 0)
 438                 info->alloced -= freed;
 439         spin_unlock(&info->lock);
 440
 441         /* The quota case may block */
 442         if (freed > 0)
 443                 shmem_inode_unacct_blocks(inode, freed);
 444 }
 445
 446 bool shmem_charge(struct inode *inode, long pages)
 447 {
 448         struct address_space *mapping = inode->i_mapping;
 449
 450         if (shmem_inode_acct_block(inode, pages))
 451                 return false;
 452
 453         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
 454         xa_lock_irq(&mapping->i_pages);
 455         mapping->nrpages += pages;
 456         xa_unlock_irq(&mapping->i_pages);
 457
 458         shmem_recalc_inode(inode, pages, 0);
 459         return true;
 460 }
 461
 462 void shmem_uncharge(struct inode *inode, long pages)
 463 {
 464         /* pages argument is currently unused: keep it to help debugging */
 465         /* nrpages adjustment done by __filemap_remove_folio() or caller */
 466
 467         shmem_recalc_inode(inode, 0, 0);
 468 }
 469
 470 /*
 471  * Replace item expected in xarray by a new item, while holding xa_lock.
 472  */
 473 static int shmem_replace_entry(struct address_space *mapping,
 474                         pgoff_t index, void *expected, void *replacement)
 475 {
 476         XA_STATE(xas, &mapping->i_pages, index);
 477         void *item;
 478
 479         VM_BUG_ON(!expected);
 480         VM_BUG_ON(!replacement);
 481         item = xas_load(&xas);
 482         if (item != expected)
 483                 return -ENOENT;
 484         xas_store(&xas, replacement);
 485         return 0;
 486 }
 487
 488 /*
 489  * Sometimes, before we decide whether to proceed or to fail, we must check
 490  * that an entry was not already brought back from swap by a racing thread.
 491  *
 492  * Checking page is not enough: by the time a SwapCache page is locked, it
 493  * might be reused, and again be SwapCache, using the same swap as before.
 494  */
 495 static bool shmem_confirm_swap(struct address_space *mapping,
 496                                pgoff_t index, swp_entry_t swap)
 497 {
 498         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
 499 }
 500
 501 /*
 502  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
 503  *
 504  * SHMEM_HUGE_NEVER:
 505  *      disables huge pages for the mount;
 506  * SHMEM_HUGE_ALWAYS:
 507  *      enables huge pages for the mount;
 508  * SHMEM_HUGE_WITHIN_SIZE:
 509  *      only allocate huge pages if the page will be fully within i_size,
 510  *      also respect fadvise()/madvise() hints;
 511  * SHMEM_HUGE_ADVISE:
 512  *      only allocate huge pages if requested with fadvise()/madvise();
 513  */
 514
 515 #define SHMEM_HUGE_NEVER        0
 516 #define SHMEM_HUGE_ALWAYS       1
 517 #define SHMEM_HUGE_WITHIN_SIZE  2
 518 #define SHMEM_HUGE_ADVISE       3
 519
 520 /*
 521  * Special values.
 522  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
 523  *
 524  * SHMEM_HUGE_DENY:
 525  *      disables huge on shm_mnt and all mounts, for emergency use;
 526  * SHMEM_HUGE_FORCE:
 527  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
 528  *
 529  */
 530 #define SHMEM_HUGE_DENY         (-1)
 531 #define SHMEM_HUGE_FORCE        (-2)
 532
 533 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 534 /* ifdef here to avoid bloating shmem.o when not necessary */
 535
 536 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
 537
 538 bool shmem_is_huge(struct inode *inode, pgoff_t index, bool shmem_huge_force,
 539                    struct mm_struct *mm, unsigned long vm_flags)
 540 {
 541         loff_t i_size;
 542
 543         if (!S_ISREG(inode->i_mode))
 544                 return false;
 545         if (mm && ((vm_flags & VM_NOHUGEPAGE) || test_bit(MMF_DISABLE_THP, &mm->flags)))
 546                 return false;
 547         if (shmem_huge == SHMEM_HUGE_DENY)
 548                 return false;
 549         if (shmem_huge_force || shmem_huge == SHMEM_HUGE_FORCE)
 550                 return true;
 551
 552         switch (SHMEM_SB(inode->i_sb)->huge) {
 553         case SHMEM_HUGE_ALWAYS:
 554                 return true;
 555         case SHMEM_HUGE_WITHIN_SIZE:
 556                 index = round_up(index + 1, HPAGE_PMD_NR);
 557                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
 558                 if (i_size >> PAGE_SHIFT >= index)
 559                         return true;
 560                 fallthrough;
 561         case SHMEM_HUGE_ADVISE:
 562                 if (mm && (vm_flags & VM_HUGEPAGE))
 563                         return true;
 564                 fallthrough;
 565         default:
 566                 return false;
 567         }
 568 }
 569
 570 #if defined(CONFIG_SYSFS)
 571 static int shmem_parse_huge(const char *str)
 572 {
 573         if (!strcmp(str, "never"))
 574                 return SHMEM_HUGE_NEVER;
 575         if (!strcmp(str, "always"))
 576                 return SHMEM_HUGE_ALWAYS;
 577         if (!strcmp(str, "within_size"))
 578                 return SHMEM_HUGE_WITHIN_SIZE;
 579         if (!strcmp(str, "advise"))
 580                 return SHMEM_HUGE_ADVISE;
 581         if (!strcmp(str, "deny"))
 582                 return SHMEM_HUGE_DENY;
 583         if (!strcmp(str, "force"))
 584                 return SHMEM_HUGE_FORCE;
 585         return -EINVAL;
 586 }
 587 #endif
 588
 589 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
 590 static const char *shmem_format_huge(int huge)
 591 {
 592         switch (huge) {
 593         case SHMEM_HUGE_NEVER:
 594                 return "never";
 595         case SHMEM_HUGE_ALWAYS:
 596                 return "always";
 597         case SHMEM_HUGE_WITHIN_SIZE:
 598                 return "within_size";
 599         case SHMEM_HUGE_ADVISE:
 600                 return "advise";
 601         case SHMEM_HUGE_DENY:
 602                 return "deny";
 603         case SHMEM_HUGE_FORCE:
 604                 return "force";
 605         default:
 606                 VM_BUG_ON(1);
 607                 return "bad_val";
 608         }
 609 }
 610 #endif
 611
 612 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 613                 struct shrink_control *sc, unsigned long nr_to_split)
 614 {
 615         LIST_HEAD(list), *pos, *next;
 616         LIST_HEAD(to_remove);
 617         struct inode *inode;
 618         struct shmem_inode_info *info;
 619         struct folio *folio;
 620         unsigned long batch = sc ? sc->nr_to_scan : 128;
 621         int split = 0;
 622
 623         if (list_empty(&sbinfo->shrinklist))
 624                 return SHRINK_STOP;
 625
 626         spin_lock(&sbinfo->shrinklist_lock);
 627         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
 628                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 629
 630                 /* pin the inode */
 631                 inode = igrab(&info->vfs_inode);
 632
 633                 /* inode is about to be evicted */
 634                 if (!inode) {
 635                         list_del_init(&info->shrinklist);
 636                         goto next;
 637                 }
 638
 639                 /* Check if there's anything to gain */
 640                 if (round_up(inode->i_size, PAGE_SIZE) ==
 641                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
 642                         list_move(&info->shrinklist, &to_remove);
 643                         goto next;
 644                 }
 645
 646                 list_move(&info->shrinklist, &list);
 647 next:
 648                 sbinfo->shrinklist_len--;
 649                 if (!--batch)
 650                         break;
 651         }
 652         spin_unlock(&sbinfo->shrinklist_lock);
 653
 654         list_for_each_safe(pos, next, &to_remove) {
 655                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 656                 inode = &info->vfs_inode;
 657                 list_del_init(&info->shrinklist);
 658                 iput(inode);
 659         }
 660
 661         list_for_each_safe(pos, next, &list) {
 662                 int ret;
 663                 pgoff_t index;
 664
 665                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 666                 inode = &info->vfs_inode;
 667
 668                 if (nr_to_split && split >= nr_to_split)
 669                         goto move_back;
 670
 671                 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
 672                 folio = filemap_get_folio(inode->i_mapping, index);
 673                 if (IS_ERR(folio))
 674                         goto drop;
 675
 676                 /* No huge page at the end of the file: nothing to split */
 677                 if (!folio_test_large(folio)) {
 678                         folio_put(folio);
 679                         goto drop;
 680                 }
 681
 682                 /*
 683                  * Move the inode on the list back to shrinklist if we failed
 684                  * to lock the page at this time.
 685                  *
 686                  * Waiting for the lock may lead to deadlock in the
 687                  * reclaim path.
 688                  */
 689                 if (!folio_trylock(folio)) {
 690                         folio_put(folio);
 691                         goto move_back;
 692                 }
 693
 694                 ret = split_folio(folio);
 695                 folio_unlock(folio);
 696                 folio_put(folio);
 697
 698                 /* If split failed move the inode on the list back to shrinklist */
 699                 if (ret)
 700                         goto move_back;
 701
 702                 split++;
 703 drop:
 704                 list_del_init(&info->shrinklist);
 705                 goto put;
 706 move_back:
 707                 /*
 708                  * Make sure the inode is either on the global list or deleted
 709                  * from any local list before iput() since it could be deleted
 710                  * in another thread once we put the inode (then the local list
 711                  * is corrupted).
 712                  */
 713                 spin_lock(&sbinfo->shrinklist_lock);
 714                 list_move(&info->shrinklist, &sbinfo->shrinklist);
 715                 sbinfo->shrinklist_len++;
 716                 spin_unlock(&sbinfo->shrinklist_lock);
 717 put:
 718                 iput(inode);
 719         }
 720
 721         return split;
 722 }
 723
 724 static long shmem_unused_huge_scan(struct super_block *sb,
 725                 struct shrink_control *sc)
 726 {
 727         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 728
 729         if (!READ_ONCE(sbinfo->shrinklist_len))
 730                 return SHRINK_STOP;
 731
 732         return shmem_unused_huge_shrink(sbinfo, sc, 0);
 733 }
 734
 735 static long shmem_unused_huge_count(struct super_block *sb,
 736                 struct shrink_control *sc)
 737 {
 738         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 739         return READ_ONCE(sbinfo->shrinklist_len);
 740 }
 741 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
 742
 743 #define shmem_huge SHMEM_HUGE_DENY
 744
 745 bool shmem_is_huge(struct inode *inode, pgoff_t index, bool shmem_huge_force,
 746                    struct mm_struct *mm, unsigned long vm_flags)
 747 {
 748         return false;
 749 }
 750
 751 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 752                 struct shrink_control *sc, unsigned long nr_to_split)
 753 {
 754         return 0;
 755 }
 756 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 757
 758 /*
 759  * Like filemap_add_folio, but error if expected item has gone.
 760  */
 761 static int shmem_add_to_page_cache(struct folio *folio,
 762                                    struct address_space *mapping,
 763                                    pgoff_t index, void *expected, gfp_t gfp,
 764                                    struct mm_struct *charge_mm)
 765 {
 766         XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
 767         long nr = folio_nr_pages(folio);
 768         int error;
 769
 770         VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
 771         VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 772         VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
 773         VM_BUG_ON(expected && folio_test_large(folio));
 774
 775         folio_ref_add(folio, nr);
 776         folio->mapping = mapping;
 777         folio->index = index;
 778
 779         if (!folio_test_swapcache(folio)) {
 780                 error = mem_cgroup_charge(folio, charge_mm, gfp);
 781                 if (error) {
 782                         if (folio_test_pmd_mappable(folio)) {
 783                                 count_vm_event(THP_FILE_FALLBACK);
 784                                 count_vm_event(THP_FILE_FALLBACK_CHARGE);
 785                         }
 786                         goto error;
 787                 }
 788         }
 789         folio_throttle_swaprate(folio, gfp);
 790
 791         do {
 792                 xas_lock_irq(&xas);
 793                 if (expected != xas_find_conflict(&xas)) {
 794                         xas_set_err(&xas, -EEXIST);
 795                         goto unlock;
 796                 }
 797                 if (expected && xas_find_conflict(&xas)) {
 798                         xas_set_err(&xas, -EEXIST);
 799                         goto unlock;
 800                 }
 801                 xas_store(&xas, folio);
 802                 if (xas_error(&xas))
 803                         goto unlock;
 804                 if (folio_test_pmd_mappable(folio)) {
 805                         count_vm_event(THP_FILE_ALLOC);
 806                         __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
 807                 }
 808                 mapping->nrpages += nr;
 809                 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
 810                 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
 811 unlock:
 812                 xas_unlock_irq(&xas);
 813         } while (xas_nomem(&xas, gfp));
 814
 815         if (xas_error(&xas)) {
 816                 error = xas_error(&xas);
 817                 goto error;
 818         }
 819
 820         return 0;
 821 error:
 822         folio->mapping = NULL;
 823         folio_ref_sub(folio, nr);
 824         return error;
 825 }
 826
 827 /*
 828  * Like delete_from_page_cache, but substitutes swap for @folio.
 829  */
 830 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
 831 {
 832         struct address_space *mapping = folio->mapping;
 833         long nr = folio_nr_pages(folio);
 834         int error;
 835
 836         xa_lock_irq(&mapping->i_pages);
 837         error = shmem_replace_entry(mapping, folio->index, folio, radswap);
 838         folio->mapping = NULL;
 839         mapping->nrpages -= nr;
 840         __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
 841         __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
 842         xa_unlock_irq(&mapping->i_pages);
 843         folio_put(folio);
 844         BUG_ON(error);
 845 }
 846
 847 /*
 848  * Remove swap entry from page cache, free the swap and its page cache.
 849  */
 850 static int shmem_free_swap(struct address_space *mapping,
 851                            pgoff_t index, void *radswap)
 852 {
 853         void *old;
 854
 855         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
 856         if (old != radswap)
 857                 return -ENOENT;
 858         free_swap_and_cache(radix_to_swp_entry(radswap));
 859         return 0;
 860 }
 861
 862 /*
 863  * Determine (in bytes) how many of the shmem object's pages mapped by the
 864  * given offsets are swapped out.
 865  *
 866  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
 867  * as long as the inode doesn't go away and racy results are not a problem.
 868  */
 869 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 870                                                 pgoff_t start, pgoff_t end)
 871 {
 872         XA_STATE(xas, &mapping->i_pages, start);
 873         struct page *page;
 874         unsigned long swapped = 0;
 875         unsigned long max = end - 1;
 876
 877         rcu_read_lock();
 878         xas_for_each(&xas, page, max) {
 879                 if (xas_retry(&xas, page))
 880                         continue;
 881                 if (xa_is_value(page))
 882                         swapped++;
 883                 if (xas.xa_index == max)
 884                         break;
 885                 if (need_resched()) {
 886                         xas_pause(&xas);
 887                         cond_resched_rcu();
 888                 }
 889         }
 890
 891         rcu_read_unlock();
 892
 893         return swapped << PAGE_SHIFT;
 894 }
 895
 896 /*
 897  * Determine (in bytes) how many of the shmem object's pages mapped by the
 898  * given vma is swapped out.
 899  *
 900  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
 901  * as long as the inode doesn't go away and racy results are not a problem.
 902  */
 903 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
 904 {
 905         struct inode *inode = file_inode(vma->vm_file);
 906         struct shmem_inode_info *info = SHMEM_I(inode);
 907         struct address_space *mapping = inode->i_mapping;
 908         unsigned long swapped;
 909
 910         /* Be careful as we don't hold info->lock */
 911         swapped = READ_ONCE(info->swapped);
 912
 913         /*
 914          * The easier cases are when the shmem object has nothing in swap, or
 915          * the vma maps it whole. Then we can simply use the stats that we
 916          * already track.
 917          */
 918         if (!swapped)
 919                 return 0;
 920
 921         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
 922                 return swapped << PAGE_SHIFT;
 923
 924         /* Here comes the more involved part */
 925         return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
 926                                         vma->vm_pgoff + vma_pages(vma));
 927 }
 928
 929 /*
 930  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
 931  */
 932 void shmem_unlock_mapping(struct address_space *mapping)
 933 {
 934         struct folio_batch fbatch;
 935         pgoff_t index = 0;
 936
 937         folio_batch_init(&fbatch);
 938         /*
 939          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
 940          */
 941         while (!mapping_unevictable(mapping) &&
 942                filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
 943                 check_move_unevictable_folios(&fbatch);
 944                 folio_batch_release(&fbatch);
 945                 cond_resched();
 946         }
 947 }
 948
 949 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
 950 {
 951         struct folio *folio;
 952
 953         /*
 954          * At first avoid shmem_get_folio(,,,SGP_READ): that fails
 955          * beyond i_size, and reports fallocated folios as holes.
 956          */
 957         folio = filemap_get_entry(inode->i_mapping, index);
 958         if (!folio)
 959                 return folio;
 960         if (!xa_is_value(folio)) {
 961                 folio_lock(folio);
 962                 if (folio->mapping == inode->i_mapping)
 963                         return folio;
 964                 /* The folio has been swapped out */
 965                 folio_unlock(folio);
 966                 folio_put(folio);
 967         }
 968         /*
 969          * But read a folio back from swap if any of it is within i_size
 970          * (although in some cases this is just a waste of time).
 971          */
 972         folio = NULL;
 973         shmem_get_folio(inode, index, &folio, SGP_READ);
 974         return folio;
 975 }
 976
 977 /*
 978  * Remove range of pages and swap entries from page cache, and free them.
 979  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
 980  */
 981 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
 982                                                                  bool unfalloc)
 983 {
 984         struct address_space *mapping = inode->i_mapping;
 985         struct shmem_inode_info *info = SHMEM_I(inode);
 986         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
 987         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
 988         struct folio_batch fbatch;
 989         pgoff_t indices[PAGEVEC_SIZE];
 990         struct folio *folio;
 991         bool same_folio;
 992         long nr_swaps_freed = 0;
 993         pgoff_t index;
 994         int i;
 995
 996         if (lend == -1)
 997                 end = -1;       /* unsigned, so actually very big */
 998
 999         if (info->fallocend > start && info->fallocend <= end && !unfalloc)
1000                 info->fallocend = start;
1001
1002         folio_batch_init(&fbatch);
1003         index = start;
1004         while (index < end && find_lock_entries(mapping, &index, end - 1,
1005                         &fbatch, indices)) {
1006                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
1007                         folio = fbatch.folios[i];
1008
1009                         if (xa_is_value(folio)) {
1010                                 if (unfalloc)
1011                                         continue;
1012                                 nr_swaps_freed += !shmem_free_swap(mapping,
1013                                                         indices[i], folio);
1014                                 continue;
1015                         }
1016
1017                         if (!unfalloc || !folio_test_uptodate(folio))
1018                                 truncate_inode_folio(mapping, folio);
1019                         folio_unlock(folio);
1020                 }
1021                 folio_batch_remove_exceptionals(&fbatch);
1022                 folio_batch_release(&fbatch);
1023                 cond_resched();
1024         }
1025
1026         /*
1027          * When undoing a failed fallocate, we want none of the partial folio
1028          * zeroing and splitting below, but shall want to truncate the whole
1029          * folio when !uptodate indicates that it was added by this fallocate,
1030          * even when [lstart, lend] covers only a part of the folio.
1031          */
1032         if (unfalloc)
1033                 goto whole_folios;
1034
1035         same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
1036         folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
1037         if (folio) {
1038                 same_folio = lend < folio_pos(folio) + folio_size(folio);
1039                 folio_mark_dirty(folio);
1040                 if (!truncate_inode_partial_folio(folio, lstart, lend)) {
1041                         start = folio_next_index(folio);
1042                         if (same_folio)
1043                                 end = folio->index;
1044                 }
1045                 folio_unlock(folio);
1046                 folio_put(folio);
1047                 folio = NULL;
1048         }
1049
1050         if (!same_folio)
1051                 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
1052         if (folio) {
1053                 folio_mark_dirty(folio);
1054                 if (!truncate_inode_partial_folio(folio, lstart, lend))
1055                         end = folio->index;
1056                 folio_unlock(folio);
1057                 folio_put(folio);
1058         }
1059
1060 whole_folios:
1061
1062         index = start;
1063         while (index < end) {
1064                 cond_resched();
1065
1066                 if (!find_get_entries(mapping, &index, end - 1, &fbatch,
1067                                 indices)) {
1068                         /* If all gone or hole-punch or unfalloc, we're done */
1069                         if (index == start || end != -1)
1070                                 break;
1071                         /* But if truncating, restart to make sure all gone */
1072                         index = start;
1073                         continue;
1074                 }
1075                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
1076                         folio = fbatch.folios[i];
1077
1078                         if (xa_is_value(folio)) {
1079                                 if (unfalloc)
1080                                         continue;
1081                                 if (shmem_free_swap(mapping, indices[i], folio)) {
1082                                         /* Swap was replaced by page: retry */
1083                                         index = indices[i];
1084                                         break;
1085                                 }
1086                                 nr_swaps_freed++;
1087                                 continue;
1088                         }
1089
1090                         folio_lock(folio);
1091
1092                         if (!unfalloc || !folio_test_uptodate(folio)) {
1093                                 if (folio_mapping(folio) != mapping) {
1094                                         /* Page was replaced by swap: retry */
1095                                         folio_unlock(folio);
1096                                         index = indices[i];
1097                                         break;
1098                                 }
1099                                 VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1100                                                 folio);
1101
1102                                 if (!folio_test_large(folio)) {
1103                                         truncate_inode_folio(mapping, folio);
1104                                 } else if (truncate_inode_partial_folio(folio, lstart, lend)) {
1105                                         /*
1106                                          * If we split a page, reset the loop so
1107                                          * that we pick up the new sub pages.
1108                                          * Otherwise the THP was entirely
1109                                          * dropped or the target range was
1110                                          * zeroed, so just continue the loop as
1111                                          * is.
1112                                          */
1113                                         if (!folio_test_large(folio)) {
1114                                                 folio_unlock(folio);
1115                                                 index = start;
1116                                                 break;
1117                                         }
1118                                 }
1119                         }
1120                         folio_unlock(folio);
1121                 }
1122                 folio_batch_remove_exceptionals(&fbatch);
1123                 folio_batch_release(&fbatch);
1124         }
1125
1126         shmem_recalc_inode(inode, 0, -nr_swaps_freed);
1127 }
1128
1129 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1130 {
1131         shmem_undo_range(inode, lstart, lend, false);
1132         inode->i_mtime = inode_set_ctime_current(inode);
1133         inode_inc_iversion(inode);
1134 }
1135 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1136
1137 static int shmem_getattr(struct mnt_idmap *idmap,
1138                          const struct path *path, struct kstat *stat,
1139                          u32 request_mask, unsigned int query_flags)
1140 {
1141         struct inode *inode = path->dentry->d_inode;
1142         struct shmem_inode_info *info = SHMEM_I(inode);
1143
1144         if (info->alloced - info->swapped != inode->i_mapping->nrpages)
1145                 shmem_recalc_inode(inode, 0, 0);
1146
1147         if (info->fsflags & FS_APPEND_FL)
1148                 stat->attributes |= STATX_ATTR_APPEND;
1149         if (info->fsflags & FS_IMMUTABLE_FL)
1150                 stat->attributes |= STATX_ATTR_IMMUTABLE;
1151         if (info->fsflags & FS_NODUMP_FL)
1152                 stat->attributes |= STATX_ATTR_NODUMP;
1153         stat->attributes_mask |= (STATX_ATTR_APPEND |
1154                         STATX_ATTR_IMMUTABLE |
1155                         STATX_ATTR_NODUMP);
1156         generic_fillattr(idmap, request_mask, inode, stat);
1157
1158         if (shmem_is_huge(inode, 0, false, NULL, 0))
1159                 stat->blksize = HPAGE_PMD_SIZE;
1160
1161         if (request_mask & STATX_BTIME) {
1162                 stat->result_mask |= STATX_BTIME;
1163                 stat->btime.tv_sec = info->i_crtime.tv_sec;
1164                 stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1165         }
1166
1167         return 0;
1168 }
1169
1170 static int shmem_setattr(struct mnt_idmap *idmap,
1171                          struct dentry *dentry, struct iattr *attr)
1172 {
1173         struct inode *inode = d_inode(dentry);
1174         struct shmem_inode_info *info = SHMEM_I(inode);
1175         int error;
1176         bool update_mtime = false;
1177         bool update_ctime = true;
1178
1179         error = setattr_prepare(idmap, dentry, attr);
1180         if (error)
1181                 return error;
1182
1183         if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) {
1184                 if ((inode->i_mode ^ attr->ia_mode) & 0111) {
1185                         return -EPERM;
1186                 }
1187         }
1188
1189         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1190                 loff_t oldsize = inode->i_size;
1191                 loff_t newsize = attr->ia_size;
1192
1193                 /* protected by i_rwsem */
1194                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1195                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1196                         return -EPERM;
1197
1198                 if (newsize != oldsize) {
1199                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1200                                         oldsize, newsize);
1201                         if (error)
1202                                 return error;
1203                         i_size_write(inode, newsize);
1204                         update_mtime = true;
1205                 } else {
1206                         update_ctime = false;
1207                 }
1208                 if (newsize <= oldsize) {
1209                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1210                         if (oldsize > holebegin)
1211                                 unmap_mapping_range(inode->i_mapping,
1212                                                         holebegin, 0, 1);
1213                         if (info->alloced)
1214                                 shmem_truncate_range(inode,
1215                                                         newsize, (loff_t)-1);
1216                         /* unmap again to remove racily COWed private pages */
1217                         if (oldsize > holebegin)
1218                                 unmap_mapping_range(inode->i_mapping,
1219                                                         holebegin, 0, 1);
1220                 }
1221         }
1222
1223         if (is_quota_modification(idmap, inode, attr)) {
1224                 error = dquot_initialize(inode);
1225                 if (error)
1226                         return error;
1227         }
1228
1229         /* Transfer quota accounting */
1230         if (i_uid_needs_update(idmap, attr, inode) ||
1231             i_gid_needs_update(idmap, attr, inode)) {
1232                 error = dquot_transfer(idmap, inode, attr);
1233
1234                 if (error)
1235                         return error;
1236         }
1237
1238         setattr_copy(idmap, inode, attr);
1239         if (attr->ia_valid & ATTR_MODE)
1240                 error = posix_acl_chmod(idmap, dentry, inode->i_mode);
1241         if (!error && update_ctime) {
1242                 inode_set_ctime_current(inode);
1243                 if (update_mtime)
1244                         inode->i_mtime = inode_get_ctime(inode);
1245                 inode_inc_iversion(inode);
1246         }
1247         return error;
1248 }
1249
1250 static void shmem_evict_inode(struct inode *inode)
1251 {
1252         struct shmem_inode_info *info = SHMEM_I(inode);
1253         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1254         size_t freed = 0;
1255
1256         if (shmem_mapping(inode->i_mapping)) {
1257                 shmem_unacct_size(info->flags, inode->i_size);
1258                 inode->i_size = 0;
1259                 mapping_set_exiting(inode->i_mapping);
1260                 shmem_truncate_range(inode, 0, (loff_t)-1);
1261                 if (!list_empty(&info->shrinklist)) {
1262                         spin_lock(&sbinfo->shrinklist_lock);
1263                         if (!list_empty(&info->shrinklist)) {
1264                                 list_del_init(&info->shrinklist);
1265                                 sbinfo->shrinklist_len--;
1266                         }
1267                         spin_unlock(&sbinfo->shrinklist_lock);
1268                 }
1269                 while (!list_empty(&info->swaplist)) {
1270                         /* Wait while shmem_unuse() is scanning this inode... */
1271                         wait_var_event(&info->stop_eviction,
1272                                        !atomic_read(&info->stop_eviction));
1273                         mutex_lock(&shmem_swaplist_mutex);
1274                         /* ...but beware of the race if we peeked too early */
1275                         if (!atomic_read(&info->stop_eviction))
1276                                 list_del_init(&info->swaplist);
1277                         mutex_unlock(&shmem_swaplist_mutex);
1278                 }
1279         }
1280
1281         simple_xattrs_free(&info->xattrs, sbinfo->max_inodes ? &freed : NULL);
1282         shmem_free_inode(inode->i_sb, freed);
1283         WARN_ON(inode->i_blocks);
1284         clear_inode(inode);
1285 #ifdef CONFIG_TMPFS_QUOTA
1286         dquot_free_inode(inode);
1287         dquot_drop(inode);
1288 #endif
1289 }
1290
1291 static int shmem_find_swap_entries(struct address_space *mapping,
1292                                    pgoff_t start, struct folio_batch *fbatch,
1293                                    pgoff_t *indices, unsigned int type)
1294 {
1295         XA_STATE(xas, &mapping->i_pages, start);
1296         struct folio *folio;
1297         swp_entry_t entry;
1298
1299         rcu_read_lock();
1300         xas_for_each(&xas, folio, ULONG_MAX) {
1301                 if (xas_retry(&xas, folio))
1302                         continue;
1303
1304                 if (!xa_is_value(folio))
1305                         continue;
1306
1307                 entry = radix_to_swp_entry(folio);
1308                 /*
1309                  * swapin error entries can be found in the mapping. But they're
1310                  * deliberately ignored here as we've done everything we can do.
1311                  */
1312                 if (swp_type(entry) != type)
1313                         continue;
1314
1315                 indices[folio_batch_count(fbatch)] = xas.xa_index;
1316                 if (!folio_batch_add(fbatch, folio))
1317                         break;
1318
1319                 if (need_resched()) {
1320                         xas_pause(&xas);
1321                         cond_resched_rcu();
1322                 }
1323         }
1324         rcu_read_unlock();
1325
1326         return xas.xa_index;
1327 }
1328
1329 /*
1330  * Move the swapped pages for an inode to page cache. Returns the count
1331  * of pages swapped in, or the error in case of failure.
1332  */
1333 static int shmem_unuse_swap_entries(struct inode *inode,
1334                 struct folio_batch *fbatch, pgoff_t *indices)
1335 {
1336         int i = 0;
1337         int ret = 0;
1338         int error = 0;
1339         struct address_space *mapping = inode->i_mapping;
1340
1341         for (i = 0; i < folio_batch_count(fbatch); i++) {
1342                 struct folio *folio = fbatch->folios[i];
1343
1344                 if (!xa_is_value(folio))
1345                         continue;
1346                 error = shmem_swapin_folio(inode, indices[i],
1347                                           &folio, SGP_CACHE,
1348                                           mapping_gfp_mask(mapping),
1349                                           NULL, NULL);
1350                 if (error == 0) {
1351                         folio_unlock(folio);
1352                         folio_put(folio);
1353                         ret++;
1354                 }
1355                 if (error == -ENOMEM)
1356                         break;
1357                 error = 0;
1358         }
1359         return error ? error : ret;
1360 }
1361
1362 /*
1363  * If swap found in inode, free it and move page from swapcache to filecache.
1364  */
1365 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1366 {
1367         struct address_space *mapping = inode->i_mapping;
1368         pgoff_t start = 0;
1369         struct folio_batch fbatch;
1370         pgoff_t indices[PAGEVEC_SIZE];
1371         int ret = 0;
1372
1373         do {
1374                 folio_batch_init(&fbatch);
1375                 shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1376                 if (folio_batch_count(&fbatch) == 0) {
1377                         ret = 0;
1378                         break;
1379                 }
1380
1381                 ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1382                 if (ret < 0)
1383                         break;
1384
1385                 start = indices[folio_batch_count(&fbatch) - 1];
1386         } while (true);
1387
1388         return ret;
1389 }
1390
1391 /*
1392  * Read all the shared memory data that resides in the swap
1393  * device 'type' back into memory, so the swap device can be
1394  * unused.
1395  */
1396 int shmem_unuse(unsigned int type)
1397 {
1398         struct shmem_inode_info *info, *next;
1399         int error = 0;
1400
1401         if (list_empty(&shmem_swaplist))
1402                 return 0;
1403
1404         mutex_lock(&shmem_swaplist_mutex);
1405         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1406                 if (!info->swapped) {
1407                         list_del_init(&info->swaplist);
1408                         continue;
1409                 }
1410                 /*
1411                  * Drop the swaplist mutex while searching the inode for swap;
1412                  * but before doing so, make sure shmem_evict_inode() will not
1413                  * remove placeholder inode from swaplist, nor let it be freed
1414                  * (igrab() would protect from unlink, but not from unmount).
1415                  */
1416                 atomic_inc(&info->stop_eviction);
1417                 mutex_unlock(&shmem_swaplist_mutex);
1418
1419                 error = shmem_unuse_inode(&info->vfs_inode, type);
1420                 cond_resched();
1421
1422                 mutex_lock(&shmem_swaplist_mutex);
1423                 next = list_next_entry(info, swaplist);
1424                 if (!info->swapped)
1425                         list_del_init(&info->swaplist);
1426                 if (atomic_dec_and_test(&info->stop_eviction))
1427                         wake_up_var(&info->stop_eviction);
1428                 if (error)
1429                         break;
1430         }
1431         mutex_unlock(&shmem_swaplist_mutex);
1432
1433         return error;
1434 }
1435
1436 /*
1437  * Move the page from the page cache to the swap cache.
1438  */
1439 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1440 {
1441         struct folio *folio = page_folio(page);
1442         struct address_space *mapping = folio->mapping;
1443         struct inode *inode = mapping->host;
1444         struct shmem_inode_info *info = SHMEM_I(inode);
1445         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1446         swp_entry_t swap;
1447         pgoff_t index;
1448
1449         /*
1450          * Our capabilities prevent regular writeback or sync from ever calling
1451          * shmem_writepage; but a stacking filesystem might use ->writepage of
1452          * its underlying filesystem, in which case tmpfs should write out to
1453          * swap only in response to memory pressure, and not for the writeback
1454          * threads or sync.
1455          */
1456         if (WARN_ON_ONCE(!wbc->for_reclaim))
1457                 goto redirty;
1458
1459         if (WARN_ON_ONCE((info->flags & VM_LOCKED) || sbinfo->noswap))
1460                 goto redirty;
1461
1462         if (!total_swap_pages)
1463                 goto redirty;
1464
1465         /*
1466          * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1467          * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1468          * and its shmem_writeback() needs them to be split when swapping.
1469          */
1470         if (folio_test_large(folio)) {
1471                 /* Ensure the subpages are still dirty */
1472                 folio_test_set_dirty(folio);
1473                 if (split_huge_page(page) < 0)
1474                         goto redirty;
1475                 folio = page_folio(page);
1476                 folio_clear_dirty(folio);
1477         }
1478
1479         index = folio->index;
1480
1481         /*
1482          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1483          * value into swapfile.c, the only way we can correctly account for a
1484          * fallocated folio arriving here is now to initialize it and write it.
1485          *
1486          * That's okay for a folio already fallocated earlier, but if we have
1487          * not yet completed the fallocation, then (a) we want to keep track
1488          * of this folio in case we have to undo it, and (b) it may not be a
1489          * good idea to continue anyway, once we're pushing into swap.  So
1490          * reactivate the folio, and let shmem_fallocate() quit when too many.
1491          */
1492         if (!folio_test_uptodate(folio)) {
1493                 if (inode->i_private) {
1494                         struct shmem_falloc *shmem_falloc;
1495                         spin_lock(&inode->i_lock);
1496                         shmem_falloc = inode->i_private;
1497                         if (shmem_falloc &&
1498                             !shmem_falloc->waitq &&
1499                             index >= shmem_falloc->start &&
1500                             index < shmem_falloc->next)
1501                                 shmem_falloc->nr_unswapped++;
1502                         else
1503                                 shmem_falloc = NULL;
1504                         spin_unlock(&inode->i_lock);
1505                         if (shmem_falloc)
1506                                 goto redirty;
1507                 }
1508                 folio_zero_range(folio, 0, folio_size(folio));
1509                 flush_dcache_folio(folio);
1510                 folio_mark_uptodate(folio);
1511         }
1512
1513         swap = folio_alloc_swap(folio);
1514         if (!swap.val)
1515                 goto redirty;
1516
1517         /*
1518          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1519          * if it's not already there.  Do it now before the folio is
1520          * moved to swap cache, when its pagelock no longer protects
1521          * the inode from eviction.  But don't unlock the mutex until
1522          * we've incremented swapped, because shmem_unuse_inode() will
1523          * prune a !swapped inode from the swaplist under this mutex.
1524          */
1525         mutex_lock(&shmem_swaplist_mutex);
1526         if (list_empty(&info->swaplist))
1527                 list_add(&info->swaplist, &shmem_swaplist);
1528
1529         if (add_to_swap_cache(folio, swap,
1530                         __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1531                         NULL) == 0) {
1532                 shmem_recalc_inode(inode, 0, 1);
1533                 swap_shmem_alloc(swap);
1534                 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1535
1536                 mutex_unlock(&shmem_swaplist_mutex);
1537                 BUG_ON(folio_mapped(folio));
1538                 swap_writepage(&folio->page, wbc);
1539                 return 0;
1540         }
1541
1542         mutex_unlock(&shmem_swaplist_mutex);
1543         put_swap_folio(folio, swap);
1544 redirty:
1545         folio_mark_dirty(folio);
1546         if (wbc->for_reclaim)
1547                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with folio locked */
1548         folio_unlock(folio);
1549         return 0;
1550 }
1551
1552 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1553 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1554 {
1555         char buffer[64];
1556
1557         if (!mpol || mpol->mode == MPOL_DEFAULT)
1558                 return;         /* show nothing */
1559
1560         mpol_to_str(buffer, sizeof(buffer), mpol);
1561
1562         seq_printf(seq, ",mpol=%s", buffer);
1563 }
1564
1565 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1566 {
1567         struct mempolicy *mpol = NULL;
1568         if (sbinfo->mpol) {
1569                 raw_spin_lock(&sbinfo->stat_lock);      /* prevent replace/use races */
1570                 mpol = sbinfo->mpol;
1571                 mpol_get(mpol);
1572                 raw_spin_unlock(&sbinfo->stat_lock);
1573         }
1574         return mpol;
1575 }
1576 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1577 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1578 {
1579 }
1580 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1581 {
1582         return NULL;
1583 }
1584 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1585 #ifndef CONFIG_NUMA
1586 #define vm_policy vm_private_data
1587 #endif
1588
1589 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1590                 struct shmem_inode_info *info, pgoff_t index)
1591 {
1592         /* Create a pseudo vma that just contains the policy */
1593         vma_init(vma, NULL);
1594         /* Bias interleave by inode number to distribute better across nodes */
1595         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1596         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1597 }
1598
1599 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1600 {
1601         /* Drop reference taken by mpol_shared_policy_lookup() */
1602         mpol_cond_put(vma->vm_policy);
1603 }
1604
1605 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1606                         struct shmem_inode_info *info, pgoff_t index)
1607 {
1608         struct vm_area_struct pvma;
1609         struct page *page;
1610         struct vm_fault vmf = {
1611                 .vma = &pvma,
1612         };
1613
1614         shmem_pseudo_vma_init(&pvma, info, index);
1615         page = swap_cluster_readahead(swap, gfp, &vmf);
1616         shmem_pseudo_vma_destroy(&pvma);
1617
1618         if (!page)
1619                 return NULL;
1620         return page_folio(page);
1621 }
1622
1623 /*
1624  * Make sure huge_gfp is always more limited than limit_gfp.
1625  * Some of the flags set permissions, while others set limitations.
1626  */
1627 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1628 {
1629         gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1630         gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1631         gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1632         gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1633
1634         /* Allow allocations only from the originally specified zones. */
1635         result |= zoneflags;
1636
1637         /*
1638          * Minimize the result gfp by taking the union with the deny flags,
1639          * and the intersection of the allow flags.
1640          */
1641         result |= (limit_gfp & denyflags);
1642         result |= (huge_gfp & limit_gfp) & allowflags;
1643
1644         return result;
1645 }
1646
1647 static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1648                 struct shmem_inode_info *info, pgoff_t index)
1649 {
1650         struct vm_area_struct pvma;
1651         struct address_space *mapping = info->vfs_inode.i_mapping;
1652         pgoff_t hindex;
1653         struct folio *folio;
1654
1655         hindex = round_down(index, HPAGE_PMD_NR);
1656         if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1657                                                                 XA_PRESENT))
1658                 return NULL;
1659
1660         shmem_pseudo_vma_init(&pvma, info, hindex);
1661         folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1662         shmem_pseudo_vma_destroy(&pvma);
1663         if (!folio)
1664                 count_vm_event(THP_FILE_FALLBACK);
1665         return folio;
1666 }
1667
1668 static struct folio *shmem_alloc_folio(gfp_t gfp,
1669                         struct shmem_inode_info *info, pgoff_t index)
1670 {
1671         struct vm_area_struct pvma;
1672         struct folio *folio;
1673
1674         shmem_pseudo_vma_init(&pvma, info, index);
1675         folio = vma_alloc_folio(gfp, 0, &pvma, 0, false);
1676         shmem_pseudo_vma_destroy(&pvma);
1677
1678         return folio;
1679 }
1680
1681 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
1682                 pgoff_t index, bool huge)
1683 {
1684         struct shmem_inode_info *info = SHMEM_I(inode);
1685         struct folio *folio;
1686         int nr;
1687         int err;
1688
1689         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1690                 huge = false;
1691         nr = huge ? HPAGE_PMD_NR : 1;
1692
1693         err = shmem_inode_acct_block(inode, nr);
1694         if (err)
1695                 goto failed;
1696
1697         if (huge)
1698                 folio = shmem_alloc_hugefolio(gfp, info, index);
1699         else
1700                 folio = shmem_alloc_folio(gfp, info, index);
1701         if (folio) {
1702                 __folio_set_locked(folio);
1703                 __folio_set_swapbacked(folio);
1704                 return folio;
1705         }
1706
1707         err = -ENOMEM;
1708         shmem_inode_unacct_blocks(inode, nr);
1709 failed:
1710         return ERR_PTR(err);
1711 }
1712
1713 /*
1714  * When a page is moved from swapcache to shmem filecache (either by the
1715  * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1716  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1717  * ignorance of the mapping it belongs to.  If that mapping has special
1718  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1719  * we may need to copy to a suitable page before moving to filecache.
1720  *
1721  * In a future release, this may well be extended to respect cpuset and
1722  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1723  * but for now it is a simple matter of zone.
1724  */
1725 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1726 {
1727         return folio_zonenum(folio) > gfp_zone(gfp);
1728 }
1729
1730 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1731                                 struct shmem_inode_info *info, pgoff_t index)
1732 {
1733         struct folio *old, *new;
1734         struct address_space *swap_mapping;
1735         swp_entry_t entry;
1736         pgoff_t swap_index;
1737         int error;
1738
1739         old = *foliop;
1740         entry = old->swap;
1741         swap_index = swp_offset(entry);
1742         swap_mapping = swap_address_space(entry);
1743
1744         /*
1745          * We have arrived here because our zones are constrained, so don't
1746          * limit chance of success by further cpuset and node constraints.
1747          */
1748         gfp &= ~GFP_CONSTRAINT_MASK;
1749         VM_BUG_ON_FOLIO(folio_test_large(old), old);
1750         new = shmem_alloc_folio(gfp, info, index);
1751         if (!new)
1752                 return -ENOMEM;
1753
1754         folio_get(new);
1755         folio_copy(new, old);
1756         flush_dcache_folio(new);
1757
1758         __folio_set_locked(new);
1759         __folio_set_swapbacked(new);
1760         folio_mark_uptodate(new);
1761         new->swap = entry;
1762         folio_set_swapcache(new);
1763
1764         /*
1765          * Our caller will very soon move newpage out of swapcache, but it's
1766          * a nice clean interface for us to replace oldpage by newpage there.
1767          */
1768         xa_lock_irq(&swap_mapping->i_pages);
1769         error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1770         if (!error) {
1771                 mem_cgroup_migrate(old, new);
1772                 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1773                 __lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1774                 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1775                 __lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1776         }
1777         xa_unlock_irq(&swap_mapping->i_pages);
1778
1779         if (unlikely(error)) {
1780                 /*
1781                  * Is this possible?  I think not, now that our callers check
1782                  * both PageSwapCache and page_private after getting page lock;
1783                  * but be defensive.  Reverse old to newpage for clear and free.
1784                  */
1785                 old = new;
1786         } else {
1787                 folio_add_lru(new);
1788                 *foliop = new;
1789         }
1790
1791         folio_clear_swapcache(old);
1792         old->private = NULL;
1793
1794         folio_unlock(old);
1795         folio_put_refs(old, 2);
1796         return error;
1797 }
1798
1799 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1800                                          struct folio *folio, swp_entry_t swap)
1801 {
1802         struct address_space *mapping = inode->i_mapping;
1803         swp_entry_t swapin_error;
1804         void *old;
1805
1806         swapin_error = make_poisoned_swp_entry();
1807         old = xa_cmpxchg_irq(&mapping->i_pages, index,
1808                              swp_to_radix_entry(swap),
1809                              swp_to_radix_entry(swapin_error), 0);
1810         if (old != swp_to_radix_entry(swap))
1811                 return;
1812
1813         folio_wait_writeback(folio);
1814         delete_from_swap_cache(folio);
1815         /*
1816          * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks
1817          * won't be 0 when inode is released and thus trigger WARN_ON(i_blocks)
1818          * in shmem_evict_inode().
1819          */
1820         shmem_recalc_inode(inode, -1, -1);
1821         swap_free(swap);
1822 }
1823
1824 /*
1825  * Swap in the folio pointed to by *foliop.
1826  * Caller has to make sure that *foliop contains a valid swapped folio.
1827  * Returns 0 and the folio in foliop if success. On failure, returns the
1828  * error code and NULL in *foliop.
1829  */
1830 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1831                              struct folio **foliop, enum sgp_type sgp,
1832                              gfp_t gfp, struct vm_area_struct *vma,
1833                              vm_fault_t *fault_type)
1834 {
1835         struct address_space *mapping = inode->i_mapping;
1836         struct shmem_inode_info *info = SHMEM_I(inode);
1837         struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1838         struct swap_info_struct *si;
1839         struct folio *folio = NULL;
1840         swp_entry_t swap;
1841         int error;
1842
1843         VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1844         swap = radix_to_swp_entry(*foliop);
1845         *foliop = NULL;
1846
1847         if (is_poisoned_swp_entry(swap))
1848                 return -EIO;
1849
1850         si = get_swap_device(swap);
1851         if (!si) {
1852                 if (!shmem_confirm_swap(mapping, index, swap))
1853                         return -EEXIST;
1854                 else
1855                         return -EINVAL;
1856         }
1857
1858         /* Look it up and read it in.. */
1859         folio = swap_cache_get_folio(swap, NULL, 0);
1860         if (!folio) {
1861                 /* Or update major stats only when swapin succeeds?? */
1862                 if (fault_type) {
1863                         *fault_type |= VM_FAULT_MAJOR;
1864                         count_vm_event(PGMAJFAULT);
1865                         count_memcg_event_mm(charge_mm, PGMAJFAULT);
1866                 }
1867                 /* Here we actually start the io */
1868                 folio = shmem_swapin(swap, gfp, info, index);
1869                 if (!folio) {
1870                         error = -ENOMEM;
1871                         goto failed;
1872                 }
1873         }
1874
1875         /* We have to do this with folio locked to prevent races */
1876         folio_lock(folio);
1877         if (!folio_test_swapcache(folio) ||
1878             folio->swap.val != swap.val ||
1879             !shmem_confirm_swap(mapping, index, swap)) {
1880                 error = -EEXIST;
1881                 goto unlock;
1882         }
1883         if (!folio_test_uptodate(folio)) {
1884                 error = -EIO;
1885                 goto failed;
1886         }
1887         folio_wait_writeback(folio);
1888
1889         /*
1890          * Some architectures may have to restore extra metadata to the
1891          * folio after reading from swap.
1892          */
1893         arch_swap_restore(swap, folio);
1894
1895         if (shmem_should_replace_folio(folio, gfp)) {
1896                 error = shmem_replace_folio(&folio, gfp, info, index);
1897                 if (error)
1898                         goto failed;
1899         }
1900
1901         error = shmem_add_to_page_cache(folio, mapping, index,
1902                                         swp_to_radix_entry(swap), gfp,
1903                                         charge_mm);
1904         if (error)
1905                 goto failed;
1906
1907         shmem_recalc_inode(inode, 0, -1);
1908
1909         if (sgp == SGP_WRITE)
1910                 folio_mark_accessed(folio);
1911
1912         delete_from_swap_cache(folio);
1913         folio_mark_dirty(folio);
1914         swap_free(swap);
1915         put_swap_device(si);
1916
1917         *foliop = folio;
1918         return 0;
1919 failed:
1920         if (!shmem_confirm_swap(mapping, index, swap))
1921                 error = -EEXIST;
1922         if (error == -EIO)
1923                 shmem_set_folio_swapin_error(inode, index, folio, swap);
1924 unlock:
1925         if (folio) {
1926                 folio_unlock(folio);
1927                 folio_put(folio);
1928         }
1929         put_swap_device(si);
1930
1931         return error;
1932 }
1933
1934 /*
1935  * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
1936  *
1937  * If we allocate a new one we do not mark it dirty. That's up to the
1938  * vm. If we swap it in we mark it dirty since we also free the swap
1939  * entry since a page cannot live in both the swap and page cache.
1940  *
1941  * vma, vmf, and fault_type are only supplied by shmem_fault:
1942  * otherwise they are NULL.
1943  */
1944 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
1945                 struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
1946                 struct vm_area_struct *vma, struct vm_fault *vmf,
1947                 vm_fault_t *fault_type)
1948 {
1949         struct address_space *mapping = inode->i_mapping;
1950         struct shmem_inode_info *info = SHMEM_I(inode);
1951         struct shmem_sb_info *sbinfo;
1952         struct mm_struct *charge_mm;
1953         struct folio *folio;
1954         pgoff_t hindex;
1955         gfp_t huge_gfp;
1956         int error;
1957         int once = 0;
1958         int alloced = 0;
1959
1960         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1961                 return -EFBIG;
1962 repeat:
1963         if (sgp <= SGP_CACHE &&
1964             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1965                 return -EINVAL;
1966         }
1967
1968         sbinfo = SHMEM_SB(inode->i_sb);
1969         charge_mm = vma ? vma->vm_mm : NULL;
1970
1971         folio = filemap_get_entry(mapping, index);
1972         if (folio && vma && userfaultfd_minor(vma)) {
1973                 if (!xa_is_value(folio))
1974                         folio_put(folio);
1975                 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1976                 return 0;
1977         }
1978
1979         if (xa_is_value(folio)) {
1980                 error = shmem_swapin_folio(inode, index, &folio,
1981                                           sgp, gfp, vma, fault_type);
1982                 if (error == -EEXIST)
1983                         goto repeat;
1984
1985                 *foliop = folio;
1986                 return error;
1987         }
1988
1989         if (folio) {
1990                 folio_lock(folio);
1991
1992                 /* Has the folio been truncated or swapped out? */
1993                 if (unlikely(folio->mapping != mapping)) {
1994                         folio_unlock(folio);
1995                         folio_put(folio);
1996                         goto repeat;
1997                 }
1998                 if (sgp == SGP_WRITE)
1999                         folio_mark_accessed(folio);
2000                 if (folio_test_uptodate(folio))
2001                         goto out;
2002                 /* fallocated folio */
2003                 if (sgp != SGP_READ)
2004                         goto clear;
2005                 folio_unlock(folio);
2006                 folio_put(folio);
2007         }
2008
2009         /*
2010          * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
2011          * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
2012          */
2013         *foliop = NULL;
2014         if (sgp == SGP_READ)
2015                 return 0;
2016         if (sgp == SGP_NOALLOC)
2017                 return -ENOENT;
2018
2019         /*
2020          * Fast cache lookup and swap lookup did not find it: allocate.
2021          */
2022
2023         if (vma && userfaultfd_missing(vma)) {
2024                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
2025                 return 0;
2026         }
2027
2028         if (!shmem_is_huge(inode, index, false,
2029                            vma ? vma->vm_mm : NULL, vma ? vma->vm_flags : 0))
2030                 goto alloc_nohuge;
2031
2032         huge_gfp = vma_thp_gfp_mask(vma);
2033         huge_gfp = limit_gfp_mask(huge_gfp, gfp);
2034         folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true);
2035         if (IS_ERR(folio)) {
2036 alloc_nohuge:
2037                 folio = shmem_alloc_and_acct_folio(gfp, inode, index, false);
2038         }
2039         if (IS_ERR(folio)) {
2040                 int retry = 5;
2041
2042                 error = PTR_ERR(folio);
2043                 folio = NULL;
2044                 if (error != -ENOSPC)
2045                         goto unlock;
2046                 /*
2047                  * Try to reclaim some space by splitting a large folio
2048                  * beyond i_size on the filesystem.
2049                  */
2050                 while (retry--) {
2051                         int ret;
2052
2053                         ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
2054                         if (ret == SHRINK_STOP)
2055                                 break;
2056                         if (ret)
2057                                 goto alloc_nohuge;
2058                 }
2059                 goto unlock;
2060         }
2061
2062         hindex = round_down(index, folio_nr_pages(folio));
2063
2064         if (sgp == SGP_WRITE)
2065                 __folio_set_referenced(folio);
2066
2067         error = shmem_add_to_page_cache(folio, mapping, hindex,
2068                                         NULL, gfp & GFP_RECLAIM_MASK,
2069                                         charge_mm);
2070         if (error)
2071                 goto unacct;
2072
2073         folio_add_lru(folio);
2074         shmem_recalc_inode(inode, folio_nr_pages(folio), 0);
2075         alloced = true;
2076
2077         if (folio_test_pmd_mappable(folio) &&
2078             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
2079                                         folio_next_index(folio) - 1) {
2080                 /*
2081                  * Part of the large folio is beyond i_size: subject
2082                  * to shrink under memory pressure.
2083                  */
2084                 spin_lock(&sbinfo->shrinklist_lock);
2085                 /*
2086                  * _careful to defend against unlocked access to
2087                  * ->shrink_list in shmem_unused_huge_shrink()
2088                  */
2089                 if (list_empty_careful(&info->shrinklist)) {
2090                         list_add_tail(&info->shrinklist,
2091                                       &sbinfo->shrinklist);
2092                         sbinfo->shrinklist_len++;
2093                 }
2094                 spin_unlock(&sbinfo->shrinklist_lock);
2095         }
2096
2097         /*
2098          * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
2099          */
2100         if (sgp == SGP_FALLOC)
2101                 sgp = SGP_WRITE;
2102 clear:
2103         /*
2104          * Let SGP_WRITE caller clear ends if write does not fill folio;
2105          * but SGP_FALLOC on a folio fallocated earlier must initialize
2106          * it now, lest undo on failure cancel our earlier guarantee.
2107          */
2108         if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
2109                 long i, n = folio_nr_pages(folio);
2110
2111                 for (i = 0; i < n; i++)
2112                         clear_highpage(folio_page(folio, i));
2113                 flush_dcache_folio(folio);
2114                 folio_mark_uptodate(folio);
2115         }
2116
2117         /* Perhaps the file has been truncated since we checked */
2118         if (sgp <= SGP_CACHE &&
2119             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2120                 if (alloced) {
2121                         folio_clear_dirty(folio);
2122                         filemap_remove_folio(folio);
2123                         shmem_recalc_inode(inode, 0, 0);
2124                 }
2125                 error = -EINVAL;
2126                 goto unlock;
2127         }
2128 out:
2129         *foliop = folio;
2130         return 0;
2131
2132         /*
2133          * Error recovery.
2134          */
2135 unacct:
2136         shmem_inode_unacct_blocks(inode, folio_nr_pages(folio));
2137
2138         if (folio_test_large(folio)) {
2139                 folio_unlock(folio);
2140                 folio_put(folio);
2141                 goto alloc_nohuge;
2142         }
2143 unlock:
2144         if (folio) {
2145                 folio_unlock(folio);
2146                 folio_put(folio);
2147         }
2148         if (error == -ENOSPC && !once++) {
2149                 shmem_recalc_inode(inode, 0, 0);
2150                 goto repeat;
2151         }
2152         if (error == -EEXIST)
2153                 goto repeat;
2154         return error;
2155 }
2156
2157 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2158                 enum sgp_type sgp)
2159 {
2160         return shmem_get_folio_gfp(inode, index, foliop, sgp,
2161                         mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
2162 }
2163
2164 /*
2165  * This is like autoremove_wake_function, but it removes the wait queue
2166  * entry unconditionally - even if something else had already woken the
2167  * target.
2168  */
2169 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2170 {
2171         int ret = default_wake_function(wait, mode, sync, key);
2172         list_del_init(&wait->entry);
2173         return ret;
2174 }
2175
2176 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2177 {
2178         struct vm_area_struct *vma = vmf->vma;
2179         struct inode *inode = file_inode(vma->vm_file);
2180         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2181         struct folio *folio = NULL;
2182         int err;
2183         vm_fault_t ret = VM_FAULT_LOCKED;
2184
2185         /*
2186          * Trinity finds that probing a hole which tmpfs is punching can
2187          * prevent the hole-punch from ever completing: which in turn
2188          * locks writers out with its hold on i_rwsem.  So refrain from
2189          * faulting pages into the hole while it's being punched.  Although
2190          * shmem_undo_range() does remove the additions, it may be unable to
2191          * keep up, as each new page needs its own unmap_mapping_range() call,
2192          * and the i_mmap tree grows ever slower to scan if new vmas are added.
2193          *
2194          * It does not matter if we sometimes reach this check just before the
2195          * hole-punch begins, so that one fault then races with the punch:
2196          * we just need to make racing faults a rare case.
2197          *
2198          * The implementation below would be much simpler if we just used a
2199          * standard mutex or completion: but we cannot take i_rwsem in fault,
2200          * and bloating every shmem inode for this unlikely case would be sad.
2201          */
2202         if (unlikely(inode->i_private)) {
2203                 struct shmem_falloc *shmem_falloc;
2204
2205                 spin_lock(&inode->i_lock);
2206                 shmem_falloc = inode->i_private;
2207                 if (shmem_falloc &&
2208                     shmem_falloc->waitq &&
2209                     vmf->pgoff >= shmem_falloc->start &&
2210                     vmf->pgoff < shmem_falloc->next) {
2211                         struct file *fpin;
2212                         wait_queue_head_t *shmem_falloc_waitq;
2213                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2214
2215                         ret = VM_FAULT_NOPAGE;
2216                         fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2217                         if (fpin)
2218                                 ret = VM_FAULT_RETRY;
2219
2220                         shmem_falloc_waitq = shmem_falloc->waitq;
2221                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2222                                         TASK_UNINTERRUPTIBLE);
2223                         spin_unlock(&inode->i_lock);
2224                         schedule();
2225
2226                         /*
2227                          * shmem_falloc_waitq points into the shmem_fallocate()
2228                          * stack of the hole-punching task: shmem_falloc_waitq
2229                          * is usually invalid by the time we reach here, but
2230                          * finish_wait() does not dereference it in that case;
2231                          * though i_lock needed lest racing with wake_up_all().
2232                          */
2233                         spin_lock(&inode->i_lock);
2234                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2235                         spin_unlock(&inode->i_lock);
2236
2237                         if (fpin)
2238                                 fput(fpin);
2239                         return ret;
2240                 }
2241                 spin_unlock(&inode->i_lock);
2242         }
2243
2244         err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2245                                   gfp, vma, vmf, &ret);
2246         if (err)
2247                 return vmf_error(err);
2248         if (folio)
2249                 vmf->page = folio_file_page(folio, vmf->pgoff);
2250         return ret;
2251 }
2252
2253 unsigned long shmem_get_unmapped_area(struct file *file,
2254                                       unsigned long uaddr, unsigned long len,
2255                                       unsigned long pgoff, unsigned long flags)
2256 {
2257         unsigned long (*get_area)(struct file *,
2258                 unsigned long, unsigned long, unsigned long, unsigned long);
2259         unsigned long addr;
2260         unsigned long offset;
2261         unsigned long inflated_len;
2262         unsigned long inflated_addr;
2263         unsigned long inflated_offset;
2264
2265         if (len > TASK_SIZE)
2266                 return -ENOMEM;
2267
2268         get_area = current->mm->get_unmapped_area;
2269         addr = get_area(file, uaddr, len, pgoff, flags);
2270
2271         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2272                 return addr;
2273         if (IS_ERR_VALUE(addr))
2274                 return addr;
2275         if (addr & ~PAGE_MASK)
2276                 return addr;
2277         if (addr > TASK_SIZE - len)
2278                 return addr;
2279
2280         if (shmem_huge == SHMEM_HUGE_DENY)
2281                 return addr;
2282         if (len < HPAGE_PMD_SIZE)
2283                 return addr;
2284         if (flags & MAP_FIXED)
2285                 return addr;
2286         /*
2287          * Our priority is to support MAP_SHARED mapped hugely;
2288          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2289          * But if caller specified an address hint and we allocated area there
2290          * successfully, respect that as before.
2291          */
2292         if (uaddr == addr)
2293                 return addr;
2294
2295         if (shmem_huge != SHMEM_HUGE_FORCE) {
2296                 struct super_block *sb;
2297
2298                 if (file) {
2299                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2300                         sb = file_inode(file)->i_sb;
2301                 } else {
2302                         /*
2303                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2304                          * for "/dev/zero", to create a shared anonymous object.
2305                          */
2306                         if (IS_ERR(shm_mnt))
2307                                 return addr;
2308                         sb = shm_mnt->mnt_sb;
2309                 }
2310                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2311                         return addr;
2312         }
2313
2314         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2315         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2316                 return addr;
2317         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2318                 return addr;
2319
2320         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2321         if (inflated_len > TASK_SIZE)
2322                 return addr;
2323         if (inflated_len < len)
2324                 return addr;
2325
2326         inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2327         if (IS_ERR_VALUE(inflated_addr))
2328                 return addr;
2329         if (inflated_addr & ~PAGE_MASK)
2330                 return addr;
2331
2332         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2333         inflated_addr += offset - inflated_offset;
2334         if (inflated_offset > offset)
2335                 inflated_addr += HPAGE_PMD_SIZE;
2336
2337         if (inflated_addr > TASK_SIZE - len)
2338                 return addr;
2339         return inflated_addr;
2340 }
2341
2342 #ifdef CONFIG_NUMA
2343 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2344 {
2345         struct inode *inode = file_inode(vma->vm_file);
2346         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2347 }
2348
2349 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2350                                           unsigned long addr)
2351 {
2352         struct inode *inode = file_inode(vma->vm_file);
2353         pgoff_t index;
2354
2355         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2356         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2357 }
2358 #endif
2359
2360 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2361 {
2362         struct inode *inode = file_inode(file);
2363         struct shmem_inode_info *info = SHMEM_I(inode);
2364         int retval = -ENOMEM;
2365
2366         /*
2367          * What serializes the accesses to info->flags?
2368          * ipc_lock_object() when called from shmctl_do_lock(),
2369          * no serialization needed when called from shm_destroy().
2370          */
2371         if (lock && !(info->flags & VM_LOCKED)) {
2372                 if (!user_shm_lock(inode->i_size, ucounts))
2373                         goto out_nomem;
2374                 info->flags |= VM_LOCKED;
2375                 mapping_set_unevictable(file->f_mapping);
2376         }
2377         if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2378                 user_shm_unlock(inode->i_size, ucounts);
2379                 info->flags &= ~VM_LOCKED;
2380                 mapping_clear_unevictable(file->f_mapping);
2381         }
2382         retval = 0;
2383
2384 out_nomem:
2385         return retval;
2386 }
2387
2388 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2389 {
2390         struct inode *inode = file_inode(file);
2391         struct shmem_inode_info *info = SHMEM_I(inode);
2392         int ret;
2393
2394         ret = seal_check_future_write(info->seals, vma);
2395         if (ret)
2396                 return ret;
2397
2398         /* arm64 - allow memory tagging on RAM-based files */
2399         vm_flags_set(vma, VM_MTE_ALLOWED);
2400
2401         file_accessed(file);
2402         /* This is anonymous shared memory if it is unlinked at the time of mmap */
2403         if (inode->i_nlink)
2404                 vma->vm_ops = &shmem_vm_ops;
2405         else
2406                 vma->vm_ops = &shmem_anon_vm_ops;
2407         return 0;
2408 }
2409
2410 static int shmem_file_open(struct inode *inode, struct file *file)
2411 {
2412         file->f_mode |= FMODE_CAN_ODIRECT;
2413         return generic_file_open(inode, file);
2414 }
2415
2416 #ifdef CONFIG_TMPFS_XATTR
2417 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2418
2419 /*
2420  * chattr's fsflags are unrelated to extended attributes,
2421  * but tmpfs has chosen to enable them under the same config option.
2422  */
2423 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2424 {
2425         unsigned int i_flags = 0;
2426
2427         if (fsflags & FS_NOATIME_FL)
2428                 i_flags |= S_NOATIME;
2429         if (fsflags & FS_APPEND_FL)
2430                 i_flags |= S_APPEND;
2431         if (fsflags & FS_IMMUTABLE_FL)
2432                 i_flags |= S_IMMUTABLE;
2433         /*
2434          * But FS_NODUMP_FL does not require any action in i_flags.
2435          */
2436         inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2437 }
2438 #else
2439 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2440 {
2441 }
2442 #define shmem_initxattrs NULL
2443 #endif
2444
2445 static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
2446 {
2447         return &SHMEM_I(inode)->dir_offsets;
2448 }
2449
2450 static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
2451                                              struct super_block *sb,
2452                                              struct inode *dir, umode_t mode,
2453                                              dev_t dev, unsigned long flags)
2454 {
2455         struct inode *inode;
2456         struct shmem_inode_info *info;
2457         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2458         ino_t ino;
2459         int err;
2460
2461         err = shmem_reserve_inode(sb, &ino);
2462         if (err)
2463                 return ERR_PTR(err);
2464
2465
2466         inode = new_inode(sb);
2467         if (!inode) {
2468                 shmem_free_inode(sb, 0);
2469                 return ERR_PTR(-ENOSPC);
2470         }
2471
2472         inode->i_ino = ino;
2473         inode_init_owner(idmap, inode, dir, mode);
2474         inode->i_blocks = 0;
2475         inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
2476         inode->i_generation = get_random_u32();
2477         info = SHMEM_I(inode);
2478         memset(info, 0, (char *)inode - (char *)info);
2479         spin_lock_init(&info->lock);
2480         atomic_set(&info->stop_eviction, 0);
2481         info->seals = F_SEAL_SEAL;
2482         info->flags = flags & VM_NORESERVE;
2483         info->i_crtime = inode->i_mtime;
2484         info->fsflags = (dir == NULL) ? 0 :
2485                 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2486         if (info->fsflags)
2487                 shmem_set_inode_flags(inode, info->fsflags);
2488         INIT_LIST_HEAD(&info->shrinklist);
2489         INIT_LIST_HEAD(&info->swaplist);
2490         INIT_LIST_HEAD(&info->swaplist);
2491         if (sbinfo->noswap)
2492                 mapping_set_unevictable(inode->i_mapping);
2493         simple_xattrs_init(&info->xattrs);
2494         cache_no_acl(inode);
2495         mapping_set_large_folios(inode->i_mapping);
2496
2497         switch (mode & S_IFMT) {
2498         default:
2499                 inode->i_op = &shmem_special_inode_operations;
2500                 init_special_inode(inode, mode, dev);
2501                 break;
2502         case S_IFREG:
2503                 inode->i_mapping->a_ops = &shmem_aops;
2504                 inode->i_op = &shmem_inode_operations;
2505                 inode->i_fop = &shmem_file_operations;
2506                 mpol_shared_policy_init(&info->policy,
2507                                          shmem_get_sbmpol(sbinfo));
2508                 break;
2509         case S_IFDIR:
2510                 inc_nlink(inode);
2511                 /* Some things misbehave if size == 0 on a directory */
2512                 inode->i_size = 2 * BOGO_DIRENT_SIZE;
2513                 inode->i_op = &shmem_dir_inode_operations;
2514                 inode->i_fop = &simple_offset_dir_operations;
2515                 simple_offset_init(shmem_get_offset_ctx(inode));
2516                 break;
2517         case S_IFLNK:
2518                 /*
2519                  * Must not load anything in the rbtree,
2520                  * mpol_free_shared_policy will not be called.
2521                  */
2522                 mpol_shared_policy_init(&info->policy, NULL);
2523                 break;
2524         }
2525
2526         lockdep_annotate_inode_mutex_key(inode);
2527         return inode;
2528 }
2529
2530 #ifdef CONFIG_TMPFS_QUOTA
2531 static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
2532                                      struct super_block *sb, struct inode *dir,
2533                                      umode_t mode, dev_t dev, unsigned long flags)
2534 {
2535         int err;
2536         struct inode *inode;
2537
2538         inode = __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
2539         if (IS_ERR(inode))
2540                 return inode;
2541
2542         err = dquot_initialize(inode);
2543         if (err)
2544                 goto errout;
2545
2546         err = dquot_alloc_inode(inode);
2547         if (err) {
2548                 dquot_drop(inode);
2549                 goto errout;
2550         }
2551         return inode;
2552
2553 errout:
2554         inode->i_flags |= S_NOQUOTA;
2555         iput(inode);
2556         return ERR_PTR(err);
2557 }
2558 #else
2559 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
2560                                      struct super_block *sb, struct inode *dir,
2561                                      umode_t mode, dev_t dev, unsigned long flags)
2562 {
2563         return __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
2564 }
2565 #endif /* CONFIG_TMPFS_QUOTA */
2566
2567 #ifdef CONFIG_USERFAULTFD
2568 int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
2569                            struct vm_area_struct *dst_vma,
2570                            unsigned long dst_addr,
2571                            unsigned long src_addr,
2572                            uffd_flags_t flags,
2573                            struct folio **foliop)
2574 {
2575         struct inode *inode = file_inode(dst_vma->vm_file);
2576         struct shmem_inode_info *info = SHMEM_I(inode);
2577         struct address_space *mapping = inode->i_mapping;
2578         gfp_t gfp = mapping_gfp_mask(mapping);
2579         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2580         void *page_kaddr;
2581         struct folio *folio;
2582         int ret;
2583         pgoff_t max_off;
2584
2585         if (shmem_inode_acct_block(inode, 1)) {
2586                 /*
2587                  * We may have got a page, returned -ENOENT triggering a retry,
2588                  * and now we find ourselves with -ENOMEM. Release the page, to
2589                  * avoid a BUG_ON in our caller.
2590                  */
2591                 if (unlikely(*foliop)) {
2592                         folio_put(*foliop);
2593                         *foliop = NULL;
2594                 }
2595                 return -ENOMEM;
2596         }
2597
2598         if (!*foliop) {
2599                 ret = -ENOMEM;
2600                 folio = shmem_alloc_folio(gfp, info, pgoff);
2601                 if (!folio)
2602                         goto out_unacct_blocks;
2603
2604                 if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) {
2605                         page_kaddr = kmap_local_folio(folio, 0);
2606                         /*
2607                          * The read mmap_lock is held here.  Despite the
2608                          * mmap_lock being read recursive a deadlock is still
2609                          * possible if a writer has taken a lock.  For example:
2610                          *
2611                          * process A thread 1 takes read lock on own mmap_lock
2612                          * process A thread 2 calls mmap, blocks taking write lock
2613                          * process B thread 1 takes page fault, read lock on own mmap lock
2614                          * process B thread 2 calls mmap, blocks taking write lock
2615                          * process A thread 1 blocks taking read lock on process B
2616                          * process B thread 1 blocks taking read lock on process A
2617                          *
2618                          * Disable page faults to prevent potential deadlock
2619                          * and retry the copy outside the mmap_lock.
2620                          */
2621                         pagefault_disable();
2622                         ret = copy_from_user(page_kaddr,
2623                                              (const void __user *)src_addr,
2624                                              PAGE_SIZE);
2625                         pagefault_enable();
2626                         kunmap_local(page_kaddr);
2627
2628                         /* fallback to copy_from_user outside mmap_lock */
2629                         if (unlikely(ret)) {
2630                                 *foliop = folio;
2631                                 ret = -ENOENT;
2632                                 /* don't free the page */
2633                                 goto out_unacct_blocks;
2634                         }
2635
2636                         flush_dcache_folio(folio);
2637                 } else {                /* ZEROPAGE */
2638                         clear_user_highpage(&folio->page, dst_addr);
2639                 }
2640         } else {
2641                 folio = *foliop;
2642                 VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2643                 *foliop = NULL;
2644         }
2645
2646         VM_BUG_ON(folio_test_locked(folio));
2647         VM_BUG_ON(folio_test_swapbacked(folio));
2648         __folio_set_locked(folio);
2649         __folio_set_swapbacked(folio);
2650         __folio_mark_uptodate(folio);
2651
2652         ret = -EFAULT;
2653         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2654         if (unlikely(pgoff >= max_off))
2655                 goto out_release;
2656
2657         ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
2658                                       gfp & GFP_RECLAIM_MASK, dst_vma->vm_mm);
2659         if (ret)
2660                 goto out_release;
2661
2662         ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,
2663                                        &folio->page, true, flags);
2664         if (ret)
2665                 goto out_delete_from_cache;
2666
2667         shmem_recalc_inode(inode, 1, 0);
2668         folio_unlock(folio);
2669         return 0;
2670 out_delete_from_cache:
2671         filemap_remove_folio(folio);
2672 out_release:
2673         folio_unlock(folio);
2674         folio_put(folio);
2675 out_unacct_blocks:
2676         shmem_inode_unacct_blocks(inode, 1);
2677         return ret;
2678 }
2679 #endif /* CONFIG_USERFAULTFD */
2680
2681 #ifdef CONFIG_TMPFS
2682 static const struct inode_operations shmem_symlink_inode_operations;
2683 static const struct inode_operations shmem_short_symlink_operations;
2684
2685 static int
2686 shmem_write_begin(struct file *file, struct address_space *mapping,
2687                         loff_t pos, unsigned len,
2688                         struct page **pagep, void **fsdata)
2689 {
2690         struct inode *inode = mapping->host;
2691         struct shmem_inode_info *info = SHMEM_I(inode);
2692         pgoff_t index = pos >> PAGE_SHIFT;
2693         struct folio *folio;
2694         int ret = 0;
2695
2696         /* i_rwsem is held by caller */
2697         if (unlikely(info->seals & (F_SEAL_GROW |
2698                                    F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2699                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2700                         return -EPERM;
2701                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2702                         return -EPERM;
2703         }
2704
2705         ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2706
2707         if (ret)
2708                 return ret;
2709
2710         *pagep = folio_file_page(folio, index);
2711         if (PageHWPoison(*pagep)) {
2712                 folio_unlock(folio);
2713                 folio_put(folio);
2714                 *pagep = NULL;
2715                 return -EIO;
2716         }
2717
2718         return 0;
2719 }
2720
2721 static int
2722 shmem_write_end(struct file *file, struct address_space *mapping,
2723                         loff_t pos, unsigned len, unsigned copied,
2724                         struct page *page, void *fsdata)
2725 {
2726         struct folio *folio = page_folio(page);
2727         struct inode *inode = mapping->host;
2728
2729         if (pos + copied > inode->i_size)
2730                 i_size_write(inode, pos + copied);
2731
2732         if (!folio_test_uptodate(folio)) {
2733                 if (copied < folio_size(folio)) {
2734                         size_t from = offset_in_folio(folio, pos);
2735                         folio_zero_segments(folio, 0, from,
2736                                         from + copied, folio_size(folio));
2737                 }
2738                 folio_mark_uptodate(folio);
2739         }
2740         folio_mark_dirty(folio);
2741         folio_unlock(folio);
2742         folio_put(folio);
2743
2744         return copied;
2745 }
2746
2747 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2748 {
2749         struct file *file = iocb->ki_filp;
2750         struct inode *inode = file_inode(file);
2751         struct address_space *mapping = inode->i_mapping;
2752         pgoff_t index;
2753         unsigned long offset;
2754         int error = 0;
2755         ssize_t retval = 0;
2756         loff_t *ppos = &iocb->ki_pos;
2757
2758         index = *ppos >> PAGE_SHIFT;
2759         offset = *ppos & ~PAGE_MASK;
2760
2761         for (;;) {
2762                 struct folio *folio = NULL;
2763                 struct page *page = NULL;
2764                 pgoff_t end_index;
2765                 unsigned long nr, ret;
2766                 loff_t i_size = i_size_read(inode);
2767
2768                 end_index = i_size >> PAGE_SHIFT;
2769                 if (index > end_index)
2770                         break;
2771                 if (index == end_index) {
2772                         nr = i_size & ~PAGE_MASK;
2773                         if (nr <= offset)
2774                                 break;
2775                 }
2776
2777                 error = shmem_get_folio(inode, index, &folio, SGP_READ);
2778                 if (error) {
2779                         if (error == -EINVAL)
2780                                 error = 0;
2781                         break;
2782                 }
2783                 if (folio) {
2784                         folio_unlock(folio);
2785
2786                         page = folio_file_page(folio, index);
2787                         if (PageHWPoison(page)) {
2788                                 folio_put(folio);
2789                                 error = -EIO;
2790                                 break;
2791                         }
2792                 }
2793
2794                 /*
2795                  * We must evaluate after, since reads (unlike writes)
2796                  * are called without i_rwsem protection against truncate
2797                  */
2798                 nr = PAGE_SIZE;
2799                 i_size = i_size_read(inode);
2800                 end_index = i_size >> PAGE_SHIFT;
2801                 if (index == end_index) {
2802                         nr = i_size & ~PAGE_MASK;
2803                         if (nr <= offset) {
2804                                 if (folio)
2805                                         folio_put(folio);
2806                                 break;
2807                         }
2808                 }
2809                 nr -= offset;
2810
2811                 if (folio) {
2812                         /*
2813                          * If users can be writing to this page using arbitrary
2814                          * virtual addresses, take care about potential aliasing
2815                          * before reading the page on the kernel side.
2816                          */
2817                         if (mapping_writably_mapped(mapping))
2818                                 flush_dcache_page(page);
2819                         /*
2820                          * Mark the page accessed if we read the beginning.
2821                          */
2822                         if (!offset)
2823                                 folio_mark_accessed(folio);
2824                         /*
2825                          * Ok, we have the page, and it's up-to-date, so
2826                          * now we can copy it to user space...
2827                          */
2828                         ret = copy_page_to_iter(page, offset, nr, to);
2829                         folio_put(folio);
2830
2831                 } else if (user_backed_iter(to)) {
2832                         /*
2833                          * Copy to user tends to be so well optimized, but
2834                          * clear_user() not so much, that it is noticeably
2835                          * faster to copy the zero page instead of clearing.
2836                          */
2837                         ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
2838                 } else {
2839                         /*
2840                          * But submitting the same page twice in a row to
2841                          * splice() - or others? - can result in confusion:
2842                          * so don't attempt that optimization on pipes etc.
2843                          */
2844                         ret = iov_iter_zero(nr, to);
2845                 }
2846
2847                 retval += ret;
2848                 offset += ret;
2849                 index += offset >> PAGE_SHIFT;
2850                 offset &= ~PAGE_MASK;
2851
2852                 if (!iov_iter_count(to))
2853                         break;
2854                 if (ret < nr) {
2855                         error = -EFAULT;
2856                         break;
2857                 }
2858                 cond_resched();
2859         }
2860
2861         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2862         file_accessed(file);
2863         return retval ? retval : error;
2864 }
2865
2866 static ssize_t shmem_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
2867 {
2868         struct file *file = iocb->ki_filp;
2869         struct inode *inode = file->f_mapping->host;
2870         ssize_t ret;
2871
2872         inode_lock(inode);
2873         ret = generic_write_checks(iocb, from);
2874         if (ret <= 0)
2875                 goto unlock;
2876         ret = file_remove_privs(file);
2877         if (ret)
2878                 goto unlock;
2879         ret = file_update_time(file);
2880         if (ret)
2881                 goto unlock;
2882         ret = generic_perform_write(iocb, from);
2883 unlock:
2884         inode_unlock(inode);
2885         return ret;
2886 }
2887
2888 static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
2889                               struct pipe_buffer *buf)
2890 {
2891         return true;
2892 }
2893
2894 static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
2895                                   struct pipe_buffer *buf)
2896 {
2897 }
2898
2899 static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
2900                                     struct pipe_buffer *buf)
2901 {
2902         return false;
2903 }
2904
2905 static const struct pipe_buf_operations zero_pipe_buf_ops = {
2906         .release        = zero_pipe_buf_release,
2907         .try_steal      = zero_pipe_buf_try_steal,
2908         .get            = zero_pipe_buf_get,
2909 };
2910
2911 static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe,
2912                                         loff_t fpos, size_t size)
2913 {
2914         size_t offset = fpos & ~PAGE_MASK;
2915
2916         size = min_t(size_t, size, PAGE_SIZE - offset);
2917
2918         if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
2919                 struct pipe_buffer *buf = pipe_head_buf(pipe);
2920
2921                 *buf = (struct pipe_buffer) {
2922                         .ops    = &zero_pipe_buf_ops,
2923                         .page   = ZERO_PAGE(0),
2924                         .offset = offset,
2925                         .len    = size,
2926                 };
2927                 pipe->head++;
2928         }
2929
2930         return size;
2931 }
2932
2933 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
2934                                       struct pipe_inode_info *pipe,
2935                                       size_t len, unsigned int flags)
2936 {
2937         struct inode *inode = file_inode(in);
2938         struct address_space *mapping = inode->i_mapping;
2939         struct folio *folio = NULL;
2940         size_t total_spliced = 0, used, npages, n, part;
2941         loff_t isize;
2942         int error = 0;
2943
2944         /* Work out how much data we can actually add into the pipe */
2945         used = pipe_occupancy(pipe->head, pipe->tail);
2946         npages = max_t(ssize_t, pipe->max_usage - used, 0);
2947         len = min_t(size_t, len, npages * PAGE_SIZE);
2948
2949         do {
2950                 if (*ppos >= i_size_read(inode))
2951                         break;
2952
2953                 error = shmem_get_folio(inode, *ppos / PAGE_SIZE, &folio,
2954                                         SGP_READ);
2955                 if (error) {
2956                         if (error == -EINVAL)
2957                                 error = 0;
2958                         break;
2959                 }
2960                 if (folio) {
2961                         folio_unlock(folio);
2962
2963                         if (folio_test_hwpoison(folio) ||
2964                             (folio_test_large(folio) &&
2965                              folio_test_has_hwpoisoned(folio))) {
2966                                 error = -EIO;
2967                                 break;
2968                         }
2969                 }
2970
2971                 /*
2972                  * i_size must be checked after we know the pages are Uptodate.
2973                  *
2974                  * Checking i_size after the check allows us to calculate
2975                  * the correct value for "nr", which means the zero-filled
2976                  * part of the page is not copied back to userspace (unless
2977                  * another truncate extends the file - this is desired though).
2978                  */
2979                 isize = i_size_read(inode);
2980                 if (unlikely(*ppos >= isize))
2981                         break;
2982                 part = min_t(loff_t, isize - *ppos, len);
2983
2984                 if (folio) {
2985                         /*
2986                          * If users can be writing to this page using arbitrary
2987                          * virtual addresses, take care about potential aliasing
2988                          * before reading the page on the kernel side.
2989                          */
2990                         if (mapping_writably_mapped(mapping))
2991                                 flush_dcache_folio(folio);
2992                         folio_mark_accessed(folio);
2993                         /*
2994                          * Ok, we have the page, and it's up-to-date, so we can
2995                          * now splice it into the pipe.
2996                          */
2997                         n = splice_folio_into_pipe(pipe, folio, *ppos, part);
2998                         folio_put(folio);
2999                         folio = NULL;
3000                 } else {
3001                         n = splice_zeropage_into_pipe(pipe, *ppos, part);
3002                 }
3003
3004                 if (!n)
3005                         break;
3006                 len -= n;
3007                 total_spliced += n;
3008                 *ppos += n;
3009                 in->f_ra.prev_pos = *ppos;
3010                 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
3011                         break;
3012
3013                 cond_resched();
3014         } while (len);
3015
3016         if (folio)
3017                 folio_put(folio);
3018
3019         file_accessed(in);
3020         return total_spliced ? total_spliced : error;
3021 }
3022
3023 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
3024 {
3025         struct address_space *mapping = file->f_mapping;
3026         struct inode *inode = mapping->host;
3027
3028         if (whence != SEEK_DATA && whence != SEEK_HOLE)
3029                 return generic_file_llseek_size(file, offset, whence,
3030                                         MAX_LFS_FILESIZE, i_size_read(inode));
3031         if (offset < 0)
3032                 return -ENXIO;
3033
3034         inode_lock(inode);
3035         /* We're holding i_rwsem so we can access i_size directly */
3036         offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
3037         if (offset >= 0)
3038                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
3039         inode_unlock(inode);
3040         return offset;
3041 }
3042
3043 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
3044                                                          loff_t len)
3045 {
3046         struct inode *inode = file_inode(file);
3047         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3048         struct shmem_inode_info *info = SHMEM_I(inode);
3049         struct shmem_falloc shmem_falloc;
3050         pgoff_t start, index, end, undo_fallocend;
3051         int error;
3052
3053         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3054                 return -EOPNOTSUPP;
3055
3056         inode_lock(inode);
3057
3058         if (mode & FALLOC_FL_PUNCH_HOLE) {
3059                 struct address_space *mapping = file->f_mapping;
3060                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
3061                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
3062                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
3063
3064                 /* protected by i_rwsem */
3065                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
3066                         error = -EPERM;
3067                         goto out;
3068                 }
3069
3070                 shmem_falloc.waitq = &shmem_falloc_waitq;
3071                 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
3072                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
3073                 spin_lock(&inode->i_lock);
3074                 inode->i_private = &shmem_falloc;
3075                 spin_unlock(&inode->i_lock);
3076
3077                 if ((u64)unmap_end > (u64)unmap_start)
3078                         unmap_mapping_range(mapping, unmap_start,
3079                                             1 + unmap_end - unmap_start, 0);
3080                 shmem_truncate_range(inode, offset, offset + len - 1);
3081                 /* No need to unmap again: hole-punching leaves COWed pages */
3082
3083                 spin_lock(&inode->i_lock);
3084                 inode->i_private = NULL;
3085                 wake_up_all(&shmem_falloc_waitq);
3086                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
3087                 spin_unlock(&inode->i_lock);
3088                 error = 0;
3089                 goto out;
3090         }
3091
3092         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
3093         error = inode_newsize_ok(inode, offset + len);
3094         if (error)
3095                 goto out;
3096
3097         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
3098                 error = -EPERM;
3099                 goto out;
3100         }
3101
3102         start = offset >> PAGE_SHIFT;
3103         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
3104         /* Try to avoid a swapstorm if len is impossible to satisfy */
3105         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
3106                 error = -ENOSPC;
3107                 goto out;
3108         }
3109
3110         shmem_falloc.waitq = NULL;
3111         shmem_falloc.start = start;
3112         shmem_falloc.next  = start;
3113         shmem_falloc.nr_falloced = 0;
3114         shmem_falloc.nr_unswapped = 0;
3115         spin_lock(&inode->i_lock);
3116         inode->i_private = &shmem_falloc;
3117         spin_unlock(&inode->i_lock);
3118
3119         /*
3120          * info->fallocend is only relevant when huge pages might be
3121          * involved: to prevent split_huge_page() freeing fallocated
3122          * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
3123          */
3124         undo_fallocend = info->fallocend;
3125         if (info->fallocend < end)
3126                 info->fallocend = end;
3127
3128         for (index = start; index < end; ) {
3129                 struct folio *folio;
3130
3131                 /*
3132                  * Good, the fallocate(2) manpage permits EINTR: we may have
3133                  * been interrupted because we are using up too much memory.
3134                  */
3135                 if (signal_pending(current))
3136                         error = -EINTR;
3137                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
3138                         error = -ENOMEM;
3139                 else
3140                         error = shmem_get_folio(inode, index, &folio,
3141                                                 SGP_FALLOC);
3142                 if (error) {
3143                         info->fallocend = undo_fallocend;
3144                         /* Remove the !uptodate folios we added */
3145                         if (index > start) {
3146                                 shmem_undo_range(inode,
3147                                     (loff_t)start << PAGE_SHIFT,
3148                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
3149                         }
3150                         goto undone;
3151                 }
3152
3153                 /*
3154                  * Here is a more important optimization than it appears:
3155                  * a second SGP_FALLOC on the same large folio will clear it,
3156                  * making it uptodate and un-undoable if we fail later.
3157                  */
3158                 index = folio_next_index(folio);
3159                 /* Beware 32-bit wraparound */
3160                 if (!index)
3161                         index--;
3162
3163                 /*
3164                  * Inform shmem_writepage() how far we have reached.
3165                  * No need for lock or barrier: we have the page lock.
3166                  */
3167                 if (!folio_test_uptodate(folio))
3168                         shmem_falloc.nr_falloced += index - shmem_falloc.next;
3169                 shmem_falloc.next = index;
3170
3171                 /*
3172                  * If !uptodate, leave it that way so that freeable folios
3173                  * can be recognized if we need to rollback on error later.
3174                  * But mark it dirty so that memory pressure will swap rather
3175                  * than free the folios we are allocating (and SGP_CACHE folios
3176                  * might still be clean: we now need to mark those dirty too).
3177                  */
3178                 folio_mark_dirty(folio);
3179                 folio_unlock(folio);
3180                 folio_put(folio);
3181                 cond_resched();
3182         }
3183
3184         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
3185                 i_size_write(inode, offset + len);
3186 undone:
3187         spin_lock(&inode->i_lock);
3188         inode->i_private = NULL;
3189         spin_unlock(&inode->i_lock);
3190 out:
3191         if (!error)
3192                 file_modified(file);
3193         inode_unlock(inode);
3194         return error;
3195 }
3196
3197 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
3198 {
3199         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
3200
3201         buf->f_type = TMPFS_MAGIC;
3202         buf->f_bsize = PAGE_SIZE;
3203         buf->f_namelen = NAME_MAX;
3204         if (sbinfo->max_blocks) {
3205                 buf->f_blocks = sbinfo->max_blocks;
3206                 buf->f_bavail =
3207                 buf->f_bfree  = sbinfo->max_blocks -
3208                                 percpu_counter_sum(&sbinfo->used_blocks);
3209         }
3210         if (sbinfo->max_inodes) {
3211                 buf->f_files = sbinfo->max_inodes;
3212                 buf->f_ffree = sbinfo->free_ispace / BOGO_INODE_SIZE;
3213         }
3214         /* else leave those fields 0 like simple_statfs */
3215
3216         buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
3217
3218         return 0;
3219 }
3220
3221 /*
3222  * File creation. Allocate an inode, and we're done..
3223  */
3224 static int
3225 shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
3226             struct dentry *dentry, umode_t mode, dev_t dev)
3227 {
3228         struct inode *inode;
3229         int error;
3230
3231         inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, dev, VM_NORESERVE);
3232         if (IS_ERR(inode))
3233                 return PTR_ERR(inode);
3234
3235         error = simple_acl_create(dir, inode);
3236         if (error)
3237                 goto out_iput;
3238         error = security_inode_init_security(inode, dir,
3239                                              &dentry->d_name,
3240                                              shmem_initxattrs, NULL);
3241         if (error && error != -EOPNOTSUPP)
3242                 goto out_iput;
3243
3244         error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3245         if (error)
3246                 goto out_iput;
3247
3248         dir->i_size += BOGO_DIRENT_SIZE;
3249         dir->i_mtime = inode_set_ctime_current(dir);
3250         inode_inc_iversion(dir);
3251         d_instantiate(dentry, inode);
3252         dget(dentry); /* Extra count - pin the dentry in core */
3253         return error;
3254
3255 out_iput:
3256         iput(inode);
3257         return error;
3258 }
3259
3260 static int
3261 shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
3262               struct file *file, umode_t mode)
3263 {
3264         struct inode *inode;
3265         int error;
3266
3267         inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, 0, VM_NORESERVE);
3268
3269         if (IS_ERR(inode)) {
3270                 error = PTR_ERR(inode);
3271                 goto err_out;
3272         }
3273
3274         error = security_inode_init_security(inode, dir,
3275                                              NULL,
3276                                              shmem_initxattrs, NULL);
3277         if (error && error != -EOPNOTSUPP)
3278                 goto out_iput;
3279         error = simple_acl_create(dir, inode);
3280         if (error)
3281                 goto out_iput;
3282         d_tmpfile(file, inode);
3283
3284 err_out:
3285         return finish_open_simple(file, error);
3286 out_iput:
3287         iput(inode);
3288         return error;
3289 }
3290
3291 static int shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
3292                        struct dentry *dentry, umode_t mode)
3293 {
3294         int error;
3295
3296         error = shmem_mknod(idmap, dir, dentry, mode | S_IFDIR, 0);
3297         if (error)
3298                 return error;
3299         inc_nlink(dir);
3300         return 0;
3301 }
3302
3303 static int shmem_create(struct mnt_idmap *idmap, struct inode *dir,
3304                         struct dentry *dentry, umode_t mode, bool excl)
3305 {
3306         return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
3307 }
3308
3309 /*
3310  * Link a file..
3311  */
3312 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
3313 {
3314         struct inode *inode = d_inode(old_dentry);
3315         int ret = 0;
3316
3317         /*
3318          * No ordinary (disk based) filesystem counts links as inodes;
3319          * but each new link needs a new dentry, pinning lowmem, and
3320          * tmpfs dentries cannot be pruned until they are unlinked.
3321          * But if an O_TMPFILE file is linked into the tmpfs, the
3322          * first link must skip that, to get the accounting right.
3323          */
3324         if (inode->i_nlink) {
3325                 ret = shmem_reserve_inode(inode->i_sb, NULL);
3326                 if (ret)
3327                         goto out;
3328         }
3329
3330         ret = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3331         if (ret) {
3332                 if (inode->i_nlink)
3333                         shmem_free_inode(inode->i_sb, 0);
3334                 goto out;
3335         }
3336
3337         dir->i_size += BOGO_DIRENT_SIZE;
3338         dir->i_mtime = inode_set_ctime_to_ts(dir,
3339                                              inode_set_ctime_current(inode));
3340         inode_inc_iversion(dir);
3341         inc_nlink(inode);
3342         ihold(inode);   /* New dentry reference */
3343         dget(dentry);           /* Extra pinning count for the created dentry */
3344         d_instantiate(dentry, inode);
3345 out:
3346         return ret;
3347 }
3348
3349 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3350 {
3351         struct inode *inode = d_inode(dentry);
3352
3353         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3354                 shmem_free_inode(inode->i_sb, 0);
3355
3356         simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
3357
3358         dir->i_size -= BOGO_DIRENT_SIZE;
3359         dir->i_mtime = inode_set_ctime_to_ts(dir,
3360                                              inode_set_ctime_current(inode));
3361         inode_inc_iversion(dir);
3362         drop_nlink(inode);
3363         dput(dentry);   /* Undo the count from "create" - this does all the work */
3364         return 0;
3365 }
3366
3367 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3368 {
3369         if (!simple_empty(dentry))
3370                 return -ENOTEMPTY;
3371
3372         drop_nlink(d_inode(dentry));
3373         drop_nlink(dir);
3374         return shmem_unlink(dir, dentry);
3375 }
3376
3377 static int shmem_whiteout(struct mnt_idmap *idmap,
3378                           struct inode *old_dir, struct dentry *old_dentry)
3379 {
3380         struct dentry *whiteout;
3381         int error;
3382
3383         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3384         if (!whiteout)
3385                 return -ENOMEM;
3386
3387         error = shmem_mknod(idmap, old_dir, whiteout,
3388                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3389         dput(whiteout);
3390         if (error)
3391                 return error;
3392
3393         /*
3394          * Cheat and hash the whiteout while the old dentry is still in
3395          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3396          *
3397          * d_lookup() will consistently find one of them at this point,
3398          * not sure which one, but that isn't even important.
3399          */
3400         d_rehash(whiteout);
3401         return 0;
3402 }
3403
3404 /*
3405  * The VFS layer already does all the dentry stuff for rename,
3406  * we just have to decrement the usage count for the target if
3407  * it exists so that the VFS layer correctly free's it when it
3408  * gets overwritten.
3409  */
3410 static int shmem_rename2(struct mnt_idmap *idmap,
3411                          struct inode *old_dir, struct dentry *old_dentry,
3412                          struct inode *new_dir, struct dentry *new_dentry,
3413                          unsigned int flags)
3414 {
3415         struct inode *inode = d_inode(old_dentry);
3416         int they_are_dirs = S_ISDIR(inode->i_mode);
3417         int error;
3418
3419         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3420                 return -EINVAL;
3421
3422         if (flags & RENAME_EXCHANGE)
3423                 return simple_offset_rename_exchange(old_dir, old_dentry,
3424                                                      new_dir, new_dentry);
3425
3426         if (!simple_empty(new_dentry))
3427                 return -ENOTEMPTY;
3428
3429         if (flags & RENAME_WHITEOUT) {
3430                 error = shmem_whiteout(idmap, old_dir, old_dentry);
3431                 if (error)
3432                         return error;
3433         }
3434
3435         simple_offset_remove(shmem_get_offset_ctx(old_dir), old_dentry);
3436         error = simple_offset_add(shmem_get_offset_ctx(new_dir), old_dentry);
3437         if (error)
3438                 return error;
3439
3440         if (d_really_is_positive(new_dentry)) {
3441                 (void) shmem_unlink(new_dir, new_dentry);
3442                 if (they_are_dirs) {
3443                         drop_nlink(d_inode(new_dentry));
3444                         drop_nlink(old_dir);
3445                 }
3446         } else if (they_are_dirs) {
3447                 drop_nlink(old_dir);
3448                 inc_nlink(new_dir);
3449         }
3450
3451         old_dir->i_size -= BOGO_DIRENT_SIZE;
3452         new_dir->i_size += BOGO_DIRENT_SIZE;
3453         simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
3454         inode_inc_iversion(old_dir);
3455         inode_inc_iversion(new_dir);
3456         return 0;
3457 }
3458
3459 static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
3460                          struct dentry *dentry, const char *symname)
3461 {
3462         int error;
3463         int len;
3464         struct inode *inode;
3465         struct folio *folio;
3466
3467         len = strlen(symname) + 1;
3468         if (len > PAGE_SIZE)
3469                 return -ENAMETOOLONG;
3470
3471         inode = shmem_get_inode(idmap, dir->i_sb, dir, S_IFLNK | 0777, 0,
3472                                 VM_NORESERVE);
3473
3474         if (IS_ERR(inode))
3475                 return PTR_ERR(inode);
3476
3477         error = security_inode_init_security(inode, dir, &dentry->d_name,
3478                                              shmem_initxattrs, NULL);
3479         if (error && error != -EOPNOTSUPP)
3480                 goto out_iput;
3481
3482         error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3483         if (error)
3484                 goto out_iput;
3485
3486         inode->i_size = len-1;
3487         if (len <= SHORT_SYMLINK_LEN) {
3488                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3489                 if (!inode->i_link) {
3490                         error = -ENOMEM;
3491                         goto out_remove_offset;
3492                 }
3493                 inode->i_op = &shmem_short_symlink_operations;
3494         } else {
3495                 inode_nohighmem(inode);
3496                 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3497                 if (error)
3498                         goto out_remove_offset;
3499                 inode->i_mapping->a_ops = &shmem_aops;
3500                 inode->i_op = &shmem_symlink_inode_operations;
3501                 memcpy(folio_address(folio), symname, len);
3502                 folio_mark_uptodate(folio);
3503                 folio_mark_dirty(folio);
3504                 folio_unlock(folio);
3505                 folio_put(folio);
3506         }
3507         dir->i_size += BOGO_DIRENT_SIZE;
3508         dir->i_mtime = inode_set_ctime_current(dir);
3509         inode_inc_iversion(dir);
3510         d_instantiate(dentry, inode);
3511         dget(dentry);
3512         return 0;
3513
3514 out_remove_offset:
3515         simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
3516 out_iput:
3517         iput(inode);
3518         return error;
3519 }
3520
3521 static void shmem_put_link(void *arg)
3522 {
3523         folio_mark_accessed(arg);
3524         folio_put(arg);
3525 }
3526
3527 static const char *shmem_get_link(struct dentry *dentry,
3528                                   struct inode *inode,
3529                                   struct delayed_call *done)
3530 {
3531         struct folio *folio = NULL;
3532         int error;
3533
3534         if (!dentry) {
3535                 folio = filemap_get_folio(inode->i_mapping, 0);
3536                 if (IS_ERR(folio))
3537                         return ERR_PTR(-ECHILD);
3538                 if (PageHWPoison(folio_page(folio, 0)) ||
3539                     !folio_test_uptodate(folio)) {
3540                         folio_put(folio);
3541                         return ERR_PTR(-ECHILD);
3542                 }
3543         } else {
3544                 error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3545                 if (error)
3546                         return ERR_PTR(error);
3547                 if (!folio)
3548                         return ERR_PTR(-ECHILD);
3549                 if (PageHWPoison(folio_page(folio, 0))) {
3550                         folio_unlock(folio);
3551                         folio_put(folio);
3552                         return ERR_PTR(-ECHILD);
3553                 }
3554                 folio_unlock(folio);
3555         }
3556         set_delayed_call(done, shmem_put_link, folio);
3557         return folio_address(folio);
3558 }
3559
3560 #ifdef CONFIG_TMPFS_XATTR
3561
3562 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3563 {
3564         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3565
3566         fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3567
3568         return 0;
3569 }
3570
3571 static int shmem_fileattr_set(struct mnt_idmap *idmap,
3572                               struct dentry *dentry, struct fileattr *fa)
3573 {
3574         struct inode *inode = d_inode(dentry);
3575         struct shmem_inode_info *info = SHMEM_I(inode);
3576
3577         if (fileattr_has_fsx(fa))
3578                 return -EOPNOTSUPP;
3579         if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3580                 return -EOPNOTSUPP;
3581
3582         info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3583                 (fa->flags & SHMEM_FL_USER_MODIFIABLE);
3584
3585         shmem_set_inode_flags(inode, info->fsflags);
3586         inode_set_ctime_current(inode);
3587         inode_inc_iversion(inode);
3588         return 0;
3589 }
3590
3591 /*
3592  * Superblocks without xattr inode operations may get some security.* xattr
3593  * support from the LSM "for free". As soon as we have any other xattrs
3594  * like ACLs, we also need to implement the security.* handlers at
3595  * filesystem level, though.
3596  */
3597
3598 /*
3599  * Callback for security_inode_init_security() for acquiring xattrs.
3600  */
3601 static int shmem_initxattrs(struct inode *inode,
3602                             const struct xattr *xattr_array,
3603                             void *fs_info)
3604 {
3605         struct shmem_inode_info *info = SHMEM_I(inode);
3606         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3607         const struct xattr *xattr;
3608         struct simple_xattr *new_xattr;
3609         size_t ispace = 0;
3610         size_t len;
3611
3612         if (sbinfo->max_inodes) {
3613                 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3614                         ispace += simple_xattr_space(xattr->name,
3615                                 xattr->value_len + XATTR_SECURITY_PREFIX_LEN);
3616                 }
3617                 if (ispace) {
3618                         raw_spin_lock(&sbinfo->stat_lock);
3619                         if (sbinfo->free_ispace < ispace)
3620                                 ispace = 0;
3621                         else
3622                                 sbinfo->free_ispace -= ispace;
3623                         raw_spin_unlock(&sbinfo->stat_lock);
3624                         if (!ispace)
3625                                 return -ENOSPC;
3626                 }
3627         }
3628
3629         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3630                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3631                 if (!new_xattr)
3632                         break;
3633
3634                 len = strlen(xattr->name) + 1;
3635                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3636                                           GFP_KERNEL_ACCOUNT);
3637                 if (!new_xattr->name) {
3638                         kvfree(new_xattr);
3639                         break;
3640                 }
3641
3642                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3643                        XATTR_SECURITY_PREFIX_LEN);
3644                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3645                        xattr->name, len);
3646
3647                 simple_xattr_add(&info->xattrs, new_xattr);
3648         }
3649
3650         if (xattr->name != NULL) {
3651                 if (ispace) {
3652                         raw_spin_lock(&sbinfo->stat_lock);
3653                         sbinfo->free_ispace += ispace;
3654                         raw_spin_unlock(&sbinfo->stat_lock);
3655                 }
3656                 simple_xattrs_free(&info->xattrs, NULL);
3657                 return -ENOMEM;
3658         }
3659
3660         return 0;
3661 }
3662
3663 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3664                                    struct dentry *unused, struct inode *inode,
3665                                    const char *name, void *buffer, size_t size)
3666 {
3667         struct shmem_inode_info *info = SHMEM_I(inode);
3668
3669         name = xattr_full_name(handler, name);
3670         return simple_xattr_get(&info->xattrs, name, buffer, size);
3671 }
3672
3673 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3674                                    struct mnt_idmap *idmap,
3675                                    struct dentry *unused, struct inode *inode,
3676                                    const char *name, const void *value,
3677                                    size_t size, int flags)
3678 {
3679         struct shmem_inode_info *info = SHMEM_I(inode);
3680         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3681         struct simple_xattr *old_xattr;
3682         size_t ispace = 0;
3683
3684         name = xattr_full_name(handler, name);
3685         if (value && sbinfo->max_inodes) {
3686                 ispace = simple_xattr_space(name, size);
3687                 raw_spin_lock(&sbinfo->stat_lock);
3688                 if (sbinfo->free_ispace < ispace)
3689                         ispace = 0;
3690                 else
3691                         sbinfo->free_ispace -= ispace;
3692                 raw_spin_unlock(&sbinfo->stat_lock);
3693                 if (!ispace)
3694                         return -ENOSPC;
3695         }
3696
3697         old_xattr = simple_xattr_set(&info->xattrs, name, value, size, flags);
3698         if (!IS_ERR(old_xattr)) {
3699                 ispace = 0;
3700                 if (old_xattr && sbinfo->max_inodes)
3701                         ispace = simple_xattr_space(old_xattr->name,
3702                                                     old_xattr->size);
3703                 simple_xattr_free(old_xattr);
3704                 old_xattr = NULL;
3705                 inode_set_ctime_current(inode);
3706                 inode_inc_iversion(inode);
3707         }
3708         if (ispace) {
3709                 raw_spin_lock(&sbinfo->stat_lock);
3710                 sbinfo->free_ispace += ispace;
3711                 raw_spin_unlock(&sbinfo->stat_lock);
3712         }
3713         return PTR_ERR(old_xattr);
3714 }
3715
3716 static const struct xattr_handler shmem_security_xattr_handler = {
3717         .prefix = XATTR_SECURITY_PREFIX,
3718         .get = shmem_xattr_handler_get,
3719         .set = shmem_xattr_handler_set,
3720 };
3721
3722 static const struct xattr_handler shmem_trusted_xattr_handler = {
3723         .prefix = XATTR_TRUSTED_PREFIX,
3724         .get = shmem_xattr_handler_get,
3725         .set = shmem_xattr_handler_set,
3726 };
3727
3728 static const struct xattr_handler shmem_user_xattr_handler = {
3729         .prefix = XATTR_USER_PREFIX,
3730         .get = shmem_xattr_handler_get,
3731         .set = shmem_xattr_handler_set,
3732 };
3733
3734 static const struct xattr_handler *shmem_xattr_handlers[] = {
3735         &shmem_security_xattr_handler,
3736         &shmem_trusted_xattr_handler,
3737         &shmem_user_xattr_handler,
3738         NULL
3739 };
3740
3741 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3742 {
3743         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3744         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3745 }
3746 #endif /* CONFIG_TMPFS_XATTR */
3747
3748 static const struct inode_operations shmem_short_symlink_operations = {
3749         .getattr        = shmem_getattr,
3750         .setattr        = shmem_setattr,
3751         .get_link       = simple_get_link,
3752 #ifdef CONFIG_TMPFS_XATTR
3753         .listxattr      = shmem_listxattr,
3754 #endif
3755 };
3756
3757 static const struct inode_operations shmem_symlink_inode_operations = {
3758         .getattr        = shmem_getattr,
3759         .setattr        = shmem_setattr,
3760         .get_link       = shmem_get_link,
3761 #ifdef CONFIG_TMPFS_XATTR
3762         .listxattr      = shmem_listxattr,
3763 #endif
3764 };
3765
3766 static struct dentry *shmem_get_parent(struct dentry *child)
3767 {
3768         return ERR_PTR(-ESTALE);
3769 }
3770
3771 static int shmem_match(struct inode *ino, void *vfh)
3772 {
3773         __u32 *fh = vfh;
3774         __u64 inum = fh[2];
3775         inum = (inum << 32) | fh[1];
3776         return ino->i_ino == inum && fh[0] == ino->i_generation;
3777 }
3778
3779 /* Find any alias of inode, but prefer a hashed alias */
3780 static struct dentry *shmem_find_alias(struct inode *inode)
3781 {
3782         struct dentry *alias = d_find_alias(inode);
3783
3784         return alias ?: d_find_any_alias(inode);
3785 }
3786
3787
3788 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3789                 struct fid *fid, int fh_len, int fh_type)
3790 {
3791         struct inode *inode;
3792         struct dentry *dentry = NULL;
3793         u64 inum;
3794
3795         if (fh_len < 3)
3796                 return NULL;
3797
3798         inum = fid->raw[2];
3799         inum = (inum << 32) | fid->raw[1];
3800
3801         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3802                         shmem_match, fid->raw);
3803         if (inode) {
3804                 dentry = shmem_find_alias(inode);
3805                 iput(inode);
3806         }
3807
3808         return dentry;
3809 }
3810
3811 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3812                                 struct inode *parent)
3813 {
3814         if (*len < 3) {
3815                 *len = 3;
3816                 return FILEID_INVALID;
3817         }
3818
3819         if (inode_unhashed(inode)) {
3820                 /* Unfortunately insert_inode_hash is not idempotent,
3821                  * so as we hash inodes here rather than at creation
3822                  * time, we need a lock to ensure we only try
3823                  * to do it once
3824                  */
3825                 static DEFINE_SPINLOCK(lock);
3826                 spin_lock(&lock);
3827                 if (inode_unhashed(inode))
3828                         __insert_inode_hash(inode,
3829                                             inode->i_ino + inode->i_generation);
3830                 spin_unlock(&lock);
3831         }
3832
3833         fh[0] = inode->i_generation;
3834         fh[1] = inode->i_ino;
3835         fh[2] = ((__u64)inode->i_ino) >> 32;
3836
3837         *len = 3;
3838         return 1;
3839 }
3840
3841 static const struct export_operations shmem_export_ops = {
3842         .get_parent     = shmem_get_parent,
3843         .encode_fh      = shmem_encode_fh,
3844         .fh_to_dentry   = shmem_fh_to_dentry,
3845 };
3846
3847 enum shmem_param {
3848         Opt_gid,
3849         Opt_huge,
3850         Opt_mode,
3851         Opt_mpol,
3852         Opt_nr_blocks,
3853         Opt_nr_inodes,
3854         Opt_size,
3855         Opt_uid,
3856         Opt_inode32,
3857         Opt_inode64,
3858         Opt_noswap,
3859         Opt_quota,
3860         Opt_usrquota,
3861         Opt_grpquota,
3862         Opt_usrquota_block_hardlimit,
3863         Opt_usrquota_inode_hardlimit,
3864         Opt_grpquota_block_hardlimit,
3865         Opt_grpquota_inode_hardlimit,
3866 };
3867
3868 static const struct constant_table shmem_param_enums_huge[] = {
3869         {"never",       SHMEM_HUGE_NEVER },
3870         {"always",      SHMEM_HUGE_ALWAYS },
3871         {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3872         {"advise",      SHMEM_HUGE_ADVISE },
3873         {}
3874 };
3875
3876 const struct fs_parameter_spec shmem_fs_parameters[] = {
3877         fsparam_u32   ("gid",           Opt_gid),
3878         fsparam_enum  ("huge",          Opt_huge,  shmem_param_enums_huge),
3879         fsparam_u32oct("mode",          Opt_mode),
3880         fsparam_string("mpol",          Opt_mpol),
3881         fsparam_string("nr_blocks",     Opt_nr_blocks),
3882         fsparam_string("nr_inodes",     Opt_nr_inodes),
3883         fsparam_string("size",          Opt_size),
3884         fsparam_u32   ("uid",           Opt_uid),
3885         fsparam_flag  ("inode32",       Opt_inode32),
3886         fsparam_flag  ("inode64",       Opt_inode64),
3887         fsparam_flag  ("noswap",        Opt_noswap),
3888 #ifdef CONFIG_TMPFS_QUOTA
3889         fsparam_flag  ("quota",         Opt_quota),
3890         fsparam_flag  ("usrquota",      Opt_usrquota),
3891         fsparam_flag  ("grpquota",      Opt_grpquota),
3892         fsparam_string("usrquota_block_hardlimit", Opt_usrquota_block_hardlimit),
3893         fsparam_string("usrquota_inode_hardlimit", Opt_usrquota_inode_hardlimit),
3894         fsparam_string("grpquota_block_hardlimit", Opt_grpquota_block_hardlimit),
3895         fsparam_string("grpquota_inode_hardlimit", Opt_grpquota_inode_hardlimit),
3896 #endif
3897         {}
3898 };
3899
3900 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3901 {
3902         struct shmem_options *ctx = fc->fs_private;
3903         struct fs_parse_result result;
3904         unsigned long long size;
3905         char *rest;
3906         int opt;
3907         kuid_t kuid;
3908         kgid_t kgid;
3909
3910         opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3911         if (opt < 0)
3912                 return opt;
3913
3914         switch (opt) {
3915         case Opt_size:
3916                 size = memparse(param->string, &rest);
3917                 if (*rest == '%') {
3918                         size <<= PAGE_SHIFT;
3919                         size *= totalram_pages();
3920                         do_div(size, 100);
3921                         rest++;
3922                 }
3923                 if (*rest)
3924                         goto bad_value;
3925                 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3926                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3927                 break;
3928         case Opt_nr_blocks:
3929                 ctx->blocks = memparse(param->string, &rest);
3930                 if (*rest || ctx->blocks > LONG_MAX)
3931                         goto bad_value;
3932                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3933                 break;
3934         case Opt_nr_inodes:
3935                 ctx->inodes = memparse(param->string, &rest);
3936                 if (*rest || ctx->inodes > ULONG_MAX / BOGO_INODE_SIZE)
3937                         goto bad_value;
3938                 ctx->seen |= SHMEM_SEEN_INODES;
3939                 break;
3940         case Opt_mode:
3941                 ctx->mode = result.uint_32 & 07777;
3942                 break;
3943         case Opt_uid:
3944                 kuid = make_kuid(current_user_ns(), result.uint_32);
3945                 if (!uid_valid(kuid))
3946                         goto bad_value;
3947
3948                 /*
3949                  * The requested uid must be representable in the
3950                  * filesystem's idmapping.
3951                  */
3952                 if (!kuid_has_mapping(fc->user_ns, kuid))
3953                         goto bad_value;
3954
3955                 ctx->uid = kuid;
3956                 break;
3957         case Opt_gid:
3958                 kgid = make_kgid(current_user_ns(), result.uint_32);
3959                 if (!gid_valid(kgid))
3960                         goto bad_value;
3961
3962                 /*
3963                  * The requested gid must be representable in the
3964                  * filesystem's idmapping.
3965                  */
3966                 if (!kgid_has_mapping(fc->user_ns, kgid))
3967                         goto bad_value;
3968
3969                 ctx->gid = kgid;
3970                 break;
3971         case Opt_huge:
3972                 ctx->huge = result.uint_32;
3973                 if (ctx->huge != SHMEM_HUGE_NEVER &&
3974                     !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3975                       has_transparent_hugepage()))
3976                         goto unsupported_parameter;
3977                 ctx->seen |= SHMEM_SEEN_HUGE;
3978                 break;
3979         case Opt_mpol:
3980                 if (IS_ENABLED(CONFIG_NUMA)) {
3981                         mpol_put(ctx->mpol);
3982                         ctx->mpol = NULL;
3983                         if (mpol_parse_str(param->string, &ctx->mpol))
3984                                 goto bad_value;
3985                         break;
3986                 }
3987                 goto unsupported_parameter;
3988         case Opt_inode32:
3989                 ctx->full_inums = false;
3990                 ctx->seen |= SHMEM_SEEN_INUMS;
3991                 break;
3992         case Opt_inode64:
3993                 if (sizeof(ino_t) < 8) {
3994                         return invalfc(fc,
3995                                        "Cannot use inode64 with <64bit inums in kernel\n");
3996                 }
3997                 ctx->full_inums = true;
3998                 ctx->seen |= SHMEM_SEEN_INUMS;
3999                 break;
4000         case Opt_noswap:
4001                 if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN)) {
4002                         return invalfc(fc,
4003                                        "Turning off swap in unprivileged tmpfs mounts unsupported");
4004                 }
4005                 ctx->noswap = true;
4006                 ctx->seen |= SHMEM_SEEN_NOSWAP;
4007                 break;
4008         case Opt_quota:
4009                 if (fc->user_ns != &init_user_ns)
4010                         return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4011                 ctx->seen |= SHMEM_SEEN_QUOTA;
4012                 ctx->quota_types |= (QTYPE_MASK_USR | QTYPE_MASK_GRP);
4013                 break;
4014         case Opt_usrquota:
4015                 if (fc->user_ns != &init_user_ns)
4016                         return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4017                 ctx->seen |= SHMEM_SEEN_QUOTA;
4018                 ctx->quota_types |= QTYPE_MASK_USR;
4019                 break;
4020         case Opt_grpquota:
4021                 if (fc->user_ns != &init_user_ns)
4022                         return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4023                 ctx->seen |= SHMEM_SEEN_QUOTA;
4024                 ctx->quota_types |= QTYPE_MASK_GRP;
4025                 break;
4026         case Opt_usrquota_block_hardlimit:
4027                 size = memparse(param->string, &rest);
4028                 if (*rest || !size)
4029                         goto bad_value;
4030                 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4031                         return invalfc(fc,
4032                                        "User quota block hardlimit too large.");
4033                 ctx->qlimits.usrquota_bhardlimit = size;
4034                 break;
4035         case Opt_grpquota_block_hardlimit:
4036                 size = memparse(param->string, &rest);
4037                 if (*rest || !size)
4038                         goto bad_value;
4039                 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4040                         return invalfc(fc,
4041                                        "Group quota block hardlimit too large.");
4042                 ctx->qlimits.grpquota_bhardlimit = size;
4043                 break;
4044         case Opt_usrquota_inode_hardlimit:
4045                 size = memparse(param->string, &rest);
4046                 if (*rest || !size)
4047                         goto bad_value;
4048                 if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4049                         return invalfc(fc,
4050                                        "User quota inode hardlimit too large.");
4051                 ctx->qlimits.usrquota_ihardlimit = size;
4052                 break;
4053         case Opt_grpquota_inode_hardlimit:
4054                 size = memparse(param->string, &rest);
4055                 if (*rest || !size)
4056                         goto bad_value;
4057                 if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4058                         return invalfc(fc,
4059                                        "Group quota inode hardlimit too large.");
4060                 ctx->qlimits.grpquota_ihardlimit = size;
4061                 break;
4062         }
4063         return 0;
4064
4065 unsupported_parameter:
4066         return invalfc(fc, "Unsupported parameter '%s'", param->key);
4067 bad_value:
4068         return invalfc(fc, "Bad value for '%s'", param->key);
4069 }
4070
4071 static int shmem_parse_options(struct fs_context *fc, void *data)
4072 {
4073         char *options = data;
4074
4075         if (options) {
4076                 int err = security_sb_eat_lsm_opts(options, &fc->security);
4077                 if (err)
4078                         return err;
4079         }
4080
4081         while (options != NULL) {
4082                 char *this_char = options;
4083                 for (;;) {
4084                         /*
4085                          * NUL-terminate this option: unfortunately,
4086                          * mount options form a comma-separated list,
4087                          * but mpol's nodelist may also contain commas.
4088                          */
4089                         options = strchr(options, ',');
4090                         if (options == NULL)
4091                                 break;
4092                         options++;
4093                         if (!isdigit(*options)) {
4094                                 options[-1] = '\0';
4095                                 break;
4096                         }
4097                 }
4098                 if (*this_char) {
4099                         char *value = strchr(this_char, '=');
4100                         size_t len = 0;
4101                         int err;
4102
4103                         if (value) {
4104                                 *value++ = '\0';
4105                                 len = strlen(value);
4106                         }
4107                         err = vfs_parse_fs_string(fc, this_char, value, len);
4108                         if (err < 0)
4109                                 return err;
4110                 }
4111         }
4112         return 0;
4113 }
4114
4115 /*
4116  * Reconfigure a shmem filesystem.
4117  */
4118 static int shmem_reconfigure(struct fs_context *fc)
4119 {
4120         struct shmem_options *ctx = fc->fs_private;
4121         struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
4122         unsigned long used_isp;
4123         struct mempolicy *mpol = NULL;
4124         const char *err;
4125
4126         raw_spin_lock(&sbinfo->stat_lock);
4127         used_isp = sbinfo->max_inodes * BOGO_INODE_SIZE - sbinfo->free_ispace;
4128
4129         if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
4130                 if (!sbinfo->max_blocks) {
4131                         err = "Cannot retroactively limit size";
4132                         goto out;
4133                 }
4134                 if (percpu_counter_compare(&sbinfo->used_blocks,
4135                                            ctx->blocks) > 0) {
4136                         err = "Too small a size for current use";
4137                         goto out;
4138                 }
4139         }
4140         if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
4141                 if (!sbinfo->max_inodes) {
4142                         err = "Cannot retroactively limit inodes";
4143                         goto out;
4144                 }
4145                 if (ctx->inodes * BOGO_INODE_SIZE < used_isp) {
4146                         err = "Too few inodes for current use";
4147                         goto out;
4148                 }
4149         }
4150
4151         if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
4152             sbinfo->next_ino > UINT_MAX) {
4153                 err = "Current inum too high to switch to 32-bit inums";
4154                 goto out;
4155         }
4156         if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) {
4157                 err = "Cannot disable swap on remount";
4158                 goto out;
4159         }
4160         if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) {
4161                 err = "Cannot enable swap on remount if it was disabled on first mount";
4162                 goto out;
4163         }
4164
4165         if (ctx->seen & SHMEM_SEEN_QUOTA &&
4166             !sb_any_quota_loaded(fc->root->d_sb)) {
4167                 err = "Cannot enable quota on remount";
4168                 goto out;
4169         }
4170
4171 #ifdef CONFIG_TMPFS_QUOTA
4172 #define CHANGED_LIMIT(name)                                             \
4173         (ctx->qlimits.name## hardlimit &&                               \
4174         (ctx->qlimits.name## hardlimit != sbinfo->qlimits.name## hardlimit))
4175
4176         if (CHANGED_LIMIT(usrquota_b) || CHANGED_LIMIT(usrquota_i) ||
4177             CHANGED_LIMIT(grpquota_b) || CHANGED_LIMIT(grpquota_i)) {
4178                 err = "Cannot change global quota limit on remount";
4179                 goto out;
4180         }
4181 #endif /* CONFIG_TMPFS_QUOTA */
4182
4183         if (ctx->seen & SHMEM_SEEN_HUGE)
4184                 sbinfo->huge = ctx->huge;
4185         if (ctx->seen & SHMEM_SEEN_INUMS)
4186                 sbinfo->full_inums = ctx->full_inums;
4187         if (ctx->seen & SHMEM_SEEN_BLOCKS)
4188                 sbinfo->max_blocks  = ctx->blocks;
4189         if (ctx->seen & SHMEM_SEEN_INODES) {
4190                 sbinfo->max_inodes  = ctx->inodes;
4191                 sbinfo->free_ispace = ctx->inodes * BOGO_INODE_SIZE - used_isp;
4192         }
4193
4194         /*
4195          * Preserve previous mempolicy unless mpol remount option was specified.
4196          */
4197         if (ctx->mpol) {
4198                 mpol = sbinfo->mpol;
4199                 sbinfo->mpol = ctx->mpol;       /* transfers initial ref */
4200                 ctx->mpol = NULL;
4201         }
4202
4203         if (ctx->noswap)
4204                 sbinfo->noswap = true;
4205
4206         raw_spin_unlock(&sbinfo->stat_lock);
4207         mpol_put(mpol);
4208         return 0;
4209 out:
4210         raw_spin_unlock(&sbinfo->stat_lock);
4211         return invalfc(fc, "%s", err);
4212 }
4213
4214 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
4215 {
4216         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
4217         struct mempolicy *mpol;
4218
4219         if (sbinfo->max_blocks != shmem_default_max_blocks())
4220                 seq_printf(seq, ",size=%luk", K(sbinfo->max_blocks));
4221         if (sbinfo->max_inodes != shmem_default_max_inodes())
4222                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
4223         if (sbinfo->mode != (0777 | S_ISVTX))
4224                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
4225         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
4226                 seq_printf(seq, ",uid=%u",
4227                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
4228         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
4229                 seq_printf(seq, ",gid=%u",
4230                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
4231
4232         /*
4233          * Showing inode{64,32} might be useful even if it's the system default,
4234          * since then people don't have to resort to checking both here and
4235          * /proc/config.gz to confirm 64-bit inums were successfully applied
4236          * (which may not even exist if IKCONFIG_PROC isn't enabled).
4237          *
4238          * We hide it when inode64 isn't the default and we are using 32-bit
4239          * inodes, since that probably just means the feature isn't even under
4240          * consideration.
4241          *
4242          * As such:
4243          *
4244          *                     +-----------------+-----------------+
4245          *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
4246          *  +------------------+-----------------+-----------------+
4247          *  | full_inums=true  | show            | show            |
4248          *  | full_inums=false | show            | hide            |
4249          *  +------------------+-----------------+-----------------+
4250          *
4251          */
4252         if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
4253                 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
4254 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4255         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
4256         if (sbinfo->huge)
4257                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
4258 #endif
4259         mpol = shmem_get_sbmpol(sbinfo);
4260         shmem_show_mpol(seq, mpol);
4261         mpol_put(mpol);
4262         if (sbinfo->noswap)
4263                 seq_printf(seq, ",noswap");
4264         return 0;
4265 }
4266
4267 #endif /* CONFIG_TMPFS */
4268
4269 static void shmem_put_super(struct super_block *sb)
4270 {
4271         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
4272
4273 #ifdef CONFIG_TMPFS_QUOTA
4274         shmem_disable_quotas(sb);
4275 #endif
4276         free_percpu(sbinfo->ino_batch);
4277         percpu_counter_destroy(&sbinfo->used_blocks);
4278         mpol_put(sbinfo->mpol);
4279         kfree(sbinfo);
4280         sb->s_fs_info = NULL;
4281 }
4282
4283 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
4284 {
4285         struct shmem_options *ctx = fc->fs_private;
4286         struct inode *inode;
4287         struct shmem_sb_info *sbinfo;
4288         int error = -ENOMEM;
4289
4290         /* Round up to L1_CACHE_BYTES to resist false sharing */
4291         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
4292                                 L1_CACHE_BYTES), GFP_KERNEL);
4293         if (!sbinfo)
4294                 return error;
4295
4296         sb->s_fs_info = sbinfo;
4297
4298 #ifdef CONFIG_TMPFS
4299         /*
4300          * Per default we only allow half of the physical ram per
4301          * tmpfs instance, limiting inodes to one per page of lowmem;
4302          * but the internal instance is left unlimited.
4303          */
4304         if (!(sb->s_flags & SB_KERNMOUNT)) {
4305                 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
4306                         ctx->blocks = shmem_default_max_blocks();
4307                 if (!(ctx->seen & SHMEM_SEEN_INODES))
4308                         ctx->inodes = shmem_default_max_inodes();
4309                 if (!(ctx->seen & SHMEM_SEEN_INUMS))
4310                         ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
4311                 sbinfo->noswap = ctx->noswap;
4312         } else {
4313                 sb->s_flags |= SB_NOUSER;
4314         }
4315         sb->s_export_op = &shmem_export_ops;
4316         sb->s_flags |= SB_NOSEC | SB_I_VERSION;
4317 #else
4318         sb->s_flags |= SB_NOUSER;
4319 #endif
4320         sbinfo->max_blocks = ctx->blocks;
4321         sbinfo->max_inodes = ctx->inodes;
4322         sbinfo->free_ispace = sbinfo->max_inodes * BOGO_INODE_SIZE;
4323         if (sb->s_flags & SB_KERNMOUNT) {
4324                 sbinfo->ino_batch = alloc_percpu(ino_t);
4325                 if (!sbinfo->ino_batch)
4326                         goto failed;
4327         }
4328         sbinfo->uid = ctx->uid;
4329         sbinfo->gid = ctx->gid;
4330         sbinfo->full_inums = ctx->full_inums;
4331         sbinfo->mode = ctx->mode;
4332         sbinfo->huge = ctx->huge;
4333         sbinfo->mpol = ctx->mpol;
4334         ctx->mpol = NULL;
4335
4336         raw_spin_lock_init(&sbinfo->stat_lock);
4337         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
4338                 goto failed;
4339         spin_lock_init(&sbinfo->shrinklist_lock);
4340         INIT_LIST_HEAD(&sbinfo->shrinklist);
4341
4342         sb->s_maxbytes = MAX_LFS_FILESIZE;
4343         sb->s_blocksize = PAGE_SIZE;
4344         sb->s_blocksize_bits = PAGE_SHIFT;
4345         sb->s_magic = TMPFS_MAGIC;
4346         sb->s_op = &shmem_ops;
4347         sb->s_time_gran = 1;
4348 #ifdef CONFIG_TMPFS_XATTR
4349         sb->s_xattr = shmem_xattr_handlers;
4350 #endif
4351 #ifdef CONFIG_TMPFS_POSIX_ACL
4352         sb->s_flags |= SB_POSIXACL;
4353 #endif
4354         uuid_gen(&sb->s_uuid);
4355
4356 #ifdef CONFIG_TMPFS_QUOTA
4357         if (ctx->seen & SHMEM_SEEN_QUOTA) {
4358                 sb->dq_op = &shmem_quota_operations;
4359                 sb->s_qcop = &dquot_quotactl_sysfile_ops;
4360                 sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP;
4361
4362                 /* Copy the default limits from ctx into sbinfo */
4363                 memcpy(&sbinfo->qlimits, &ctx->qlimits,
4364                        sizeof(struct shmem_quota_limits));
4365
4366                 if (shmem_enable_quotas(sb, ctx->quota_types))
4367                         goto failed;
4368         }
4369 #endif /* CONFIG_TMPFS_QUOTA */
4370
4371         inode = shmem_get_inode(&nop_mnt_idmap, sb, NULL, S_IFDIR | sbinfo->mode, 0,
4372                                 VM_NORESERVE);
4373         if (IS_ERR(inode)) {
4374                 error = PTR_ERR(inode);
4375                 goto failed;
4376         }
4377         inode->i_uid = sbinfo->uid;
4378         inode->i_gid = sbinfo->gid;
4379         sb->s_root = d_make_root(inode);
4380         if (!sb->s_root)
4381                 goto failed;
4382         return 0;
4383
4384 failed:
4385         shmem_put_super(sb);
4386         return error;
4387 }
4388
4389 static int shmem_get_tree(struct fs_context *fc)
4390 {
4391         return get_tree_nodev(fc, shmem_fill_super);
4392 }
4393
4394 static void shmem_free_fc(struct fs_context *fc)
4395 {
4396         struct shmem_options *ctx = fc->fs_private;
4397
4398         if (ctx) {
4399                 mpol_put(ctx->mpol);
4400                 kfree(ctx);
4401         }
4402 }
4403
4404 static const struct fs_context_operations shmem_fs_context_ops = {
4405         .free                   = shmem_free_fc,
4406         .get_tree               = shmem_get_tree,
4407 #ifdef CONFIG_TMPFS
4408         .parse_monolithic       = shmem_parse_options,
4409         .parse_param            = shmem_parse_one,
4410         .reconfigure            = shmem_reconfigure,
4411 #endif
4412 };
4413
4414 static struct kmem_cache *shmem_inode_cachep;
4415
4416 static struct inode *shmem_alloc_inode(struct super_block *sb)
4417 {
4418         struct shmem_inode_info *info;
4419         info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
4420         if (!info)
4421                 return NULL;
4422         return &info->vfs_inode;
4423 }
4424
4425 static void shmem_free_in_core_inode(struct inode *inode)
4426 {
4427         if (S_ISLNK(inode->i_mode))
4428                 kfree(inode->i_link);
4429         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
4430 }
4431
4432 static void shmem_destroy_inode(struct inode *inode)
4433 {
4434         if (S_ISREG(inode->i_mode))
4435                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
4436         if (S_ISDIR(inode->i_mode))
4437                 simple_offset_destroy(shmem_get_offset_ctx(inode));
4438 }
4439
4440 static void shmem_init_inode(void *foo)
4441 {
4442         struct shmem_inode_info *info = foo;
4443         inode_init_once(&info->vfs_inode);
4444 }
4445
4446 static void shmem_init_inodecache(void)
4447 {
4448         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
4449                                 sizeof(struct shmem_inode_info),
4450                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
4451 }
4452
4453 static void shmem_destroy_inodecache(void)
4454 {
4455         kmem_cache_destroy(shmem_inode_cachep);
4456 }
4457
4458 /* Keep the page in page cache instead of truncating it */
4459 static int shmem_error_remove_page(struct address_space *mapping,
4460                                    struct page *page)
4461 {
4462         return 0;
4463 }
4464
4465 const struct address_space_operations shmem_aops = {
4466         .writepage      = shmem_writepage,
4467         .dirty_folio    = noop_dirty_folio,
4468 #ifdef CONFIG_TMPFS
4469         .write_begin    = shmem_write_begin,
4470         .write_end      = shmem_write_end,
4471 #endif
4472 #ifdef CONFIG_MIGRATION
4473         .migrate_folio  = migrate_folio,
4474 #endif
4475         .error_remove_page = shmem_error_remove_page,
4476 };
4477 EXPORT_SYMBOL(shmem_aops);
4478
4479 static const struct file_operations shmem_file_operations = {
4480         .mmap           = shmem_mmap,
4481         .open           = shmem_file_open,
4482         .get_unmapped_area = shmem_get_unmapped_area,
4483 #ifdef CONFIG_TMPFS
4484         .llseek         = shmem_file_llseek,
4485         .read_iter      = shmem_file_read_iter,
4486         .write_iter     = shmem_file_write_iter,
4487         .fsync          = noop_fsync,
4488         .splice_read    = shmem_file_splice_read,
4489         .splice_write   = iter_file_splice_write,
4490         .fallocate      = shmem_fallocate,
4491 #endif
4492 };
4493
4494 static const struct inode_operations shmem_inode_operations = {
4495         .getattr        = shmem_getattr,
4496         .setattr        = shmem_setattr,
4497 #ifdef CONFIG_TMPFS_XATTR
4498         .listxattr      = shmem_listxattr,
4499         .set_acl        = simple_set_acl,
4500         .fileattr_get   = shmem_fileattr_get,
4501         .fileattr_set   = shmem_fileattr_set,
4502 #endif
4503 };
4504
4505 static const struct inode_operations shmem_dir_inode_operations = {
4506 #ifdef CONFIG_TMPFS
4507         .getattr        = shmem_getattr,
4508         .create         = shmem_create,
4509         .lookup         = simple_lookup,
4510         .link           = shmem_link,
4511         .unlink         = shmem_unlink,
4512         .symlink        = shmem_symlink,
4513         .mkdir          = shmem_mkdir,
4514         .rmdir          = shmem_rmdir,
4515         .mknod          = shmem_mknod,
4516         .rename         = shmem_rename2,
4517         .tmpfile        = shmem_tmpfile,
4518         .get_offset_ctx = shmem_get_offset_ctx,
4519 #endif
4520 #ifdef CONFIG_TMPFS_XATTR
4521         .listxattr      = shmem_listxattr,
4522         .fileattr_get   = shmem_fileattr_get,
4523         .fileattr_set   = shmem_fileattr_set,
4524 #endif
4525 #ifdef CONFIG_TMPFS_POSIX_ACL
4526         .setattr        = shmem_setattr,
4527         .set_acl        = simple_set_acl,
4528 #endif
4529 };
4530
4531 static const struct inode_operations shmem_special_inode_operations = {
4532         .getattr        = shmem_getattr,
4533 #ifdef CONFIG_TMPFS_XATTR
4534         .listxattr      = shmem_listxattr,
4535 #endif
4536 #ifdef CONFIG_TMPFS_POSIX_ACL
4537         .setattr        = shmem_setattr,
4538         .set_acl        = simple_set_acl,
4539 #endif
4540 };
4541
4542 static const struct super_operations shmem_ops = {
4543         .alloc_inode    = shmem_alloc_inode,
4544         .free_inode     = shmem_free_in_core_inode,
4545         .destroy_inode  = shmem_destroy_inode,
4546 #ifdef CONFIG_TMPFS
4547         .statfs         = shmem_statfs,
4548         .show_options   = shmem_show_options,
4549 #endif
4550 #ifdef CONFIG_TMPFS_QUOTA
4551         .get_dquots     = shmem_get_dquots,
4552 #endif
4553         .evict_inode    = shmem_evict_inode,
4554         .drop_inode     = generic_delete_inode,
4555         .put_super      = shmem_put_super,
4556 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4557         .nr_cached_objects      = shmem_unused_huge_count,
4558         .free_cached_objects    = shmem_unused_huge_scan,
4559 #endif
4560 };
4561
4562 static const struct vm_operations_struct shmem_vm_ops = {
4563         .fault          = shmem_fault,
4564         .map_pages      = filemap_map_pages,
4565 #ifdef CONFIG_NUMA
4566         .set_policy     = shmem_set_policy,
4567         .get_policy     = shmem_get_policy,
4568 #endif
4569 };
4570
4571 static const struct vm_operations_struct shmem_anon_vm_ops = {
4572         .fault          = shmem_fault,
4573         .map_pages      = filemap_map_pages,
4574 #ifdef CONFIG_NUMA
4575         .set_policy     = shmem_set_policy,
4576         .get_policy     = shmem_get_policy,
4577 #endif
4578 };
4579
4580 int shmem_init_fs_context(struct fs_context *fc)
4581 {
4582         struct shmem_options *ctx;
4583
4584         ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
4585         if (!ctx)
4586                 return -ENOMEM;
4587
4588         ctx->mode = 0777 | S_ISVTX;
4589         ctx->uid = current_fsuid();
4590         ctx->gid = current_fsgid();
4591
4592         fc->fs_private = ctx;
4593         fc->ops = &shmem_fs_context_ops;
4594         return 0;
4595 }
4596
4597 static struct file_system_type shmem_fs_type = {
4598         .owner          = THIS_MODULE,
4599         .name           = "tmpfs",
4600         .init_fs_context = shmem_init_fs_context,
4601 #ifdef CONFIG_TMPFS
4602         .parameters     = shmem_fs_parameters,
4603 #endif
4604         .kill_sb        = kill_litter_super,
4605 #ifdef CONFIG_SHMEM
4606         .fs_flags       = FS_USERNS_MOUNT | FS_ALLOW_IDMAP,
4607 #else
4608         .fs_flags       = FS_USERNS_MOUNT,
4609 #endif
4610 };
4611
4612 void __init shmem_init(void)
4613 {
4614         int error;
4615
4616         shmem_init_inodecache();
4617
4618 #ifdef CONFIG_TMPFS_QUOTA
4619         error = register_quota_format(&shmem_quota_format);
4620         if (error < 0) {
4621                 pr_err("Could not register quota format\n");
4622                 goto out3;
4623         }
4624 #endif
4625
4626         error = register_filesystem(&shmem_fs_type);
4627         if (error) {
4628                 pr_err("Could not register tmpfs\n");
4629                 goto out2;
4630         }
4631
4632         shm_mnt = kern_mount(&shmem_fs_type);
4633         if (IS_ERR(shm_mnt)) {
4634                 error = PTR_ERR(shm_mnt);
4635                 pr_err("Could not kern_mount tmpfs\n");
4636                 goto out1;
4637         }
4638
4639 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4640         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4641                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4642         else
4643                 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4644 #endif
4645         return;
4646
4647 out1:
4648         unregister_filesystem(&shmem_fs_type);
4649 out2:
4650 #ifdef CONFIG_TMPFS_QUOTA
4651         unregister_quota_format(&shmem_quota_format);
4652 out3:
4653 #endif
4654         shmem_destroy_inodecache();
4655         shm_mnt = ERR_PTR(error);
4656 }
4657
4658 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
4659 static ssize_t shmem_enabled_show(struct kobject *kobj,
4660                                   struct kobj_attribute *attr, char *buf)
4661 {
4662         static const int values[] = {
4663                 SHMEM_HUGE_ALWAYS,
4664                 SHMEM_HUGE_WITHIN_SIZE,
4665                 SHMEM_HUGE_ADVISE,
4666                 SHMEM_HUGE_NEVER,
4667                 SHMEM_HUGE_DENY,
4668                 SHMEM_HUGE_FORCE,
4669         };
4670         int len = 0;
4671         int i;
4672
4673         for (i = 0; i < ARRAY_SIZE(values); i++) {
4674                 len += sysfs_emit_at(buf, len,
4675                                      shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4676                                      i ? " " : "",
4677                                      shmem_format_huge(values[i]));
4678         }
4679
4680         len += sysfs_emit_at(buf, len, "\n");
4681
4682         return len;
4683 }
4684
4685 static ssize_t shmem_enabled_store(struct kobject *kobj,
4686                 struct kobj_attribute *attr, const char *buf, size_t count)
4687 {
4688         char tmp[16];
4689         int huge;
4690
4691         if (count + 1 > sizeof(tmp))
4692                 return -EINVAL;
4693         memcpy(tmp, buf, count);
4694         tmp[count] = '\0';
4695         if (count && tmp[count - 1] == '\n')
4696                 tmp[count - 1] = '\0';
4697
4698         huge = shmem_parse_huge(tmp);
4699         if (huge == -EINVAL)
4700                 return -EINVAL;
4701         if (!has_transparent_hugepage() &&
4702                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4703                 return -EINVAL;
4704
4705         shmem_huge = huge;
4706         if (shmem_huge > SHMEM_HUGE_DENY)
4707                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4708         return count;
4709 }
4710
4711 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4712 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4713
4714 #else /* !CONFIG_SHMEM */
4715
4716 /*
4717  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4718  *
4719  * This is intended for small system where the benefits of the full
4720  * shmem code (swap-backed and resource-limited) are outweighed by
4721  * their complexity. On systems without swap this code should be
4722  * effectively equivalent, but much lighter weight.
4723  */
4724
4725 static struct file_system_type shmem_fs_type = {
4726         .name           = "tmpfs",
4727         .init_fs_context = ramfs_init_fs_context,
4728         .parameters     = ramfs_fs_parameters,
4729         .kill_sb        = ramfs_kill_sb,
4730         .fs_flags       = FS_USERNS_MOUNT,
4731 };
4732
4733 void __init shmem_init(void)
4734 {
4735         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4736
4737         shm_mnt = kern_mount(&shmem_fs_type);
4738         BUG_ON(IS_ERR(shm_mnt));
4739 }
4740
4741 int shmem_unuse(unsigned int type)
4742 {
4743         return 0;
4744 }
4745
4746 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4747 {
4748         return 0;
4749 }
4750
4751 void shmem_unlock_mapping(struct address_space *mapping)
4752 {
4753 }
4754
4755 #ifdef CONFIG_MMU
4756 unsigned long shmem_get_unmapped_area(struct file *file,
4757                                       unsigned long addr, unsigned long len,
4758                                       unsigned long pgoff, unsigned long flags)
4759 {
4760         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4761 }
4762 #endif
4763
4764 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4765 {
4766         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4767 }
4768 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4769
4770 #define shmem_vm_ops                            generic_file_vm_ops
4771 #define shmem_anon_vm_ops                       generic_file_vm_ops
4772 #define shmem_file_operations                   ramfs_file_operations
4773 #define shmem_acct_size(flags, size)            0
4774 #define shmem_unacct_size(flags, size)          do {} while (0)
4775
4776 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap, struct super_block *sb, struct inode *dir,
4777                                             umode_t mode, dev_t dev, unsigned long flags)
4778 {
4779         struct inode *inode = ramfs_get_inode(sb, dir, mode, dev);
4780         return inode ? inode : ERR_PTR(-ENOSPC);
4781 }
4782
4783 #endif /* CONFIG_SHMEM */
4784
4785 /* common code */
4786
4787 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4788                                        unsigned long flags, unsigned int i_flags)
4789 {
4790         struct inode *inode;
4791         struct file *res;
4792
4793         if (IS_ERR(mnt))
4794                 return ERR_CAST(mnt);
4795
4796         if (size < 0 || size > MAX_LFS_FILESIZE)
4797                 return ERR_PTR(-EINVAL);
4798
4799         if (shmem_acct_size(flags, size))
4800                 return ERR_PTR(-ENOMEM);
4801
4802         if (is_idmapped_mnt(mnt))
4803                 return ERR_PTR(-EINVAL);
4804
4805         inode = shmem_get_inode(&nop_mnt_idmap, mnt->mnt_sb, NULL,
4806                                 S_IFREG | S_IRWXUGO, 0, flags);
4807
4808         if (IS_ERR(inode)) {
4809                 shmem_unacct_size(flags, size);
4810                 return ERR_CAST(inode);
4811         }
4812         inode->i_flags |= i_flags;
4813         inode->i_size = size;
4814         clear_nlink(inode);     /* It is unlinked */
4815         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4816         if (!IS_ERR(res))
4817                 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4818                                 &shmem_file_operations);
4819         if (IS_ERR(res))
4820                 iput(inode);
4821         return res;
4822 }
4823
4824 /**
4825  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4826  *      kernel internal.  There will be NO LSM permission checks against the
4827  *      underlying inode.  So users of this interface must do LSM checks at a
4828  *      higher layer.  The users are the big_key and shm implementations.  LSM
4829  *      checks are provided at the key or shm level rather than the inode.
4830  * @name: name for dentry (to be seen in /proc/<pid>/maps
4831  * @size: size to be set for the file
4832  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4833  */
4834 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4835 {
4836         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4837 }
4838
4839 /**
4840  * shmem_file_setup - get an unlinked file living in tmpfs
4841  * @name: name for dentry (to be seen in /proc/<pid>/maps
4842  * @size: size to be set for the file
4843  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4844  */
4845 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4846 {
4847         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4848 }
4849 EXPORT_SYMBOL_GPL(shmem_file_setup);
4850
4851 /**
4852  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4853  * @mnt: the tmpfs mount where the file will be created
4854  * @name: name for dentry (to be seen in /proc/<pid>/maps
4855  * @size: size to be set for the file
4856  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4857  */
4858 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4859                                        loff_t size, unsigned long flags)
4860 {
4861         return __shmem_file_setup(mnt, name, size, flags, 0);
4862 }
4863 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4864
4865 /**
4866  * shmem_zero_setup - setup a shared anonymous mapping
4867  * @vma: the vma to be mmapped is prepared by do_mmap
4868  */
4869 int shmem_zero_setup(struct vm_area_struct *vma)
4870 {
4871         struct file *file;
4872         loff_t size = vma->vm_end - vma->vm_start;
4873
4874         /*
4875          * Cloning a new file under mmap_lock leads to a lock ordering conflict
4876          * between XFS directory reading and selinux: since this file is only
4877          * accessible to the user through its mapping, use S_PRIVATE flag to
4878          * bypass file security, in the same way as shmem_kernel_file_setup().
4879          */
4880         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4881         if (IS_ERR(file))
4882                 return PTR_ERR(file);
4883
4884         if (vma->vm_file)
4885                 fput(vma->vm_file);
4886         vma->vm_file = file;
4887         vma->vm_ops = &shmem_anon_vm_ops;
4888
4889         return 0;
4890 }
4891
4892 /**
4893  * shmem_read_folio_gfp - read into page cache, using specified page allocation flags.
4894  * @mapping:    the folio's address_space
4895  * @index:      the folio index
4896  * @gfp:        the page allocator flags to use if allocating
4897  *
4898  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4899  * with any new page allocations done using the specified allocation flags.
4900  * But read_cache_page_gfp() uses the ->read_folio() method: which does not
4901  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4902  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4903  *
4904  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4905  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4906  */
4907 struct folio *shmem_read_folio_gfp(struct address_space *mapping,
4908                 pgoff_t index, gfp_t gfp)
4909 {
4910 #ifdef CONFIG_SHMEM
4911         struct inode *inode = mapping->host;
4912         struct folio *folio;
4913         int error;
4914
4915         BUG_ON(!shmem_mapping(mapping));
4916         error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
4917                                   gfp, NULL, NULL, NULL);
4918         if (error)
4919                 return ERR_PTR(error);
4920
4921         folio_unlock(folio);
4922         return folio;
4923 #else
4924         /*
4925          * The tiny !SHMEM case uses ramfs without swap
4926          */
4927         return mapping_read_folio_gfp(mapping, index, gfp);
4928 #endif
4929 }
4930 EXPORT_SYMBOL_GPL(shmem_read_folio_gfp);
4931
4932 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4933                                          pgoff_t index, gfp_t gfp)
4934 {
4935         struct folio *folio = shmem_read_folio_gfp(mapping, index, gfp);
4936         struct page *page;
4937
4938         if (IS_ERR(folio))
4939                 return &folio->page;
4940
4941         page = folio_file_page(folio, index);
4942         if (PageHWPoison(page)) {
4943                 folio_put(folio);
4944                 return ERR_PTR(-EIO);
4945         }
4946
4947         return page;
4948 }
4949 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);