lib/iov_iter.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <crypto/hash.h>
   3 #include <linux/export.h>
   4 #include <linux/bvec.h>
   5 #include <linux/fault-inject-usercopy.h>
   6 #include <linux/uio.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/highmem.h>
   9 #include <linux/slab.h>
  10 #include <linux/vmalloc.h>
  11 #include <linux/splice.h>
  12 #include <linux/compat.h>
  13 #include <net/checksum.h>
  14 #include <linux/scatterlist.h>
  15 #include <linux/instrumented.h>
  16
  17 /* covers ubuf and kbuf alike */
  18 #define iterate_buf(i, n, base, len, off, __p, STEP) {          \
  19         size_t __maybe_unused off = 0;                          \
  20         len = n;                                                \
  21         base = __p + i->iov_offset;                             \
  22         len -= (STEP);                                          \
  23         i->iov_offset += len;                                   \
  24         n = len;                                                \
  25 }
  26
  27 /* covers iovec and kvec alike */
  28 #define iterate_iovec(i, n, base, len, off, __p, STEP) {        \
  29         size_t off = 0;                                         \
  30         size_t skip = i->iov_offset;                            \
  31         do {                                                    \
  32                 len = min(n, __p->iov_len - skip);              \
  33                 if (likely(len)) {                              \
  34                         base = __p->iov_base + skip;            \
  35                         len -= (STEP);                          \
  36                         off += len;                             \
  37                         skip += len;                            \
  38                         n -= len;                               \
  39                         if (skip < __p->iov_len)                \
  40                                 break;                          \
  41                 }                                               \
  42                 __p++;                                          \
  43                 skip = 0;                                       \
  44         } while (n);                                            \
  45         i->iov_offset = skip;                                   \
  46         n = off;                                                \
  47 }
  48
  49 #define iterate_bvec(i, n, base, len, off, p, STEP) {           \
  50         size_t off = 0;                                         \
  51         unsigned skip = i->iov_offset;                          \
  52         while (n) {                                             \
  53                 unsigned offset = p->bv_offset + skip;          \
  54                 unsigned left;                                  \
  55                 void *kaddr = kmap_local_page(p->bv_page +      \
  56                                         offset / PAGE_SIZE);    \
  57                 base = kaddr + offset % PAGE_SIZE;              \
  58                 len = min(min(n, (size_t)(p->bv_len - skip)),   \
  59                      (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \
  60                 left = (STEP);                                  \
  61                 kunmap_local(kaddr);                            \
  62                 len -= left;                                    \
  63                 off += len;                                     \
  64                 skip += len;                                    \
  65                 if (skip == p->bv_len) {                        \
  66                         skip = 0;                               \
  67                         p++;                                    \
  68                 }                                               \
  69                 n -= len;                                       \
  70                 if (left)                                       \
  71                         break;                                  \
  72         }                                                       \
  73         i->iov_offset = skip;                                   \
  74         n = off;                                                \
  75 }
  76
  77 #define iterate_xarray(i, n, base, len, __off, STEP) {          \
  78         __label__ __out;                                        \
  79         size_t __off = 0;                                       \
  80         struct folio *folio;                                    \
  81         loff_t start = i->xarray_start + i->iov_offset;         \
  82         pgoff_t index = start / PAGE_SIZE;                      \
  83         XA_STATE(xas, i->xarray, index);                        \
  84                                                                 \
  85         len = PAGE_SIZE - offset_in_page(start);                \
  86         rcu_read_lock();                                        \
  87         xas_for_each(&xas, folio, ULONG_MAX) {                  \
  88                 unsigned left;                                  \
  89                 size_t offset;                                  \
  90                 if (xas_retry(&xas, folio))                     \
  91                         continue;                               \
  92                 if (WARN_ON(xa_is_value(folio)))                \
  93                         break;                                  \
  94                 if (WARN_ON(folio_test_hugetlb(folio)))         \
  95                         break;                                  \
  96                 offset = offset_in_folio(folio, start + __off); \
  97                 while (offset < folio_size(folio)) {            \
  98                         base = kmap_local_folio(folio, offset); \
  99                         len = min(n, len);                      \
 100                         left = (STEP);                          \
 101                         kunmap_local(base);                     \
 102                         len -= left;                            \
 103                         __off += len;                           \
 104                         n -= len;                               \
 105                         if (left || n == 0)                     \
 106                                 goto __out;                     \
 107                         offset += len;                          \
 108                         len = PAGE_SIZE;                        \
 109                 }                                               \
 110         }                                                       \
 111 __out:                                                          \
 112         rcu_read_unlock();                                      \
 113         i->iov_offset += __off;                                 \
 114         n = __off;                                              \
 115 }
 116
 117 #define __iterate_and_advance(i, n, base, len, off, I, K) {     \
 118         if (unlikely(i->count < n))                             \
 119                 n = i->count;                                   \
 120         if (likely(n)) {                                        \
 121                 if (likely(iter_is_ubuf(i))) {                  \
 122                         void __user *base;                      \
 123                         size_t len;                             \
 124                         iterate_buf(i, n, base, len, off,       \
 125                                                 i->ubuf, (I))   \
 126                 } else if (likely(iter_is_iovec(i))) {          \
 127                         const struct iovec *iov = iter_iov(i);  \
 128                         void __user *base;                      \
 129                         size_t len;                             \
 130                         iterate_iovec(i, n, base, len, off,     \
 131                                                 iov, (I))       \
 132                         i->nr_segs -= iov - iter_iov(i);        \
 133                         i->__iov = iov;                         \
 134                 } else if (iov_iter_is_bvec(i)) {               \
 135                         const struct bio_vec *bvec = i->bvec;   \
 136                         void *base;                             \
 137                         size_t len;                             \
 138                         iterate_bvec(i, n, base, len, off,      \
 139                                                 bvec, (K))      \
 140                         i->nr_segs -= bvec - i->bvec;           \
 141                         i->bvec = bvec;                         \
 142                 } else if (iov_iter_is_kvec(i)) {               \
 143                         const struct kvec *kvec = i->kvec;      \
 144                         void *base;                             \
 145                         size_t len;                             \
 146                         iterate_iovec(i, n, base, len, off,     \
 147                                                 kvec, (K))      \
 148                         i->nr_segs -= kvec - i->kvec;           \
 149                         i->kvec = kvec;                         \
 150                 } else if (iov_iter_is_xarray(i)) {             \
 151                         void *base;                             \
 152                         size_t len;                             \
 153                         iterate_xarray(i, n, base, len, off,    \
 154                                                         (K))    \
 155                 }                                               \
 156                 i->count -= n;                                  \
 157         }                                                       \
 158 }
 159 #define iterate_and_advance(i, n, base, len, off, I, K) \
 160         __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0))
 161
 162 static int copyout(void __user *to, const void *from, size_t n)
 163 {
 164         if (should_fail_usercopy())
 165                 return n;
 166         if (access_ok(to, n)) {
 167                 instrument_copy_to_user(to, from, n);
 168                 n = raw_copy_to_user(to, from, n);
 169         }
 170         return n;
 171 }
 172
 173 static int copyout_nofault(void __user *to, const void *from, size_t n)
 174 {
 175         long res;
 176
 177         if (should_fail_usercopy())
 178                 return n;
 179
 180         res = copy_to_user_nofault(to, from, n);
 181
 182         return res < 0 ? n : res;
 183 }
 184
 185 static int copyin(void *to, const void __user *from, size_t n)
 186 {
 187         size_t res = n;
 188
 189         if (should_fail_usercopy())
 190                 return n;
 191         if (access_ok(from, n)) {
 192                 instrument_copy_from_user_before(to, from, n);
 193                 res = raw_copy_from_user(to, from, n);
 194                 instrument_copy_from_user_after(to, from, n, res);
 195         }
 196         return res;
 197 }
 198
 199 /*
 200  * fault_in_iov_iter_readable - fault in iov iterator for reading
 201  * @i: iterator
 202  * @size: maximum length
 203  *
 204  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 205  * @size.  For each iovec, fault in each page that constitutes the iovec.
 206  *
 207  * Returns the number of bytes not faulted in (like copy_to_user() and
 208  * copy_from_user()).
 209  *
 210  * Always returns 0 for non-userspace iterators.
 211  */
 212 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
 213 {
 214         if (iter_is_ubuf(i)) {
 215                 size_t n = min(size, iov_iter_count(i));
 216                 n -= fault_in_readable(i->ubuf + i->iov_offset, n);
 217                 return size - n;
 218         } else if (iter_is_iovec(i)) {
 219                 size_t count = min(size, iov_iter_count(i));
 220                 const struct iovec *p;
 221                 size_t skip;
 222
 223                 size -= count;
 224                 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
 225                         size_t len = min(count, p->iov_len - skip);
 226                         size_t ret;
 227
 228                         if (unlikely(!len))
 229                                 continue;
 230                         ret = fault_in_readable(p->iov_base + skip, len);
 231                         count -= len - ret;
 232                         if (ret)
 233                                 break;
 234                 }
 235                 return count + size;
 236         }
 237         return 0;
 238 }
 239 EXPORT_SYMBOL(fault_in_iov_iter_readable);
 240
 241 /*
 242  * fault_in_iov_iter_writeable - fault in iov iterator for writing
 243  * @i: iterator
 244  * @size: maximum length
 245  *
 246  * Faults in the iterator using get_user_pages(), i.e., without triggering
 247  * hardware page faults.  This is primarily useful when we already know that
 248  * some or all of the pages in @i aren't in memory.
 249  *
 250  * Returns the number of bytes not faulted in, like copy_to_user() and
 251  * copy_from_user().
 252  *
 253  * Always returns 0 for non-user-space iterators.
 254  */
 255 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
 256 {
 257         if (iter_is_ubuf(i)) {
 258                 size_t n = min(size, iov_iter_count(i));
 259                 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);
 260                 return size - n;
 261         } else if (iter_is_iovec(i)) {
 262                 size_t count = min(size, iov_iter_count(i));
 263                 const struct iovec *p;
 264                 size_t skip;
 265
 266                 size -= count;
 267                 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
 268                         size_t len = min(count, p->iov_len - skip);
 269                         size_t ret;
 270
 271                         if (unlikely(!len))
 272                                 continue;
 273                         ret = fault_in_safe_writeable(p->iov_base + skip, len);
 274                         count -= len - ret;
 275                         if (ret)
 276                                 break;
 277                 }
 278                 return count + size;
 279         }
 280         return 0;
 281 }
 282 EXPORT_SYMBOL(fault_in_iov_iter_writeable);
 283
 284 void iov_iter_init(struct iov_iter *i, unsigned int direction,
 285                         const struct iovec *iov, unsigned long nr_segs,
 286                         size_t count)
 287 {
 288         WARN_ON(direction & ~(READ | WRITE));
 289         *i = (struct iov_iter) {
 290                 .iter_type = ITER_IOVEC,
 291                 .copy_mc = false,
 292                 .nofault = false,
 293                 .user_backed = true,
 294                 .data_source = direction,
 295                 .__iov = iov,
 296                 .nr_segs = nr_segs,
 297                 .iov_offset = 0,
 298                 .count = count
 299         };
 300 }
 301 EXPORT_SYMBOL(iov_iter_init);
 302
 303 static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
 304                               __wsum sum, size_t off)
 305 {
 306         __wsum next = csum_partial_copy_nocheck(from, to, len);
 307         return csum_block_add(sum, next, off);
 308 }
 309
 310 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 311 {
 312         if (WARN_ON_ONCE(i->data_source))
 313                 return 0;
 314         if (user_backed_iter(i))
 315                 might_fault();
 316         iterate_and_advance(i, bytes, base, len, off,
 317                 copyout(base, addr + off, len),
 318                 memcpy(base, addr + off, len)
 319         )
 320
 321         return bytes;
 322 }
 323 EXPORT_SYMBOL(_copy_to_iter);
 324
 325 #ifdef CONFIG_ARCH_HAS_COPY_MC
 326 static int copyout_mc(void __user *to, const void *from, size_t n)
 327 {
 328         if (access_ok(to, n)) {
 329                 instrument_copy_to_user(to, from, n);
 330                 n = copy_mc_to_user((__force void *) to, from, n);
 331         }
 332         return n;
 333 }
 334
 335 /**
 336  * _copy_mc_to_iter - copy to iter with source memory error exception handling
 337  * @addr: source kernel address
 338  * @bytes: total transfer length
 339  * @i: destination iterator
 340  *
 341  * The pmem driver deploys this for the dax operation
 342  * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
 343  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
 344  * successfully copied.
 345  *
 346  * The main differences between this and typical _copy_to_iter().
 347  *
 348  * * Typical tail/residue handling after a fault retries the copy
 349  *   byte-by-byte until the fault happens again. Re-triggering machine
 350  *   checks is potentially fatal so the implementation uses source
 351  *   alignment and poison alignment assumptions to avoid re-triggering
 352  *   hardware exceptions.
 353  *
 354  * * ITER_KVEC and ITER_BVEC can return short copies.  Compare to
 355  *   copy_to_iter() where only ITER_IOVEC attempts might return a short copy.
 356  *
 357  * Return: number of bytes copied (may be %0)
 358  */
 359 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 360 {
 361         if (WARN_ON_ONCE(i->data_source))
 362                 return 0;
 363         if (user_backed_iter(i))
 364                 might_fault();
 365         __iterate_and_advance(i, bytes, base, len, off,
 366                 copyout_mc(base, addr + off, len),
 367                 copy_mc_to_kernel(base, addr + off, len)
 368         )
 369
 370         return bytes;
 371 }
 372 EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
 373 #endif /* CONFIG_ARCH_HAS_COPY_MC */
 374
 375 static void *memcpy_from_iter(struct iov_iter *i, void *to, const void *from,
 376                                  size_t size)
 377 {
 378         if (iov_iter_is_copy_mc(i))
 379                 return (void *)copy_mc_to_kernel(to, from, size);
 380         return memcpy(to, from, size);
 381 }
 382
 383 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 384 {
 385         if (WARN_ON_ONCE(!i->data_source))
 386                 return 0;
 387
 388         if (user_backed_iter(i))
 389                 might_fault();
 390         iterate_and_advance(i, bytes, base, len, off,
 391                 copyin(addr + off, base, len),
 392                 memcpy_from_iter(i, addr + off, base, len)
 393         )
 394
 395         return bytes;
 396 }
 397 EXPORT_SYMBOL(_copy_from_iter);
 398
 399 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
 400 {
 401         if (WARN_ON_ONCE(!i->data_source))
 402                 return 0;
 403
 404         iterate_and_advance(i, bytes, base, len, off,
 405                 __copy_from_user_inatomic_nocache(addr + off, base, len),
 406                 memcpy(addr + off, base, len)
 407         )
 408
 409         return bytes;
 410 }
 411 EXPORT_SYMBOL(_copy_from_iter_nocache);
 412
 413 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 414 /**
 415  * _copy_from_iter_flushcache - write destination through cpu cache
 416  * @addr: destination kernel address
 417  * @bytes: total transfer length
 418  * @i: source iterator
 419  *
 420  * The pmem driver arranges for filesystem-dax to use this facility via
 421  * dax_copy_from_iter() for ensuring that writes to persistent memory
 422  * are flushed through the CPU cache. It is differentiated from
 423  * _copy_from_iter_nocache() in that guarantees all data is flushed for
 424  * all iterator types. The _copy_from_iter_nocache() only attempts to
 425  * bypass the cache for the ITER_IOVEC case, and on some archs may use
 426  * instructions that strand dirty-data in the cache.
 427  *
 428  * Return: number of bytes copied (may be %0)
 429  */
 430 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
 431 {
 432         if (WARN_ON_ONCE(!i->data_source))
 433                 return 0;
 434
 435         iterate_and_advance(i, bytes, base, len, off,
 436                 __copy_from_user_flushcache(addr + off, base, len),
 437                 memcpy_flushcache(addr + off, base, len)
 438         )
 439
 440         return bytes;
 441 }
 442 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
 443 #endif
 444
 445 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
 446 {
 447         struct page *head;
 448         size_t v = n + offset;
 449
 450         /*
 451          * The general case needs to access the page order in order
 452          * to compute the page size.
 453          * However, we mostly deal with order-0 pages and thus can
 454          * avoid a possible cache line miss for requests that fit all
 455          * page orders.
 456          */
 457         if (n <= v && v <= PAGE_SIZE)
 458                 return true;
 459
 460         head = compound_head(page);
 461         v += (page - head) << PAGE_SHIFT;
 462
 463         if (WARN_ON(n > v || v > page_size(head)))
 464                 return false;
 465         return true;
 466 }
 467
 468 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 469                          struct iov_iter *i)
 470 {
 471         size_t res = 0;
 472         if (!page_copy_sane(page, offset, bytes))
 473                 return 0;
 474         if (WARN_ON_ONCE(i->data_source))
 475                 return 0;
 476         page += offset / PAGE_SIZE; // first subpage
 477         offset %= PAGE_SIZE;
 478         while (1) {
 479                 void *kaddr = kmap_local_page(page);
 480                 size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
 481                 n = _copy_to_iter(kaddr + offset, n, i);
 482                 kunmap_local(kaddr);
 483                 res += n;
 484                 bytes -= n;
 485                 if (!bytes || !n)
 486                         break;
 487                 offset += n;
 488                 if (offset == PAGE_SIZE) {
 489                         page++;
 490                         offset = 0;
 491                 }
 492         }
 493         return res;
 494 }
 495 EXPORT_SYMBOL(copy_page_to_iter);
 496
 497 size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes,
 498                                  struct iov_iter *i)
 499 {
 500         size_t res = 0;
 501
 502         if (!page_copy_sane(page, offset, bytes))
 503                 return 0;
 504         if (WARN_ON_ONCE(i->data_source))
 505                 return 0;
 506         page += offset / PAGE_SIZE; // first subpage
 507         offset %= PAGE_SIZE;
 508         while (1) {
 509                 void *kaddr = kmap_local_page(page);
 510                 size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
 511
 512                 iterate_and_advance(i, n, base, len, off,
 513                         copyout_nofault(base, kaddr + offset + off, len),
 514                         memcpy(base, kaddr + offset + off, len)
 515                 )
 516                 kunmap_local(kaddr);
 517                 res += n;
 518                 bytes -= n;
 519                 if (!bytes || !n)
 520                         break;
 521                 offset += n;
 522                 if (offset == PAGE_SIZE) {
 523                         page++;
 524                         offset = 0;
 525                 }
 526         }
 527         return res;
 528 }
 529 EXPORT_SYMBOL(copy_page_to_iter_nofault);
 530
 531 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 532                          struct iov_iter *i)
 533 {
 534         size_t res = 0;
 535         if (!page_copy_sane(page, offset, bytes))
 536                 return 0;
 537         page += offset / PAGE_SIZE; // first subpage
 538         offset %= PAGE_SIZE;
 539         while (1) {
 540                 void *kaddr = kmap_local_page(page);
 541                 size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
 542                 n = _copy_from_iter(kaddr + offset, n, i);
 543                 kunmap_local(kaddr);
 544                 res += n;
 545                 bytes -= n;
 546                 if (!bytes || !n)
 547                         break;
 548                 offset += n;
 549                 if (offset == PAGE_SIZE) {
 550                         page++;
 551                         offset = 0;
 552                 }
 553         }
 554         return res;
 555 }
 556 EXPORT_SYMBOL(copy_page_from_iter);
 557
 558 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
 559 {
 560         iterate_and_advance(i, bytes, base, len, count,
 561                 clear_user(base, len),
 562                 memset(base, 0, len)
 563         )
 564
 565         return bytes;
 566 }
 567 EXPORT_SYMBOL(iov_iter_zero);
 568
 569 size_t copy_page_from_iter_atomic(struct page *page, size_t offset,
 570                 size_t bytes, struct iov_iter *i)
 571 {
 572         size_t n, copied = 0;
 573
 574         if (!page_copy_sane(page, offset, bytes))
 575                 return 0;
 576         if (WARN_ON_ONCE(!i->data_source))
 577                 return 0;
 578
 579         do {
 580                 char *p;
 581
 582                 n = bytes - copied;
 583                 if (PageHighMem(page)) {
 584                         page += offset / PAGE_SIZE;
 585                         offset %= PAGE_SIZE;
 586                         n = min_t(size_t, n, PAGE_SIZE - offset);
 587                 }
 588
 589                 p = kmap_atomic(page) + offset;
 590                 iterate_and_advance(i, n, base, len, off,
 591                         copyin(p + off, base, len),
 592                         memcpy_from_iter(i, p + off, base, len)
 593                 )
 594                 kunmap_atomic(p);
 595                 copied += n;
 596                 offset += n;
 597         } while (PageHighMem(page) && copied != bytes && n > 0);
 598
 599         return copied;
 600 }
 601 EXPORT_SYMBOL(copy_page_from_iter_atomic);
 602
 603 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
 604 {
 605         const struct bio_vec *bvec, *end;
 606
 607         if (!i->count)
 608                 return;
 609         i->count -= size;
 610
 611         size += i->iov_offset;
 612
 613         for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) {
 614                 if (likely(size < bvec->bv_len))
 615                         break;
 616                 size -= bvec->bv_len;
 617         }
 618         i->iov_offset = size;
 619         i->nr_segs -= bvec - i->bvec;
 620         i->bvec = bvec;
 621 }
 622
 623 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)
 624 {
 625         const struct iovec *iov, *end;
 626
 627         if (!i->count)
 628                 return;
 629         i->count -= size;
 630
 631         size += i->iov_offset; // from beginning of current segment
 632         for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) {
 633                 if (likely(size < iov->iov_len))
 634                         break;
 635                 size -= iov->iov_len;
 636         }
 637         i->iov_offset = size;
 638         i->nr_segs -= iov - iter_iov(i);
 639         i->__iov = iov;
 640 }
 641
 642 void iov_iter_advance(struct iov_iter *i, size_t size)
 643 {
 644         if (unlikely(i->count < size))
 645                 size = i->count;
 646         if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
 647                 i->iov_offset += size;
 648                 i->count -= size;
 649         } else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) {
 650                 /* iovec and kvec have identical layouts */
 651                 iov_iter_iovec_advance(i, size);
 652         } else if (iov_iter_is_bvec(i)) {
 653                 iov_iter_bvec_advance(i, size);
 654         } else if (iov_iter_is_discard(i)) {
 655                 i->count -= size;
 656         }
 657 }
 658 EXPORT_SYMBOL(iov_iter_advance);
 659
 660 void iov_iter_revert(struct iov_iter *i, size_t unroll)
 661 {
 662         if (!unroll)
 663                 return;
 664         if (WARN_ON(unroll > MAX_RW_COUNT))
 665                 return;
 666         i->count += unroll;
 667         if (unlikely(iov_iter_is_discard(i)))
 668                 return;
 669         if (unroll <= i->iov_offset) {
 670                 i->iov_offset -= unroll;
 671                 return;
 672         }
 673         unroll -= i->iov_offset;
 674         if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) {
 675                 BUG(); /* We should never go beyond the start of the specified
 676                         * range since we might then be straying into pages that
 677                         * aren't pinned.
 678                         */
 679         } else if (iov_iter_is_bvec(i)) {
 680                 const struct bio_vec *bvec = i->bvec;
 681                 while (1) {
 682                         size_t n = (--bvec)->bv_len;
 683                         i->nr_segs++;
 684                         if (unroll <= n) {
 685                                 i->bvec = bvec;
 686                                 i->iov_offset = n - unroll;
 687                                 return;
 688                         }
 689                         unroll -= n;
 690                 }
 691         } else { /* same logics for iovec and kvec */
 692                 const struct iovec *iov = iter_iov(i);
 693                 while (1) {
 694                         size_t n = (--iov)->iov_len;
 695                         i->nr_segs++;
 696                         if (unroll <= n) {
 697                                 i->__iov = iov;
 698                                 i->iov_offset = n - unroll;
 699                                 return;
 700                         }
 701                         unroll -= n;
 702                 }
 703         }
 704 }
 705 EXPORT_SYMBOL(iov_iter_revert);
 706
 707 /*
 708  * Return the count of just the current iov_iter segment.
 709  */
 710 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 711 {
 712         if (i->nr_segs > 1) {
 713                 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
 714                         return min(i->count, iter_iov(i)->iov_len - i->iov_offset);
 715                 if (iov_iter_is_bvec(i))
 716                         return min(i->count, i->bvec->bv_len - i->iov_offset);
 717         }
 718         return i->count;
 719 }
 720 EXPORT_SYMBOL(iov_iter_single_seg_count);
 721
 722 void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
 723                         const struct kvec *kvec, unsigned long nr_segs,
 724                         size_t count)
 725 {
 726         WARN_ON(direction & ~(READ | WRITE));
 727         *i = (struct iov_iter){
 728                 .iter_type = ITER_KVEC,
 729                 .copy_mc = false,
 730                 .data_source = direction,
 731                 .kvec = kvec,
 732                 .nr_segs = nr_segs,
 733                 .iov_offset = 0,
 734                 .count = count
 735         };
 736 }
 737 EXPORT_SYMBOL(iov_iter_kvec);
 738
 739 void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
 740                         const struct bio_vec *bvec, unsigned long nr_segs,
 741                         size_t count)
 742 {
 743         WARN_ON(direction & ~(READ | WRITE));
 744         *i = (struct iov_iter){
 745                 .iter_type = ITER_BVEC,
 746                 .copy_mc = false,
 747                 .data_source = direction,
 748                 .bvec = bvec,
 749                 .nr_segs = nr_segs,
 750                 .iov_offset = 0,
 751                 .count = count
 752         };
 753 }
 754 EXPORT_SYMBOL(iov_iter_bvec);
 755
 756 /**
 757  * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
 758  * @i: The iterator to initialise.
 759  * @direction: The direction of the transfer.
 760  * @xarray: The xarray to access.
 761  * @start: The start file position.
 762  * @count: The size of the I/O buffer in bytes.
 763  *
 764  * Set up an I/O iterator to either draw data out of the pages attached to an
 765  * inode or to inject data into those pages.  The pages *must* be prevented
 766  * from evaporation, either by taking a ref on them or locking them by the
 767  * caller.
 768  */
 769 void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
 770                      struct xarray *xarray, loff_t start, size_t count)
 771 {
 772         BUG_ON(direction & ~1);
 773         *i = (struct iov_iter) {
 774                 .iter_type = ITER_XARRAY,
 775                 .copy_mc = false,
 776                 .data_source = direction,
 777                 .xarray = xarray,
 778                 .xarray_start = start,
 779                 .count = count,
 780                 .iov_offset = 0
 781         };
 782 }
 783 EXPORT_SYMBOL(iov_iter_xarray);
 784
 785 /**
 786  * iov_iter_discard - Initialise an I/O iterator that discards data
 787  * @i: The iterator to initialise.
 788  * @direction: The direction of the transfer.
 789  * @count: The size of the I/O buffer in bytes.
 790  *
 791  * Set up an I/O iterator that just discards everything that's written to it.
 792  * It's only available as a READ iterator.
 793  */
 794 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
 795 {
 796         BUG_ON(direction != READ);
 797         *i = (struct iov_iter){
 798                 .iter_type = ITER_DISCARD,
 799                 .copy_mc = false,
 800                 .data_source = false,
 801                 .count = count,
 802                 .iov_offset = 0
 803         };
 804 }
 805 EXPORT_SYMBOL(iov_iter_discard);
 806
 807 static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
 808                                    unsigned len_mask)
 809 {
 810         size_t size = i->count;
 811         size_t skip = i->iov_offset;
 812         unsigned k;
 813
 814         for (k = 0; k < i->nr_segs; k++, skip = 0) {
 815                 const struct iovec *iov = iter_iov(i) + k;
 816                 size_t len = iov->iov_len - skip;
 817
 818                 if (len > size)
 819                         len = size;
 820                 if (len & len_mask)
 821                         return false;
 822                 if ((unsigned long)(iov->iov_base + skip) & addr_mask)
 823                         return false;
 824
 825                 size -= len;
 826                 if (!size)
 827                         break;
 828         }
 829         return true;
 830 }
 831
 832 static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
 833                                   unsigned len_mask)
 834 {
 835         size_t size = i->count;
 836         unsigned skip = i->iov_offset;
 837         unsigned k;
 838
 839         for (k = 0; k < i->nr_segs; k++, skip = 0) {
 840                 size_t len = i->bvec[k].bv_len - skip;
 841
 842                 if (len > size)
 843                         len = size;
 844                 if (len & len_mask)
 845                         return false;
 846                 if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
 847                         return false;
 848
 849                 size -= len;
 850                 if (!size)
 851                         break;
 852         }
 853         return true;
 854 }
 855
 856 /**
 857  * iov_iter_is_aligned() - Check if the addresses and lengths of each segments
 858  *      are aligned to the parameters.
 859  *
 860  * @i: &struct iov_iter to restore
 861  * @addr_mask: bit mask to check against the iov element's addresses
 862  * @len_mask: bit mask to check against the iov element's lengths
 863  *
 864  * Return: false if any addresses or lengths intersect with the provided masks
 865  */
 866 bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
 867                          unsigned len_mask)
 868 {
 869         if (likely(iter_is_ubuf(i))) {
 870                 if (i->count & len_mask)
 871                         return false;
 872                 if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask)
 873                         return false;
 874                 return true;
 875         }
 876
 877         if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
 878                 return iov_iter_aligned_iovec(i, addr_mask, len_mask);
 879
 880         if (iov_iter_is_bvec(i))
 881                 return iov_iter_aligned_bvec(i, addr_mask, len_mask);
 882
 883         if (iov_iter_is_xarray(i)) {
 884                 if (i->count & len_mask)
 885                         return false;
 886                 if ((i->xarray_start + i->iov_offset) & addr_mask)
 887                         return false;
 888         }
 889
 890         return true;
 891 }
 892 EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
 893
 894 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
 895 {
 896         unsigned long res = 0;
 897         size_t size = i->count;
 898         size_t skip = i->iov_offset;
 899         unsigned k;
 900
 901         for (k = 0; k < i->nr_segs; k++, skip = 0) {
 902                 const struct iovec *iov = iter_iov(i) + k;
 903                 size_t len = iov->iov_len - skip;
 904                 if (len) {
 905                         res |= (unsigned long)iov->iov_base + skip;
 906                         if (len > size)
 907                                 len = size;
 908                         res |= len;
 909                         size -= len;
 910                         if (!size)
 911                                 break;
 912                 }
 913         }
 914         return res;
 915 }
 916
 917 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)
 918 {
 919         unsigned res = 0;
 920         size_t size = i->count;
 921         unsigned skip = i->iov_offset;
 922         unsigned k;
 923
 924         for (k = 0; k < i->nr_segs; k++, skip = 0) {
 925                 size_t len = i->bvec[k].bv_len - skip;
 926                 res |= (unsigned long)i->bvec[k].bv_offset + skip;
 927                 if (len > size)
 928                         len = size;
 929                 res |= len;
 930                 size -= len;
 931                 if (!size)
 932                         break;
 933         }
 934         return res;
 935 }
 936
 937 unsigned long iov_iter_alignment(const struct iov_iter *i)
 938 {
 939         if (likely(iter_is_ubuf(i))) {
 940                 size_t size = i->count;
 941                 if (size)
 942                         return ((unsigned long)i->ubuf + i->iov_offset) | size;
 943                 return 0;
 944         }
 945
 946         /* iovec and kvec have identical layouts */
 947         if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
 948                 return iov_iter_alignment_iovec(i);
 949
 950         if (iov_iter_is_bvec(i))
 951                 return iov_iter_alignment_bvec(i);
 952
 953         if (iov_iter_is_xarray(i))
 954                 return (i->xarray_start + i->iov_offset) | i->count;
 955
 956         return 0;
 957 }
 958 EXPORT_SYMBOL(iov_iter_alignment);
 959
 960 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
 961 {
 962         unsigned long res = 0;
 963         unsigned long v = 0;
 964         size_t size = i->count;
 965         unsigned k;
 966
 967         if (iter_is_ubuf(i))
 968                 return 0;
 969
 970         if (WARN_ON(!iter_is_iovec(i)))
 971                 return ~0U;
 972
 973         for (k = 0; k < i->nr_segs; k++) {
 974                 const struct iovec *iov = iter_iov(i) + k;
 975                 if (iov->iov_len) {
 976                         unsigned long base = (unsigned long)iov->iov_base;
 977                         if (v) // if not the first one
 978                                 res |= base | v; // this start | previous end
 979                         v = base + iov->iov_len;
 980                         if (size <= iov->iov_len)
 981                                 break;
 982                         size -= iov->iov_len;
 983                 }
 984         }
 985         return res;
 986 }
 987 EXPORT_SYMBOL(iov_iter_gap_alignment);
 988
 989 static int want_pages_array(struct page ***res, size_t size,
 990                             size_t start, unsigned int maxpages)
 991 {
 992         unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE);
 993
 994         if (count > maxpages)
 995                 count = maxpages;
 996         WARN_ON(!count);        // caller should've prevented that
 997         if (!*res) {
 998                 *res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
 999                 if (!*res)
1000                         return 0;
1001         }
1002         return count;
1003 }
1004
1005 static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,
1006                                           pgoff_t index, unsigned int nr_pages)
1007 {
1008         XA_STATE(xas, xa, index);
1009         struct page *page;
1010         unsigned int ret = 0;
1011
1012         rcu_read_lock();
1013         for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1014                 if (xas_retry(&xas, page))
1015                         continue;
1016
1017                 /* Has the page moved or been split? */
1018                 if (unlikely(page != xas_reload(&xas))) {
1019                         xas_reset(&xas);
1020                         continue;
1021                 }
1022
1023                 pages[ret] = find_subpage(page, xas.xa_index);
1024                 get_page(pages[ret]);
1025                 if (++ret == nr_pages)
1026                         break;
1027         }
1028         rcu_read_unlock();
1029         return ret;
1030 }
1031
1032 static ssize_t iter_xarray_get_pages(struct iov_iter *i,
1033                                      struct page ***pages, size_t maxsize,
1034                                      unsigned maxpages, size_t *_start_offset)
1035 {
1036         unsigned nr, offset, count;
1037         pgoff_t index;
1038         loff_t pos;
1039
1040         pos = i->xarray_start + i->iov_offset;
1041         index = pos >> PAGE_SHIFT;
1042         offset = pos & ~PAGE_MASK;
1043         *_start_offset = offset;
1044
1045         count = want_pages_array(pages, maxsize, offset, maxpages);
1046         if (!count)
1047                 return -ENOMEM;
1048         nr = iter_xarray_populate_pages(*pages, i->xarray, index, count);
1049         if (nr == 0)
1050                 return 0;
1051
1052         maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
1053         i->iov_offset += maxsize;
1054         i->count -= maxsize;
1055         return maxsize;
1056 }
1057
1058 /* must be done on non-empty ITER_UBUF or ITER_IOVEC one */
1059 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)
1060 {
1061         size_t skip;
1062         long k;
1063
1064         if (iter_is_ubuf(i))
1065                 return (unsigned long)i->ubuf + i->iov_offset;
1066
1067         for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
1068                 const struct iovec *iov = iter_iov(i) + k;
1069                 size_t len = iov->iov_len - skip;
1070
1071                 if (unlikely(!len))
1072                         continue;
1073                 if (*size > len)
1074                         *size = len;
1075                 return (unsigned long)iov->iov_base + skip;
1076         }
1077         BUG(); // if it had been empty, we wouldn't get called
1078 }
1079
1080 /* must be done on non-empty ITER_BVEC one */
1081 static struct page *first_bvec_segment(const struct iov_iter *i,
1082                                        size_t *size, size_t *start)
1083 {
1084         struct page *page;
1085         size_t skip = i->iov_offset, len;
1086
1087         len = i->bvec->bv_len - skip;
1088         if (*size > len)
1089                 *size = len;
1090         skip += i->bvec->bv_offset;
1091         page = i->bvec->bv_page + skip / PAGE_SIZE;
1092         *start = skip % PAGE_SIZE;
1093         return page;
1094 }
1095
1096 static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
1097                    struct page ***pages, size_t maxsize,
1098                    unsigned int maxpages, size_t *start)
1099 {
1100         unsigned int n, gup_flags = 0;
1101
1102         if (maxsize > i->count)
1103                 maxsize = i->count;
1104         if (!maxsize)
1105                 return 0;
1106         if (maxsize > MAX_RW_COUNT)
1107                 maxsize = MAX_RW_COUNT;
1108
1109         if (likely(user_backed_iter(i))) {
1110                 unsigned long addr;
1111                 int res;
1112
1113                 if (iov_iter_rw(i) != WRITE)
1114                         gup_flags |= FOLL_WRITE;
1115                 if (i->nofault)
1116                         gup_flags |= FOLL_NOFAULT;
1117
1118                 addr = first_iovec_segment(i, &maxsize);
1119                 *start = addr % PAGE_SIZE;
1120                 addr &= PAGE_MASK;
1121                 n = want_pages_array(pages, maxsize, *start, maxpages);
1122                 if (!n)
1123                         return -ENOMEM;
1124                 res = get_user_pages_fast(addr, n, gup_flags, *pages);
1125                 if (unlikely(res <= 0))
1126                         return res;
1127                 maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);
1128                 iov_iter_advance(i, maxsize);
1129                 return maxsize;
1130         }
1131         if (iov_iter_is_bvec(i)) {
1132                 struct page **p;
1133                 struct page *page;
1134
1135                 page = first_bvec_segment(i, &maxsize, start);
1136                 n = want_pages_array(pages, maxsize, *start, maxpages);
1137                 if (!n)
1138                         return -ENOMEM;
1139                 p = *pages;
1140                 for (int k = 0; k < n; k++)
1141                         get_page(p[k] = page + k);
1142                 maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);
1143                 i->count -= maxsize;
1144                 i->iov_offset += maxsize;
1145                 if (i->iov_offset == i->bvec->bv_len) {
1146                         i->iov_offset = 0;
1147                         i->bvec++;
1148                         i->nr_segs--;
1149                 }
1150                 return maxsize;
1151         }
1152         if (iov_iter_is_xarray(i))
1153                 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
1154         return -EFAULT;
1155 }
1156
1157 ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
1158                 size_t maxsize, unsigned maxpages, size_t *start)
1159 {
1160         if (!maxpages)
1161                 return 0;
1162         BUG_ON(!pages);
1163
1164         return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start);
1165 }
1166 EXPORT_SYMBOL(iov_iter_get_pages2);
1167
1168 ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i,
1169                 struct page ***pages, size_t maxsize, size_t *start)
1170 {
1171         ssize_t len;
1172
1173         *pages = NULL;
1174
1175         len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start);
1176         if (len <= 0) {
1177                 kvfree(*pages);
1178                 *pages = NULL;
1179         }
1180         return len;
1181 }
1182 EXPORT_SYMBOL(iov_iter_get_pages_alloc2);
1183
1184 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1185                                struct iov_iter *i)
1186 {
1187         __wsum sum, next;
1188         sum = *csum;
1189         if (WARN_ON_ONCE(!i->data_source))
1190                 return 0;
1191
1192         iterate_and_advance(i, bytes, base, len, off, ({
1193                 next = csum_and_copy_from_user(base, addr + off, len);
1194                 sum = csum_block_add(sum, next, off);
1195                 next ? 0 : len;
1196         }), ({
1197                 sum = csum_and_memcpy(addr + off, base, len, sum, off);
1198         })
1199         )
1200         *csum = sum;
1201         return bytes;
1202 }
1203 EXPORT_SYMBOL(csum_and_copy_from_iter);
1204
1205 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
1206                              struct iov_iter *i)
1207 {
1208         struct csum_state *csstate = _csstate;
1209         __wsum sum, next;
1210
1211         if (WARN_ON_ONCE(i->data_source))
1212                 return 0;
1213         if (unlikely(iov_iter_is_discard(i))) {
1214                 // can't use csum_memcpy() for that one - data is not copied
1215                 csstate->csum = csum_block_add(csstate->csum,
1216                                                csum_partial(addr, bytes, 0),
1217                                                csstate->off);
1218                 csstate->off += bytes;
1219                 return bytes;
1220         }
1221
1222         sum = csum_shift(csstate->csum, csstate->off);
1223         iterate_and_advance(i, bytes, base, len, off, ({
1224                 next = csum_and_copy_to_user(addr + off, base, len);
1225                 sum = csum_block_add(sum, next, off);
1226                 next ? 0 : len;
1227         }), ({
1228                 sum = csum_and_memcpy(base, addr + off, len, sum, off);
1229         })
1230         )
1231         csstate->csum = csum_shift(sum, csstate->off);
1232         csstate->off += bytes;
1233         return bytes;
1234 }
1235 EXPORT_SYMBOL(csum_and_copy_to_iter);
1236
1237 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1238                 struct iov_iter *i)
1239 {
1240 #ifdef CONFIG_CRYPTO_HASH
1241         struct ahash_request *hash = hashp;
1242         struct scatterlist sg;
1243         size_t copied;
1244
1245         copied = copy_to_iter(addr, bytes, i);
1246         sg_init_one(&sg, addr, copied);
1247         ahash_request_set_crypt(hash, &sg, NULL, copied);
1248         crypto_ahash_update(hash);
1249         return copied;
1250 #else
1251         return 0;
1252 #endif
1253 }
1254 EXPORT_SYMBOL(hash_and_copy_to_iter);
1255
1256 static int iov_npages(const struct iov_iter *i, int maxpages)
1257 {
1258         size_t skip = i->iov_offset, size = i->count;
1259         const struct iovec *p;
1260         int npages = 0;
1261
1262         for (p = iter_iov(i); size; skip = 0, p++) {
1263                 unsigned offs = offset_in_page(p->iov_base + skip);
1264                 size_t len = min(p->iov_len - skip, size);
1265
1266                 if (len) {
1267                         size -= len;
1268                         npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
1269                         if (unlikely(npages > maxpages))
1270                                 return maxpages;
1271                 }
1272         }
1273         return npages;
1274 }
1275
1276 static int bvec_npages(const struct iov_iter *i, int maxpages)
1277 {
1278         size_t skip = i->iov_offset, size = i->count;
1279         const struct bio_vec *p;
1280         int npages = 0;
1281
1282         for (p = i->bvec; size; skip = 0, p++) {
1283                 unsigned offs = (p->bv_offset + skip) % PAGE_SIZE;
1284                 size_t len = min(p->bv_len - skip, size);
1285
1286                 size -= len;
1287                 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
1288                 if (unlikely(npages > maxpages))
1289                         return maxpages;
1290         }
1291         return npages;
1292 }
1293
1294 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1295 {
1296         if (unlikely(!i->count))
1297                 return 0;
1298         if (likely(iter_is_ubuf(i))) {
1299                 unsigned offs = offset_in_page(i->ubuf + i->iov_offset);
1300                 int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE);
1301                 return min(npages, maxpages);
1302         }
1303         /* iovec and kvec have identical layouts */
1304         if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
1305                 return iov_npages(i, maxpages);
1306         if (iov_iter_is_bvec(i))
1307                 return bvec_npages(i, maxpages);
1308         if (iov_iter_is_xarray(i)) {
1309                 unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;
1310                 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);
1311                 return min(npages, maxpages);
1312         }
1313         return 0;
1314 }
1315 EXPORT_SYMBOL(iov_iter_npages);
1316
1317 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1318 {
1319         *new = *old;
1320         if (iov_iter_is_bvec(new))
1321                 return new->bvec = kmemdup(new->bvec,
1322                                     new->nr_segs * sizeof(struct bio_vec),
1323                                     flags);
1324         else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
1325                 /* iovec and kvec have identical layout */
1326                 return new->__iov = kmemdup(new->__iov,
1327                                    new->nr_segs * sizeof(struct iovec),
1328                                    flags);
1329         return NULL;
1330 }
1331 EXPORT_SYMBOL(dup_iter);
1332
1333 static __noclone int copy_compat_iovec_from_user(struct iovec *iov,
1334                 const struct iovec __user *uvec, unsigned long nr_segs)
1335 {
1336         const struct compat_iovec __user *uiov =
1337                 (const struct compat_iovec __user *)uvec;
1338         int ret = -EFAULT, i;
1339
1340         if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1341                 return -EFAULT;
1342
1343         for (i = 0; i < nr_segs; i++) {
1344                 compat_uptr_t buf;
1345                 compat_ssize_t len;
1346
1347                 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1348                 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1349
1350                 /* check for compat_size_t not fitting in compat_ssize_t .. */
1351                 if (len < 0) {
1352                         ret = -EINVAL;
1353                         goto uaccess_end;
1354                 }
1355                 iov[i].iov_base = compat_ptr(buf);
1356                 iov[i].iov_len = len;
1357         }
1358
1359         ret = 0;
1360 uaccess_end:
1361         user_access_end();
1362         return ret;
1363 }
1364
1365 static __noclone int copy_iovec_from_user(struct iovec *iov,
1366                 const struct iovec __user *uiov, unsigned long nr_segs)
1367 {
1368         int ret = -EFAULT;
1369
1370         if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1371                 return -EFAULT;
1372
1373         do {
1374                 void __user *buf;
1375                 ssize_t len;
1376
1377                 unsafe_get_user(len, &uiov->iov_len, uaccess_end);
1378                 unsafe_get_user(buf, &uiov->iov_base, uaccess_end);
1379
1380                 /* check for size_t not fitting in ssize_t .. */
1381                 if (unlikely(len < 0)) {
1382                         ret = -EINVAL;
1383                         goto uaccess_end;
1384                 }
1385                 iov->iov_base = buf;
1386                 iov->iov_len = len;
1387
1388                 uiov++; iov++;
1389         } while (--nr_segs);
1390
1391         ret = 0;
1392 uaccess_end:
1393         user_access_end();
1394         return ret;
1395 }
1396
1397 struct iovec *iovec_from_user(const struct iovec __user *uvec,
1398                 unsigned long nr_segs, unsigned long fast_segs,
1399                 struct iovec *fast_iov, bool compat)
1400 {
1401         struct iovec *iov = fast_iov;
1402         int ret;
1403
1404         /*
1405          * SuS says "The readv() function *may* fail if the iovcnt argument was
1406          * less than or equal to 0, or greater than {IOV_MAX}.  Linux has
1407          * traditionally returned zero for zero segments, so...
1408          */
1409         if (nr_segs == 0)
1410                 return iov;
1411         if (nr_segs > UIO_MAXIOV)
1412                 return ERR_PTR(-EINVAL);
1413         if (nr_segs > fast_segs) {
1414                 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
1415                 if (!iov)
1416                         return ERR_PTR(-ENOMEM);
1417         }
1418
1419         if (unlikely(compat))
1420                 ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
1421         else
1422                 ret = copy_iovec_from_user(iov, uvec, nr_segs);
1423         if (ret) {
1424                 if (iov != fast_iov)
1425                         kfree(iov);
1426                 return ERR_PTR(ret);
1427         }
1428
1429         return iov;
1430 }
1431
1432 /*
1433  * Single segment iovec supplied by the user, import it as ITER_UBUF.
1434  */
1435 static ssize_t __import_iovec_ubuf(int type, const struct iovec __user *uvec,
1436                                    struct iovec **iovp, struct iov_iter *i,
1437                                    bool compat)
1438 {
1439         struct iovec *iov = *iovp;
1440         ssize_t ret;
1441
1442         if (compat)
1443                 ret = copy_compat_iovec_from_user(iov, uvec, 1);
1444         else
1445                 ret = copy_iovec_from_user(iov, uvec, 1);
1446         if (unlikely(ret))
1447                 return ret;
1448
1449         ret = import_ubuf(type, iov->iov_base, iov->iov_len, i);
1450         if (unlikely(ret))
1451                 return ret;
1452         *iovp = NULL;
1453         return i->count;
1454 }
1455
1456 ssize_t __import_iovec(int type, const struct iovec __user *uvec,
1457                  unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
1458                  struct iov_iter *i, bool compat)
1459 {
1460         ssize_t total_len = 0;
1461         unsigned long seg;
1462         struct iovec *iov;
1463
1464         if (nr_segs == 1)
1465                 return __import_iovec_ubuf(type, uvec, iovp, i, compat);
1466
1467         iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
1468         if (IS_ERR(iov)) {
1469                 *iovp = NULL;
1470                 return PTR_ERR(iov);
1471         }
1472
1473         /*
1474          * According to the Single Unix Specification we should return EINVAL if
1475          * an element length is < 0 when cast to ssize_t or if the total length
1476          * would overflow the ssize_t return value of the system call.
1477          *
1478          * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
1479          * overflow case.
1480          */
1481         for (seg = 0; seg < nr_segs; seg++) {
1482                 ssize_t len = (ssize_t)iov[seg].iov_len;
1483
1484                 if (!access_ok(iov[seg].iov_base, len)) {
1485                         if (iov != *iovp)
1486                                 kfree(iov);
1487                         *iovp = NULL;
1488                         return -EFAULT;
1489                 }
1490
1491                 if (len > MAX_RW_COUNT - total_len) {
1492                         len = MAX_RW_COUNT - total_len;
1493                         iov[seg].iov_len = len;
1494                 }
1495                 total_len += len;
1496         }
1497
1498         iov_iter_init(i, type, iov, nr_segs, total_len);
1499         if (iov == *iovp)
1500                 *iovp = NULL;
1501         else
1502                 *iovp = iov;
1503         return total_len;
1504 }
1505
1506 /**
1507  * import_iovec() - Copy an array of &struct iovec from userspace
1508  *     into the kernel, check that it is valid, and initialize a new
1509  *     &struct iov_iter iterator to access it.
1510  *
1511  * @type: One of %READ or %WRITE.
1512  * @uvec: Pointer to the userspace array.
1513  * @nr_segs: Number of elements in userspace array.
1514  * @fast_segs: Number of elements in @iov.
1515  * @iovp: (input and output parameter) Pointer to pointer to (usually small
1516  *     on-stack) kernel array.
1517  * @i: Pointer to iterator that will be initialized on success.
1518  *
1519  * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1520  * then this function places %NULL in *@iov on return. Otherwise, a new
1521  * array will be allocated and the result placed in *@iov. This means that
1522  * the caller may call kfree() on *@iov regardless of whether the small
1523  * on-stack array was used or not (and regardless of whether this function
1524  * returns an error or not).
1525  *
1526  * Return: Negative error code on error, bytes imported on success
1527  */
1528 ssize_t import_iovec(int type, const struct iovec __user *uvec,
1529                  unsigned nr_segs, unsigned fast_segs,
1530                  struct iovec **iovp, struct iov_iter *i)
1531 {
1532         return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
1533                               in_compat_syscall());
1534 }
1535 EXPORT_SYMBOL(import_iovec);
1536
1537 int import_single_range(int rw, void __user *buf, size_t len,
1538                  struct iovec *iov, struct iov_iter *i)
1539 {
1540         if (len > MAX_RW_COUNT)
1541                 len = MAX_RW_COUNT;
1542         if (unlikely(!access_ok(buf, len)))
1543                 return -EFAULT;
1544
1545         iov_iter_ubuf(i, rw, buf, len);
1546         return 0;
1547 }
1548 EXPORT_SYMBOL(import_single_range);
1549
1550 int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i)
1551 {
1552         if (len > MAX_RW_COUNT)
1553                 len = MAX_RW_COUNT;
1554         if (unlikely(!access_ok(buf, len)))
1555                 return -EFAULT;
1556
1557         iov_iter_ubuf(i, rw, buf, len);
1558         return 0;
1559 }
1560
1561 /**
1562  * iov_iter_restore() - Restore a &struct iov_iter to the same state as when
1563  *     iov_iter_save_state() was called.
1564  *
1565  * @i: &struct iov_iter to restore
1566  * @state: state to restore from
1567  *
1568  * Used after iov_iter_save_state() to bring restore @i, if operations may
1569  * have advanced it.
1570  *
1571  * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
1572  */
1573 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
1574 {
1575         if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) &&
1576                          !iter_is_ubuf(i)) && !iov_iter_is_kvec(i))
1577                 return;
1578         i->iov_offset = state->iov_offset;
1579         i->count = state->count;
1580         if (iter_is_ubuf(i))
1581                 return;
1582         /*
1583          * For the *vec iters, nr_segs + iov is constant - if we increment
1584          * the vec, then we also decrement the nr_segs count. Hence we don't
1585          * need to track both of these, just one is enough and we can deduct
1586          * the other from that. ITER_KVEC and ITER_IOVEC are the same struct
1587          * size, so we can just increment the iov pointer as they are unionzed.
1588          * ITER_BVEC _may_ be the same size on some archs, but on others it is
1589          * not. Be safe and handle it separately.
1590          */
1591         BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
1592         if (iov_iter_is_bvec(i))
1593                 i->bvec -= state->nr_segs - i->nr_segs;
1594         else
1595                 i->__iov -= state->nr_segs - i->nr_segs;
1596         i->nr_segs = state->nr_segs;
1597 }
1598
1599 /*
1600  * Extract a list of contiguous pages from an ITER_XARRAY iterator.  This does not
1601  * get references on the pages, nor does it get a pin on them.
1602  */
1603 static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i,
1604                                              struct page ***pages, size_t maxsize,
1605                                              unsigned int maxpages,
1606                                              iov_iter_extraction_t extraction_flags,
1607                                              size_t *offset0)
1608 {
1609         struct page *page, **p;
1610         unsigned int nr = 0, offset;
1611         loff_t pos = i->xarray_start + i->iov_offset;
1612         pgoff_t index = pos >> PAGE_SHIFT;
1613         XA_STATE(xas, i->xarray, index);
1614
1615         offset = pos & ~PAGE_MASK;
1616         *offset0 = offset;
1617
1618         maxpages = want_pages_array(pages, maxsize, offset, maxpages);
1619         if (!maxpages)
1620                 return -ENOMEM;
1621         p = *pages;
1622
1623         rcu_read_lock();
1624         for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1625                 if (xas_retry(&xas, page))
1626                         continue;
1627
1628                 /* Has the page moved or been split? */
1629                 if (unlikely(page != xas_reload(&xas))) {
1630                         xas_reset(&xas);
1631                         continue;
1632                 }
1633
1634                 p[nr++] = find_subpage(page, xas.xa_index);
1635                 if (nr == maxpages)
1636                         break;
1637         }
1638         rcu_read_unlock();
1639
1640         maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
1641         iov_iter_advance(i, maxsize);
1642         return maxsize;
1643 }
1644
1645 /*
1646  * Extract a list of contiguous pages from an ITER_BVEC iterator.  This does
1647  * not get references on the pages, nor does it get a pin on them.
1648  */
1649 static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i,
1650                                            struct page ***pages, size_t maxsize,
1651                                            unsigned int maxpages,
1652                                            iov_iter_extraction_t extraction_flags,
1653                                            size_t *offset0)
1654 {
1655         struct page **p, *page;
1656         size_t skip = i->iov_offset, offset;
1657         int k;
1658
1659         for (;;) {
1660                 if (i->nr_segs == 0)
1661                         return 0;
1662                 maxsize = min(maxsize, i->bvec->bv_len - skip);
1663                 if (maxsize)
1664                         break;
1665                 i->iov_offset = 0;
1666                 i->nr_segs--;
1667                 i->bvec++;
1668                 skip = 0;
1669         }
1670
1671         skip += i->bvec->bv_offset;
1672         page = i->bvec->bv_page + skip / PAGE_SIZE;
1673         offset = skip % PAGE_SIZE;
1674         *offset0 = offset;
1675
1676         maxpages = want_pages_array(pages, maxsize, offset, maxpages);
1677         if (!maxpages)
1678                 return -ENOMEM;
1679         p = *pages;
1680         for (k = 0; k < maxpages; k++)
1681                 p[k] = page + k;
1682
1683         maxsize = min_t(size_t, maxsize, maxpages * PAGE_SIZE - offset);
1684         iov_iter_advance(i, maxsize);
1685         return maxsize;
1686 }
1687
1688 /*
1689  * Extract a list of virtually contiguous pages from an ITER_KVEC iterator.
1690  * This does not get references on the pages, nor does it get a pin on them.
1691  */
1692 static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i,
1693                                            struct page ***pages, size_t maxsize,
1694                                            unsigned int maxpages,
1695                                            iov_iter_extraction_t extraction_flags,
1696                                            size_t *offset0)
1697 {
1698         struct page **p, *page;
1699         const void *kaddr;
1700         size_t skip = i->iov_offset, offset, len;
1701         int k;
1702
1703         for (;;) {
1704                 if (i->nr_segs == 0)
1705                         return 0;
1706                 maxsize = min(maxsize, i->kvec->iov_len - skip);
1707                 if (maxsize)
1708                         break;
1709                 i->iov_offset = 0;
1710                 i->nr_segs--;
1711                 i->kvec++;
1712                 skip = 0;
1713         }
1714
1715         kaddr = i->kvec->iov_base + skip;
1716         offset = (unsigned long)kaddr & ~PAGE_MASK;
1717         *offset0 = offset;
1718
1719         maxpages = want_pages_array(pages, maxsize, offset, maxpages);
1720         if (!maxpages)
1721                 return -ENOMEM;
1722         p = *pages;
1723
1724         kaddr -= offset;
1725         len = offset + maxsize;
1726         for (k = 0; k < maxpages; k++) {
1727                 size_t seg = min_t(size_t, len, PAGE_SIZE);
1728
1729                 if (is_vmalloc_or_module_addr(kaddr))
1730                         page = vmalloc_to_page(kaddr);
1731                 else
1732                         page = virt_to_page(kaddr);
1733
1734                 p[k] = page;
1735                 len -= seg;
1736                 kaddr += PAGE_SIZE;
1737         }
1738
1739         maxsize = min_t(size_t, maxsize, maxpages * PAGE_SIZE - offset);
1740         iov_iter_advance(i, maxsize);
1741         return maxsize;
1742 }
1743
1744 /*
1745  * Extract a list of contiguous pages from a user iterator and get a pin on
1746  * each of them.  This should only be used if the iterator is user-backed
1747  * (IOBUF/UBUF).
1748  *
1749  * It does not get refs on the pages, but the pages must be unpinned by the
1750  * caller once the transfer is complete.
1751  *
1752  * This is safe to be used where background IO/DMA *is* going to be modifying
1753  * the buffer; using a pin rather than a ref makes forces fork() to give the
1754  * child a copy of the page.
1755  */
1756 static ssize_t iov_iter_extract_user_pages(struct iov_iter *i,
1757                                            struct page ***pages,
1758                                            size_t maxsize,
1759                                            unsigned int maxpages,
1760                                            iov_iter_extraction_t extraction_flags,
1761                                            size_t *offset0)
1762 {
1763         unsigned long addr;
1764         unsigned int gup_flags = 0;
1765         size_t offset;
1766         int res;
1767
1768         if (i->data_source == ITER_DEST)
1769                 gup_flags |= FOLL_WRITE;
1770         if (extraction_flags & ITER_ALLOW_P2PDMA)
1771                 gup_flags |= FOLL_PCI_P2PDMA;
1772         if (i->nofault)
1773                 gup_flags |= FOLL_NOFAULT;
1774
1775         addr = first_iovec_segment(i, &maxsize);
1776         *offset0 = offset = addr % PAGE_SIZE;
1777         addr &= PAGE_MASK;
1778         maxpages = want_pages_array(pages, maxsize, offset, maxpages);
1779         if (!maxpages)
1780                 return -ENOMEM;
1781         res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages);
1782         if (unlikely(res <= 0))
1783                 return res;
1784         maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset);
1785         iov_iter_advance(i, maxsize);
1786         return maxsize;
1787 }
1788
1789 /**
1790  * iov_iter_extract_pages - Extract a list of contiguous pages from an iterator
1791  * @i: The iterator to extract from
1792  * @pages: Where to return the list of pages
1793  * @maxsize: The maximum amount of iterator to extract
1794  * @maxpages: The maximum size of the list of pages
1795  * @extraction_flags: Flags to qualify request
1796  * @offset0: Where to return the starting offset into (*@pages)[0]
1797  *
1798  * Extract a list of contiguous pages from the current point of the iterator,
1799  * advancing the iterator.  The maximum number of pages and the maximum amount
1800  * of page contents can be set.
1801  *
1802  * If *@pages is NULL, a page list will be allocated to the required size and
1803  * *@pages will be set to its base.  If *@pages is not NULL, it will be assumed
1804  * that the caller allocated a page list at least @maxpages in size and this
1805  * will be filled in.
1806  *
1807  * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA
1808  * be allowed on the pages extracted.
1809  *
1810  * The iov_iter_extract_will_pin() function can be used to query how cleanup
1811  * should be performed.
1812  *
1813  * Extra refs or pins on the pages may be obtained as follows:
1814  *
1815  *  (*) If the iterator is user-backed (ITER_IOVEC/ITER_UBUF), pins will be
1816  *      added to the pages, but refs will not be taken.
1817  *      iov_iter_extract_will_pin() will return true.
1818  *
1819  *  (*) If the iterator is ITER_KVEC, ITER_BVEC or ITER_XARRAY, the pages are
1820  *      merely listed; no extra refs or pins are obtained.
1821  *      iov_iter_extract_will_pin() will return 0.
1822  *
1823  * Note also:
1824  *
1825  *  (*) Use with ITER_DISCARD is not supported as that has no content.
1826  *
1827  * On success, the function sets *@pages to the new pagelist, if allocated, and
1828  * sets *offset0 to the offset into the first page.
1829  *
1830  * It may also return -ENOMEM and -EFAULT.
1831  */
1832 ssize_t iov_iter_extract_pages(struct iov_iter *i,
1833                                struct page ***pages,
1834                                size_t maxsize,
1835                                unsigned int maxpages,
1836                                iov_iter_extraction_t extraction_flags,
1837                                size_t *offset0)
1838 {
1839         maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);
1840         if (!maxsize)
1841                 return 0;
1842
1843         if (likely(user_backed_iter(i)))
1844                 return iov_iter_extract_user_pages(i, pages, maxsize,
1845                                                    maxpages, extraction_flags,
1846                                                    offset0);
1847         if (iov_iter_is_kvec(i))
1848                 return iov_iter_extract_kvec_pages(i, pages, maxsize,
1849                                                    maxpages, extraction_flags,
1850                                                    offset0);
1851         if (iov_iter_is_bvec(i))
1852                 return iov_iter_extract_bvec_pages(i, pages, maxsize,
1853                                                    maxpages, extraction_flags,
1854                                                    offset0);
1855         if (iov_iter_is_xarray(i))
1856                 return iov_iter_extract_xarray_pages(i, pages, maxsize,
1857                                                      maxpages, extraction_flags,
1858                                                      offset0);
1859         return -EFAULT;
1860 }
1861 EXPORT_SYMBOL_GPL(iov_iter_extract_pages);