lib/iov_iter.c

   1 #include <linux/export.h>
   2 #include <linux/uio.h>
   3 #include <linux/pagemap.h>
   4 #include <linux/slab.h>
   5 #include <linux/vmalloc.h>
   6 #include <net/checksum.h>
   7
   8 #define iterate_iovec(i, n, __v, __p, skip, STEP) {     \
   9         size_t left;                                    \
  10         size_t wanted = n;                              \
  11         __p = i->iov;                                   \
  12         __v.iov_len = min(n, __p->iov_len - skip);      \
  13         if (likely(__v.iov_len)) {                      \
  14                 __v.iov_base = __p->iov_base + skip;    \
  15                 left = (STEP);                          \
  16                 __v.iov_len -= left;                    \
  17                 skip += __v.iov_len;                    \
  18                 n -= __v.iov_len;                       \
  19         } else {                                        \
  20                 left = 0;                               \
  21         }                                               \
  22         while (unlikely(!left && n)) {                  \
  23                 __p++;                                  \
  24                 __v.iov_len = min(n, __p->iov_len);     \
  25                 if (unlikely(!__v.iov_len))             \
  26                         continue;                       \
  27                 __v.iov_base = __p->iov_base;           \
  28                 left = (STEP);                          \
  29                 __v.iov_len -= left;                    \
  30                 skip = __v.iov_len;                     \
  31                 n -= __v.iov_len;                       \
  32         }                                               \
  33         n = wanted - n;                                 \
  34 }
  35
  36 #define iterate_kvec(i, n, __v, __p, skip, STEP) {      \
  37         size_t wanted = n;                              \
  38         __p = i->kvec;                                  \
  39         __v.iov_len = min(n, __p->iov_len - skip);      \
  40         if (likely(__v.iov_len)) {                      \
  41                 __v.iov_base = __p->iov_base + skip;    \
  42                 (void)(STEP);                           \
  43                 skip += __v.iov_len;                    \
  44                 n -= __v.iov_len;                       \
  45         }                                               \
  46         while (unlikely(n)) {                           \
  47                 __p++;                                  \
  48                 __v.iov_len = min(n, __p->iov_len);     \
  49                 if (unlikely(!__v.iov_len))             \
  50                         continue;                       \
  51                 __v.iov_base = __p->iov_base;           \
  52                 (void)(STEP);                           \
  53                 skip = __v.iov_len;                     \
  54                 n -= __v.iov_len;                       \
  55         }                                               \
  56         n = wanted;                                     \
  57 }
  58
  59 #define iterate_bvec(i, n, __v, __p, skip, STEP) {      \
  60         size_t wanted = n;                              \
  61         __p = i->bvec;                                  \
  62         __v.bv_len = min_t(size_t, n, __p->bv_len - skip);      \
  63         if (likely(__v.bv_len)) {                       \
  64                 __v.bv_page = __p->bv_page;             \
  65                 __v.bv_offset = __p->bv_offset + skip;  \
  66                 (void)(STEP);                           \
  67                 skip += __v.bv_len;                     \
  68                 n -= __v.bv_len;                        \
  69         }                                               \
  70         while (unlikely(n)) {                           \
  71                 __p++;                                  \
  72                 __v.bv_len = min_t(size_t, n, __p->bv_len);     \
  73                 if (unlikely(!__v.bv_len))              \
  74                         continue;                       \
  75                 __v.bv_page = __p->bv_page;             \
  76                 __v.bv_offset = __p->bv_offset;         \
  77                 (void)(STEP);                           \
  78                 skip = __v.bv_len;                      \
  79                 n -= __v.bv_len;                        \
  80         }                                               \
  81         n = wanted;                                     \
  82 }
  83
  84 #define iterate_all_kinds(i, n, v, I, B, K) {                   \
  85         size_t skip = i->iov_offset;                            \
  86         if (unlikely(i->type & ITER_BVEC)) {                    \
  87                 const struct bio_vec *bvec;                     \
  88                 struct bio_vec v;                               \
  89                 iterate_bvec(i, n, v, bvec, skip, (B))          \
  90         } else if (unlikely(i->type & ITER_KVEC)) {             \
  91                 const struct kvec *kvec;                        \
  92                 struct kvec v;                                  \
  93                 iterate_kvec(i, n, v, kvec, skip, (K))          \
  94         } else {                                                \
  95                 const struct iovec *iov;                        \
  96                 struct iovec v;                                 \
  97                 iterate_iovec(i, n, v, iov, skip, (I))          \
  98         }                                                       \
  99 }
 100
 101 #define iterate_and_advance(i, n, v, I, B, K) {                 \
 102         if (unlikely(i->count < n))                             \
 103                 n = i->count;                                   \
 104         if (i->count) {                                         \
 105                 size_t skip = i->iov_offset;                    \
 106                 if (unlikely(i->type & ITER_BVEC)) {            \
 107                         const struct bio_vec *bvec;             \
 108                         struct bio_vec v;                       \
 109                         iterate_bvec(i, n, v, bvec, skip, (B))  \
 110                         if (skip == bvec->bv_len) {             \
 111                                 bvec++;                         \
 112                                 skip = 0;                       \
 113                         }                                       \
 114                         i->nr_segs -= bvec - i->bvec;           \
 115                         i->bvec = bvec;                         \
 116                 } else if (unlikely(i->type & ITER_KVEC)) {     \
 117                         const struct kvec *kvec;                \
 118                         struct kvec v;                          \
 119                         iterate_kvec(i, n, v, kvec, skip, (K))  \
 120                         if (skip == kvec->iov_len) {            \
 121                                 kvec++;                         \
 122                                 skip = 0;                       \
 123                         }                                       \
 124                         i->nr_segs -= kvec - i->kvec;           \
 125                         i->kvec = kvec;                         \
 126                 } else {                                        \
 127                         const struct iovec *iov;                \
 128                         struct iovec v;                         \
 129                         iterate_iovec(i, n, v, iov, skip, (I))  \
 130                         if (skip == iov->iov_len) {             \
 131                                 iov++;                          \
 132                                 skip = 0;                       \
 133                         }                                       \
 134                         i->nr_segs -= iov - i->iov;             \
 135                         i->iov = iov;                           \
 136                 }                                               \
 137                 i->count -= n;                                  \
 138                 i->iov_offset = skip;                           \
 139         }                                                       \
 140 }
 141
 142 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
 143                          struct iov_iter *i)
 144 {
 145         size_t skip, copy, left, wanted;
 146         const struct iovec *iov;
 147         char __user *buf;
 148         void *kaddr, *from;
 149
 150         if (unlikely(bytes > i->count))
 151                 bytes = i->count;
 152
 153         if (unlikely(!bytes))
 154                 return 0;
 155
 156         wanted = bytes;
 157         iov = i->iov;
 158         skip = i->iov_offset;
 159         buf = iov->iov_base + skip;
 160         copy = min(bytes, iov->iov_len - skip);
 161
 162         if (!fault_in_pages_writeable(buf, copy)) {
 163                 kaddr = kmap_atomic(page);
 164                 from = kaddr + offset;
 165
 166                 /* first chunk, usually the only one */
 167                 left = __copy_to_user_inatomic(buf, from, copy);
 168                 copy -= left;
 169                 skip += copy;
 170                 from += copy;
 171                 bytes -= copy;
 172
 173                 while (unlikely(!left && bytes)) {
 174                         iov++;
 175                         buf = iov->iov_base;
 176                         copy = min(bytes, iov->iov_len);
 177                         left = __copy_to_user_inatomic(buf, from, copy);
 178                         copy -= left;
 179                         skip = copy;
 180                         from += copy;
 181                         bytes -= copy;
 182                 }
 183                 if (likely(!bytes)) {
 184                         kunmap_atomic(kaddr);
 185                         goto done;
 186                 }
 187                 offset = from - kaddr;
 188                 buf += copy;
 189                 kunmap_atomic(kaddr);
 190                 copy = min(bytes, iov->iov_len - skip);
 191         }
 192         /* Too bad - revert to non-atomic kmap */
 193         kaddr = kmap(page);
 194         from = kaddr + offset;
 195         left = __copy_to_user(buf, from, copy);
 196         copy -= left;
 197         skip += copy;
 198         from += copy;
 199         bytes -= copy;
 200         while (unlikely(!left && bytes)) {
 201                 iov++;
 202                 buf = iov->iov_base;
 203                 copy = min(bytes, iov->iov_len);
 204                 left = __copy_to_user(buf, from, copy);
 205                 copy -= left;
 206                 skip = copy;
 207                 from += copy;
 208                 bytes -= copy;
 209         }
 210         kunmap(page);
 211 done:
 212         if (skip == iov->iov_len) {
 213                 iov++;
 214                 skip = 0;
 215         }
 216         i->count -= wanted - bytes;
 217         i->nr_segs -= iov - i->iov;
 218         i->iov = iov;
 219         i->iov_offset = skip;
 220         return wanted - bytes;
 221 }
 222
 223 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
 224                          struct iov_iter *i)
 225 {
 226         size_t skip, copy, left, wanted;
 227         const struct iovec *iov;
 228         char __user *buf;
 229         void *kaddr, *to;
 230
 231         if (unlikely(bytes > i->count))
 232                 bytes = i->count;
 233
 234         if (unlikely(!bytes))
 235                 return 0;
 236
 237         wanted = bytes;
 238         iov = i->iov;
 239         skip = i->iov_offset;
 240         buf = iov->iov_base + skip;
 241         copy = min(bytes, iov->iov_len - skip);
 242
 243         if (!fault_in_pages_readable(buf, copy)) {
 244                 kaddr = kmap_atomic(page);
 245                 to = kaddr + offset;
 246
 247                 /* first chunk, usually the only one */
 248                 left = __copy_from_user_inatomic(to, buf, copy);
 249                 copy -= left;
 250                 skip += copy;
 251                 to += copy;
 252                 bytes -= copy;
 253
 254                 while (unlikely(!left && bytes)) {
 255                         iov++;
 256                         buf = iov->iov_base;
 257                         copy = min(bytes, iov->iov_len);
 258                         left = __copy_from_user_inatomic(to, buf, copy);
 259                         copy -= left;
 260                         skip = copy;
 261                         to += copy;
 262                         bytes -= copy;
 263                 }
 264                 if (likely(!bytes)) {
 265                         kunmap_atomic(kaddr);
 266                         goto done;
 267                 }
 268                 offset = to - kaddr;
 269                 buf += copy;
 270                 kunmap_atomic(kaddr);
 271                 copy = min(bytes, iov->iov_len - skip);
 272         }
 273         /* Too bad - revert to non-atomic kmap */
 274         kaddr = kmap(page);
 275         to = kaddr + offset;
 276         left = __copy_from_user(to, buf, copy);
 277         copy -= left;
 278         skip += copy;
 279         to += copy;
 280         bytes -= copy;
 281         while (unlikely(!left && bytes)) {
 282                 iov++;
 283                 buf = iov->iov_base;
 284                 copy = min(bytes, iov->iov_len);
 285                 left = __copy_from_user(to, buf, copy);
 286                 copy -= left;
 287                 skip = copy;
 288                 to += copy;
 289                 bytes -= copy;
 290         }
 291         kunmap(page);
 292 done:
 293         if (skip == iov->iov_len) {
 294                 iov++;
 295                 skip = 0;
 296         }
 297         i->count -= wanted - bytes;
 298         i->nr_segs -= iov - i->iov;
 299         i->iov = iov;
 300         i->iov_offset = skip;
 301         return wanted - bytes;
 302 }
 303
 304 /*
 305  * Fault in the first iovec of the given iov_iter, to a maximum length
 306  * of bytes. Returns 0 on success, or non-zero if the memory could not be
 307  * accessed (ie. because it is an invalid address).
 308  *
 309  * writev-intensive code may want this to prefault several iovecs -- that
 310  * would be possible (callers must not rely on the fact that _only_ the
 311  * first iovec will be faulted with the current implementation).
 312  */
 313 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 314 {
 315         if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
 316                 char __user *buf = i->iov->iov_base + i->iov_offset;
 317                 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
 318                 return fault_in_pages_readable(buf, bytes);
 319         }
 320         return 0;
 321 }
 322 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 323
 324 /*
 325  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 326  * bytes.  For each iovec, fault in each page that constitutes the iovec.
 327  *
 328  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
 329  * because it is an invalid address).
 330  */
 331 int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes)
 332 {
 333         size_t skip = i->iov_offset;
 334         const struct iovec *iov;
 335         int err;
 336         struct iovec v;
 337
 338         if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
 339                 iterate_iovec(i, bytes, v, iov, skip, ({
 340                         err = fault_in_multipages_readable(v.iov_base,
 341                                         v.iov_len);
 342                         if (unlikely(err))
 343                         return err;
 344                 0;}))
 345         }
 346         return 0;
 347 }
 348 EXPORT_SYMBOL(iov_iter_fault_in_multipages_readable);
 349
 350 void iov_iter_init(struct iov_iter *i, int direction,
 351                         const struct iovec *iov, unsigned long nr_segs,
 352                         size_t count)
 353 {
 354         /* It will get better.  Eventually... */
 355         if (segment_eq(get_fs(), KERNEL_DS)) {
 356                 direction |= ITER_KVEC;
 357                 i->type = direction;
 358                 i->kvec = (struct kvec *)iov;
 359         } else {
 360                 i->type = direction;
 361                 i->iov = iov;
 362         }
 363         i->nr_segs = nr_segs;
 364         i->iov_offset = 0;
 365         i->count = count;
 366 }
 367 EXPORT_SYMBOL(iov_iter_init);
 368
 369 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
 370 {
 371         char *from = kmap_atomic(page);
 372         memcpy(to, from + offset, len);
 373         kunmap_atomic(from);
 374 }
 375
 376 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
 377 {
 378         char *to = kmap_atomic(page);
 379         memcpy(to + offset, from, len);
 380         kunmap_atomic(to);
 381 }
 382
 383 static void memzero_page(struct page *page, size_t offset, size_t len)
 384 {
 385         char *addr = kmap_atomic(page);
 386         memset(addr + offset, 0, len);
 387         kunmap_atomic(addr);
 388 }
 389
 390 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 391 {
 392         const char *from = addr;
 393         iterate_and_advance(i, bytes, v,
 394                 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
 395                                v.iov_len),
 396                 memcpy_to_page(v.bv_page, v.bv_offset,
 397                                (from += v.bv_len) - v.bv_len, v.bv_len),
 398                 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
 399         )
 400
 401         return bytes;
 402 }
 403 EXPORT_SYMBOL(copy_to_iter);
 404
 405 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 406 {
 407         char *to = addr;
 408         iterate_and_advance(i, bytes, v,
 409                 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
 410                                  v.iov_len),
 411                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 412                                  v.bv_offset, v.bv_len),
 413                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 414         )
 415
 416         return bytes;
 417 }
 418 EXPORT_SYMBOL(copy_from_iter);
 419
 420 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
 421 {
 422         char *to = addr;
 423         iterate_and_advance(i, bytes, v,
 424                 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
 425                                          v.iov_base, v.iov_len),
 426                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 427                                  v.bv_offset, v.bv_len),
 428                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 429         )
 430
 431         return bytes;
 432 }
 433 EXPORT_SYMBOL(copy_from_iter_nocache);
 434
 435 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 436                          struct iov_iter *i)
 437 {
 438         if (i->type & (ITER_BVEC|ITER_KVEC)) {
 439                 void *kaddr = kmap_atomic(page);
 440                 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
 441                 kunmap_atomic(kaddr);
 442                 return wanted;
 443         } else
 444                 return copy_page_to_iter_iovec(page, offset, bytes, i);
 445 }
 446 EXPORT_SYMBOL(copy_page_to_iter);
 447
 448 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 449                          struct iov_iter *i)
 450 {
 451         if (i->type & (ITER_BVEC|ITER_KVEC)) {
 452                 void *kaddr = kmap_atomic(page);
 453                 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
 454                 kunmap_atomic(kaddr);
 455                 return wanted;
 456         } else
 457                 return copy_page_from_iter_iovec(page, offset, bytes, i);
 458 }
 459 EXPORT_SYMBOL(copy_page_from_iter);
 460
 461 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
 462 {
 463         iterate_and_advance(i, bytes, v,
 464                 __clear_user(v.iov_base, v.iov_len),
 465                 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
 466                 memset(v.iov_base, 0, v.iov_len)
 467         )
 468
 469         return bytes;
 470 }
 471 EXPORT_SYMBOL(iov_iter_zero);
 472
 473 size_t iov_iter_copy_from_user_atomic(struct page *page,
 474                 struct iov_iter *i, unsigned long offset, size_t bytes)
 475 {
 476         char *kaddr = kmap_atomic(page), *p = kaddr + offset;
 477         iterate_all_kinds(i, bytes, v,
 478                 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
 479                                           v.iov_base, v.iov_len),
 480                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
 481                                  v.bv_offset, v.bv_len),
 482                 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 483         )
 484         kunmap_atomic(kaddr);
 485         return bytes;
 486 }
 487 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
 488
 489 void iov_iter_advance(struct iov_iter *i, size_t size)
 490 {
 491         iterate_and_advance(i, size, v, 0, 0, 0)
 492 }
 493 EXPORT_SYMBOL(iov_iter_advance);
 494
 495 /*
 496  * Return the count of just the current iov_iter segment.
 497  */
 498 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 499 {
 500         if (i->nr_segs == 1)
 501                 return i->count;
 502         else if (i->type & ITER_BVEC)
 503                 return min(i->count, i->bvec->bv_len - i->iov_offset);
 504         else
 505                 return min(i->count, i->iov->iov_len - i->iov_offset);
 506 }
 507 EXPORT_SYMBOL(iov_iter_single_seg_count);
 508
 509 void iov_iter_kvec(struct iov_iter *i, int direction,
 510                         const struct kvec *kvec, unsigned long nr_segs,
 511                         size_t count)
 512 {
 513         BUG_ON(!(direction & ITER_KVEC));
 514         i->type = direction;
 515         i->kvec = kvec;
 516         i->nr_segs = nr_segs;
 517         i->iov_offset = 0;
 518         i->count = count;
 519 }
 520 EXPORT_SYMBOL(iov_iter_kvec);
 521
 522 void iov_iter_bvec(struct iov_iter *i, int direction,
 523                         const struct bio_vec *bvec, unsigned long nr_segs,
 524                         size_t count)
 525 {
 526         BUG_ON(!(direction & ITER_BVEC));
 527         i->type = direction;
 528         i->bvec = bvec;
 529         i->nr_segs = nr_segs;
 530         i->iov_offset = 0;
 531         i->count = count;
 532 }
 533 EXPORT_SYMBOL(iov_iter_bvec);
 534
 535 unsigned long iov_iter_alignment(const struct iov_iter *i)
 536 {
 537         unsigned long res = 0;
 538         size_t size = i->count;
 539
 540         if (!size)
 541                 return 0;
 542
 543         iterate_all_kinds(i, size, v,
 544                 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
 545                 res |= v.bv_offset | v.bv_len,
 546                 res |= (unsigned long)v.iov_base | v.iov_len
 547         )
 548         return res;
 549 }
 550 EXPORT_SYMBOL(iov_iter_alignment);
 551
 552 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
 553 {
 554         unsigned long res = 0;
 555         size_t size = i->count;
 556         if (!size)
 557                 return 0;
 558
 559         iterate_all_kinds(i, size, v,
 560                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
 561                         (size != v.iov_len ? size : 0), 0),
 562                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
 563                         (size != v.bv_len ? size : 0)),
 564                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
 565                         (size != v.iov_len ? size : 0))
 566                 );
 567                 return res;
 568 }
 569 EXPORT_SYMBOL(iov_iter_gap_alignment);
 570
 571 ssize_t iov_iter_get_pages(struct iov_iter *i,
 572                    struct page **pages, size_t maxsize, unsigned maxpages,
 573                    size_t *start)
 574 {
 575         if (maxsize > i->count)
 576                 maxsize = i->count;
 577
 578         if (!maxsize)
 579                 return 0;
 580
 581         iterate_all_kinds(i, maxsize, v, ({
 582                 unsigned long addr = (unsigned long)v.iov_base;
 583                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
 584                 int n;
 585                 int res;
 586
 587                 if (len > maxpages * PAGE_SIZE)
 588                         len = maxpages * PAGE_SIZE;
 589                 addr &= ~(PAGE_SIZE - 1);
 590                 n = DIV_ROUND_UP(len, PAGE_SIZE);
 591                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
 592                 if (unlikely(res < 0))
 593                         return res;
 594                 return (res == n ? len : res * PAGE_SIZE) - *start;
 595         0;}),({
 596                 /* can't be more than PAGE_SIZE */
 597                 *start = v.bv_offset;
 598                 get_page(*pages = v.bv_page);
 599                 return v.bv_len;
 600         }),({
 601                 return -EFAULT;
 602         })
 603         )
 604         return 0;
 605 }
 606 EXPORT_SYMBOL(iov_iter_get_pages);
 607
 608 static struct page **get_pages_array(size_t n)
 609 {
 610         struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
 611         if (!p)
 612                 p = vmalloc(n * sizeof(struct page *));
 613         return p;
 614 }
 615
 616 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
 617                    struct page ***pages, size_t maxsize,
 618                    size_t *start)
 619 {
 620         struct page **p;
 621
 622         if (maxsize > i->count)
 623                 maxsize = i->count;
 624
 625         if (!maxsize)
 626                 return 0;
 627
 628         iterate_all_kinds(i, maxsize, v, ({
 629                 unsigned long addr = (unsigned long)v.iov_base;
 630                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
 631                 int n;
 632                 int res;
 633
 634                 addr &= ~(PAGE_SIZE - 1);
 635                 n = DIV_ROUND_UP(len, PAGE_SIZE);
 636                 p = get_pages_array(n);
 637                 if (!p)
 638                         return -ENOMEM;
 639                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
 640                 if (unlikely(res < 0)) {
 641                         kvfree(p);
 642                         return res;
 643                 }
 644                 *pages = p;
 645                 return (res == n ? len : res * PAGE_SIZE) - *start;
 646         0;}),({
 647                 /* can't be more than PAGE_SIZE */
 648                 *start = v.bv_offset;
 649                 *pages = p = get_pages_array(1);
 650                 if (!p)
 651                         return -ENOMEM;
 652                 get_page(*p = v.bv_page);
 653                 return v.bv_len;
 654         }),({
 655                 return -EFAULT;
 656         })
 657         )
 658         return 0;
 659 }
 660 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
 661
 662 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
 663                                struct iov_iter *i)
 664 {
 665         char *to = addr;
 666         __wsum sum, next;
 667         size_t off = 0;
 668         sum = *csum;
 669         iterate_and_advance(i, bytes, v, ({
 670                 int err = 0;
 671                 next = csum_and_copy_from_user(v.iov_base,
 672                                                (to += v.iov_len) - v.iov_len,
 673                                                v.iov_len, 0, &err);
 674                 if (!err) {
 675                         sum = csum_block_add(sum, next, off);
 676                         off += v.iov_len;
 677                 }
 678                 err ? v.iov_len : 0;
 679         }), ({
 680                 char *p = kmap_atomic(v.bv_page);
 681                 next = csum_partial_copy_nocheck(p + v.bv_offset,
 682                                                  (to += v.bv_len) - v.bv_len,
 683                                                  v.bv_len, 0);
 684                 kunmap_atomic(p);
 685                 sum = csum_block_add(sum, next, off);
 686                 off += v.bv_len;
 687         }),({
 688                 next = csum_partial_copy_nocheck(v.iov_base,
 689                                                  (to += v.iov_len) - v.iov_len,
 690                                                  v.iov_len, 0);
 691                 sum = csum_block_add(sum, next, off);
 692                 off += v.iov_len;
 693         })
 694         )
 695         *csum = sum;
 696         return bytes;
 697 }
 698 EXPORT_SYMBOL(csum_and_copy_from_iter);
 699
 700 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
 701                              struct iov_iter *i)
 702 {
 703         const char *from = addr;
 704         __wsum sum, next;
 705         size_t off = 0;
 706         sum = *csum;
 707         iterate_and_advance(i, bytes, v, ({
 708                 int err = 0;
 709                 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
 710                                              v.iov_base,
 711                                              v.iov_len, 0, &err);
 712                 if (!err) {
 713                         sum = csum_block_add(sum, next, off);
 714                         off += v.iov_len;
 715                 }
 716                 err ? v.iov_len : 0;
 717         }), ({
 718                 char *p = kmap_atomic(v.bv_page);
 719                 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
 720                                                  p + v.bv_offset,
 721                                                  v.bv_len, 0);
 722                 kunmap_atomic(p);
 723                 sum = csum_block_add(sum, next, off);
 724                 off += v.bv_len;
 725         }),({
 726                 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
 727                                                  v.iov_base,
 728                                                  v.iov_len, 0);
 729                 sum = csum_block_add(sum, next, off);
 730                 off += v.iov_len;
 731         })
 732         )
 733         *csum = sum;
 734         return bytes;
 735 }
 736 EXPORT_SYMBOL(csum_and_copy_to_iter);
 737
 738 int iov_iter_npages(const struct iov_iter *i, int maxpages)
 739 {
 740         size_t size = i->count;
 741         int npages = 0;
 742
 743         if (!size)
 744                 return 0;
 745
 746         iterate_all_kinds(i, size, v, ({
 747                 unsigned long p = (unsigned long)v.iov_base;
 748                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
 749                         - p / PAGE_SIZE;
 750                 if (npages >= maxpages)
 751                         return maxpages;
 752         0;}),({
 753                 npages++;
 754                 if (npages >= maxpages)
 755                         return maxpages;
 756         }),({
 757                 unsigned long p = (unsigned long)v.iov_base;
 758                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
 759                         - p / PAGE_SIZE;
 760                 if (npages >= maxpages)
 761                         return maxpages;
 762         })
 763         )
 764         return npages;
 765 }
 766 EXPORT_SYMBOL(iov_iter_npages);
 767
 768 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
 769 {
 770         *new = *old;
 771         if (new->type & ITER_BVEC)
 772                 return new->bvec = kmemdup(new->bvec,
 773                                     new->nr_segs * sizeof(struct bio_vec),
 774                                     flags);
 775         else
 776                 /* iovec and kvec have identical layout */
 777                 return new->iov = kmemdup(new->iov,
 778                                    new->nr_segs * sizeof(struct iovec),
 779                                    flags);
 780 }
 781 EXPORT_SYMBOL(dup_iter);
 782
 783 int import_iovec(int type, const struct iovec __user * uvector,
 784                  unsigned nr_segs, unsigned fast_segs,
 785                  struct iovec **iov, struct iov_iter *i)
 786 {
 787         ssize_t n;
 788         struct iovec *p;
 789         n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
 790                                   *iov, &p);
 791         if (n < 0) {
 792                 if (p != *iov)
 793                         kfree(p);
 794                 *iov = NULL;
 795                 return n;
 796         }
 797         iov_iter_init(i, type, p, nr_segs, n);
 798         *iov = p == *iov ? NULL : p;
 799         return 0;
 800 }
 801 EXPORT_SYMBOL(import_iovec);
 802
 803 #ifdef CONFIG_COMPAT
 804 #include <linux/compat.h>
 805
 806 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
 807                  unsigned nr_segs, unsigned fast_segs,
 808                  struct iovec **iov, struct iov_iter *i)
 809 {
 810         ssize_t n;
 811         struct iovec *p;
 812         n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
 813                                   *iov, &p);
 814         if (n < 0) {
 815                 if (p != *iov)
 816                         kfree(p);
 817                 *iov = NULL;
 818                 return n;
 819         }
 820         iov_iter_init(i, type, p, nr_segs, n);
 821         *iov = p == *iov ? NULL : p;
 822         return 0;
 823 }
 824 #endif
 825
 826 int import_single_range(int rw, void __user *buf, size_t len,
 827                  struct iovec *iov, struct iov_iter *i)
 828 {
 829         if (len > MAX_RW_COUNT)
 830                 len = MAX_RW_COUNT;
 831         if (unlikely(!access_ok(!rw, buf, len)))
 832                 return -EFAULT;
 833
 834         iov->iov_base = buf;
 835         iov->iov_len = len;
 836         iov_iter_init(i, rw, iov, 1, len);
 837         return 0;
 838 }
 839 EXPORT_SYMBOL(import_single_range);