1 // SPDX-License-Identifier: GPL-2.0-only
3 * "splice": joining two ropes together by interweaving their strands.
5 * This is the "extended pipe" functionality, where a pipe is used as
6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
7 * buffer that you can use to transfer data from one end to the other.
9 * The traditional unix read/write is extended with a "splice()" operation
10 * that transfers data buffers to or from a pipe buffer.
12 * Named by Larry McVoy, original implementation from Linus, extended by
13 * Jens to support splicing to files, network, direct splicing, etc and
14 * fixing lots of bugs.
16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
21 #include <linux/bvec.h>
23 #include <linux/file.h>
24 #include <linux/pagemap.h>
25 #include <linux/splice.h>
26 #include <linux/memcontrol.h>
27 #include <linux/mm_inline.h>
28 #include <linux/swap.h>
29 #include <linux/writeback.h>
30 #include <linux/export.h>
31 #include <linux/syscalls.h>
32 #include <linux/uio.h>
33 #include <linux/fsnotify.h>
34 #include <linux/security.h>
35 #include <linux/gfp.h>
36 #include <linux/net.h>
37 #include <linux/socket.h>
38 #include <linux/sched/signal.h>
43 * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to
44 * indicate they support non-blocking reads or writes, we must clear it
45 * here if set to avoid blocking other users of this pipe if splice is
48 static noinline void noinline pipe_clear_nowait(struct file *file)
50 fmode_t fmode = READ_ONCE(file->f_mode);
53 if (!(fmode & FMODE_NOWAIT))
55 } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));
59 * Attempt to steal a page from a pipe buffer. This should perhaps go into
60 * a vm helper function, it's already simplified quite a bit by the
61 * addition of remove_mapping(). If success is returned, the caller may
62 * attempt to reuse this page for another destination.
64 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
65 struct pipe_buffer *buf)
67 struct folio *folio = page_folio(buf->page);
68 struct address_space *mapping;
72 mapping = folio_mapping(folio);
74 WARN_ON(!folio_test_uptodate(folio));
77 * At least for ext2 with nobh option, we need to wait on
78 * writeback completing on this folio, since we'll remove it
79 * from the pagecache. Otherwise truncate wont wait on the
80 * folio, allowing the disk blocks to be reused by someone else
81 * before we actually wrote our data to them. fs corruption
84 folio_wait_writeback(folio);
86 if (folio_has_private(folio) &&
87 !filemap_release_folio(folio, GFP_KERNEL))
91 * If we succeeded in removing the mapping, set LRU flag
94 if (remove_mapping(mapping, folio)) {
95 buf->flags |= PIPE_BUF_FLAG_LRU;
101 * Raced with truncate or failed to remove folio from current
102 * address space, unlock and return failure.
109 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
110 struct pipe_buffer *buf)
113 buf->flags &= ~PIPE_BUF_FLAG_LRU;
117 * Check whether the contents of buf is OK to access. Since the content
118 * is a page cache page, IO may be in flight.
120 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
121 struct pipe_buffer *buf)
123 struct page *page = buf->page;
126 if (!PageUptodate(page)) {
130 * Page got truncated/unhashed. This will cause a 0-byte
131 * splice, if this is the first page.
133 if (!page->mapping) {
139 * Uh oh, read-error from disk.
141 if (!PageUptodate(page)) {
147 * Page is ok afterall, we are done.
158 const struct pipe_buf_operations page_cache_pipe_buf_ops = {
159 .confirm = page_cache_pipe_buf_confirm,
160 .release = page_cache_pipe_buf_release,
161 .try_steal = page_cache_pipe_buf_try_steal,
162 .get = generic_pipe_buf_get,
165 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
166 struct pipe_buffer *buf)
168 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
171 buf->flags |= PIPE_BUF_FLAG_LRU;
172 return generic_pipe_buf_try_steal(pipe, buf);
175 static const struct pipe_buf_operations user_page_pipe_buf_ops = {
176 .release = page_cache_pipe_buf_release,
177 .try_steal = user_page_pipe_buf_try_steal,
178 .get = generic_pipe_buf_get,
181 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
184 if (waitqueue_active(&pipe->rd_wait))
185 wake_up_interruptible(&pipe->rd_wait);
186 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
190 * splice_to_pipe - fill passed data into a pipe
191 * @pipe: pipe to fill
195 * @spd contains a map of pages and len/offset tuples, along with
196 * the struct pipe_buf_operations associated with these pages. This
197 * function will link that data to the pipe.
200 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
201 struct splice_pipe_desc *spd)
203 unsigned int spd_pages = spd->nr_pages;
204 unsigned int tail = pipe->tail;
205 unsigned int head = pipe->head;
206 unsigned int mask = pipe->ring_size - 1;
207 int ret = 0, page_nr = 0;
212 if (unlikely(!pipe->readers)) {
213 send_sig(SIGPIPE, current, 0);
218 while (!pipe_full(head, tail, pipe->max_usage)) {
219 struct pipe_buffer *buf = &pipe->bufs[head & mask];
221 buf->page = spd->pages[page_nr];
222 buf->offset = spd->partial[page_nr].offset;
223 buf->len = spd->partial[page_nr].len;
224 buf->private = spd->partial[page_nr].private;
233 if (!--spd->nr_pages)
241 while (page_nr < spd_pages)
242 spd->spd_release(spd, page_nr++);
246 EXPORT_SYMBOL_GPL(splice_to_pipe);
248 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
250 unsigned int head = pipe->head;
251 unsigned int tail = pipe->tail;
252 unsigned int mask = pipe->ring_size - 1;
255 if (unlikely(!pipe->readers)) {
256 send_sig(SIGPIPE, current, 0);
258 } else if (pipe_full(head, tail, pipe->max_usage)) {
261 pipe->bufs[head & mask] = *buf;
262 pipe->head = head + 1;
265 pipe_buf_release(pipe, buf);
268 EXPORT_SYMBOL(add_to_pipe);
271 * Check if we need to grow the arrays holding pages and partial page
274 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
276 unsigned int max_usage = READ_ONCE(pipe->max_usage);
278 spd->nr_pages_max = max_usage;
279 if (max_usage <= PIPE_DEF_BUFFERS)
282 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
283 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
286 if (spd->pages && spd->partial)
294 void splice_shrink_spd(struct splice_pipe_desc *spd)
296 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
304 * Splice data from an O_DIRECT file into pages and then add them to the output
307 ssize_t direct_splice_read(struct file *in, loff_t *ppos,
308 struct pipe_inode_info *pipe,
309 size_t len, unsigned int flags)
316 size_t used, npages, chunk, remain, reclaim;
319 /* Work out how much data we can actually add into the pipe */
320 used = pipe_occupancy(pipe->head, pipe->tail);
321 npages = max_t(ssize_t, pipe->max_usage - used, 0);
322 len = min_t(size_t, len, npages * PAGE_SIZE);
323 npages = DIV_ROUND_UP(len, PAGE_SIZE);
325 bv = kzalloc(array_size(npages, sizeof(bv[0])) +
326 array_size(npages, sizeof(struct page *)), GFP_KERNEL);
330 pages = (void *)(bv + npages);
331 npages = alloc_pages_bulk_array(GFP_USER, npages, pages);
337 remain = len = min_t(size_t, len, npages * PAGE_SIZE);
339 for (i = 0; i < npages; i++) {
340 chunk = min_t(size_t, PAGE_SIZE, remain);
341 bv[i].bv_page = pages[i];
343 bv[i].bv_len = chunk;
348 iov_iter_bvec(&to, ITER_DEST, bv, npages, len);
349 init_sync_kiocb(&kiocb, in);
350 kiocb.ki_pos = *ppos;
351 ret = call_read_iter(in, &kiocb, &to);
353 reclaim = npages * PAGE_SIZE;
358 *ppos = kiocb.ki_pos;
360 } else if (ret < 0) {
362 * callers of ->splice_read() expect -EAGAIN on
363 * "can't put anything in there", rather than -EFAULT.
369 /* Free any pages that didn't get touched at all. */
370 reclaim /= PAGE_SIZE;
373 release_pages(pages + npages, reclaim);
376 /* Push the remaining pages into the pipe. */
377 for (i = 0; i < npages; i++) {
378 struct pipe_buffer *buf = pipe_head_buf(pipe);
380 chunk = min_t(size_t, remain, PAGE_SIZE);
381 *buf = (struct pipe_buffer) {
382 .ops = &default_pipe_buf_ops,
383 .page = bv[i].bv_page,
394 EXPORT_SYMBOL(direct_splice_read);
397 * generic_file_splice_read - splice data from file to a pipe
398 * @in: file to splice from
399 * @ppos: position in @in
400 * @pipe: pipe to splice to
401 * @len: number of bytes to splice
402 * @flags: splice modifier flags
405 * Will read pages from given file and fill them into a pipe. Can be
406 * used as long as it has more or less sane ->read_iter().
409 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
410 struct pipe_inode_info *pipe, size_t len,
417 iov_iter_pipe(&to, ITER_DEST, pipe, len);
418 init_sync_kiocb(&kiocb, in);
419 kiocb.ki_pos = *ppos;
420 ret = call_read_iter(in, &kiocb, &to);
422 *ppos = kiocb.ki_pos;
424 } else if (ret < 0) {
425 /* free what was emitted */
426 pipe_discard_from(pipe, to.start_head);
428 * callers of ->splice_read() expect -EAGAIN on
429 * "can't put anything in there", rather than -EFAULT.
437 EXPORT_SYMBOL(generic_file_splice_read);
439 const struct pipe_buf_operations default_pipe_buf_ops = {
440 .release = generic_pipe_buf_release,
441 .try_steal = generic_pipe_buf_try_steal,
442 .get = generic_pipe_buf_get,
445 /* Pipe buffer operations for a socket and similar. */
446 const struct pipe_buf_operations nosteal_pipe_buf_ops = {
447 .release = generic_pipe_buf_release,
448 .get = generic_pipe_buf_get,
450 EXPORT_SYMBOL(nosteal_pipe_buf_ops);
452 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
455 if (waitqueue_active(&pipe->wr_wait))
456 wake_up_interruptible(&pipe->wr_wait);
457 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
461 * splice_from_pipe_feed - feed available data from a pipe to a file
462 * @pipe: pipe to splice from
463 * @sd: information to @actor
464 * @actor: handler that splices the data
467 * This function loops over the pipe and calls @actor to do the
468 * actual moving of a single struct pipe_buffer to the desired
469 * destination. It returns when there's no more buffers left in
470 * the pipe or if the requested number of bytes (@sd->total_len)
471 * have been copied. It returns a positive number (one) if the
472 * pipe needs to be filled with more data, zero if the required
473 * number of bytes have been copied and -errno on error.
475 * This, together with splice_from_pipe_{begin,end,next}, may be
476 * used to implement the functionality of __splice_from_pipe() when
477 * locking is required around copying the pipe buffers to the
480 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
483 unsigned int head = pipe->head;
484 unsigned int tail = pipe->tail;
485 unsigned int mask = pipe->ring_size - 1;
488 while (!pipe_empty(head, tail)) {
489 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
492 if (sd->len > sd->total_len)
493 sd->len = sd->total_len;
495 ret = pipe_buf_confirm(pipe, buf);
502 ret = actor(pipe, buf, sd);
509 sd->num_spliced += ret;
512 sd->total_len -= ret;
515 pipe_buf_release(pipe, buf);
519 sd->need_wakeup = true;
529 /* We know we have a pipe buffer, but maybe it's empty? */
530 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
532 unsigned int tail = pipe->tail;
533 unsigned int mask = pipe->ring_size - 1;
534 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
536 if (unlikely(!buf->len)) {
537 pipe_buf_release(pipe, buf);
546 * splice_from_pipe_next - wait for some data to splice from
547 * @pipe: pipe to splice from
548 * @sd: information about the splice operation
551 * This function will wait for some data and return a positive
552 * value (one) if pipe buffers are available. It will return zero
553 * or -errno if no more data needs to be spliced.
555 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
558 * Check for signal early to make process killable when there are
559 * always buffers available
561 if (signal_pending(current))
565 while (pipe_empty(pipe->head, pipe->tail)) {
572 if (sd->flags & SPLICE_F_NONBLOCK)
575 if (signal_pending(current))
578 if (sd->need_wakeup) {
579 wakeup_pipe_writers(pipe);
580 sd->need_wakeup = false;
583 pipe_wait_readable(pipe);
586 if (eat_empty_buffer(pipe))
593 * splice_from_pipe_begin - start splicing from pipe
594 * @sd: information about the splice operation
597 * This function should be called before a loop containing
598 * splice_from_pipe_next() and splice_from_pipe_feed() to
599 * initialize the necessary fields of @sd.
601 static void splice_from_pipe_begin(struct splice_desc *sd)
604 sd->need_wakeup = false;
608 * splice_from_pipe_end - finish splicing from pipe
609 * @pipe: pipe to splice from
610 * @sd: information about the splice operation
613 * This function will wake up pipe writers if necessary. It should
614 * be called after a loop containing splice_from_pipe_next() and
615 * splice_from_pipe_feed().
617 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
620 wakeup_pipe_writers(pipe);
624 * __splice_from_pipe - splice data from a pipe to given actor
625 * @pipe: pipe to splice from
626 * @sd: information to @actor
627 * @actor: handler that splices the data
630 * This function does little more than loop over the pipe and call
631 * @actor to do the actual moving of a single struct pipe_buffer to
632 * the desired destination. See pipe_to_file, pipe_to_sendmsg, or
636 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
641 splice_from_pipe_begin(sd);
644 ret = splice_from_pipe_next(pipe, sd);
646 ret = splice_from_pipe_feed(pipe, sd, actor);
648 splice_from_pipe_end(pipe, sd);
650 return sd->num_spliced ? sd->num_spliced : ret;
652 EXPORT_SYMBOL(__splice_from_pipe);
655 * splice_from_pipe - splice data from a pipe to a file
656 * @pipe: pipe to splice from
657 * @out: file to splice to
658 * @ppos: position in @out
659 * @len: how many bytes to splice
660 * @flags: splice modifier flags
661 * @actor: handler that splices the data
664 * See __splice_from_pipe. This function locks the pipe inode,
665 * otherwise it's identical to __splice_from_pipe().
668 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
669 loff_t *ppos, size_t len, unsigned int flags,
673 struct splice_desc sd = {
681 ret = __splice_from_pipe(pipe, &sd, actor);
688 * iter_file_splice_write - splice data from a pipe to a file
690 * @out: file to write to
691 * @ppos: position in @out
692 * @len: number of bytes to splice
693 * @flags: splice modifier flags
696 * Will either move or copy pages (determined by @flags options) from
697 * the given pipe inode to the given file.
698 * This one is ->write_iter-based.
702 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
703 loff_t *ppos, size_t len, unsigned int flags)
705 struct splice_desc sd = {
711 int nbufs = pipe->max_usage;
712 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
716 if (unlikely(!array))
721 splice_from_pipe_begin(&sd);
722 while (sd.total_len) {
723 struct iov_iter from;
724 unsigned int head, tail, mask;
728 ret = splice_from_pipe_next(pipe, &sd);
732 if (unlikely(nbufs < pipe->max_usage)) {
734 nbufs = pipe->max_usage;
735 array = kcalloc(nbufs, sizeof(struct bio_vec),
745 mask = pipe->ring_size - 1;
747 /* build the vector */
749 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
750 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
751 size_t this_len = buf->len;
753 /* zero-length bvecs are not supported, skip them */
756 this_len = min(this_len, left);
758 ret = pipe_buf_confirm(pipe, buf);
765 bvec_set_page(&array[n], buf->page, this_len,
771 iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);
772 ret = vfs_iter_write(out, &from, &sd.pos, 0);
776 sd.num_spliced += ret;
780 /* dismiss the fully eaten buffers, adjust the partial one */
783 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
784 if (ret >= buf->len) {
787 pipe_buf_release(pipe, buf);
791 sd.need_wakeup = true;
801 splice_from_pipe_end(pipe, &sd);
806 ret = sd.num_spliced;
811 EXPORT_SYMBOL(iter_file_splice_write);
815 * splice_to_socket - splice data from a pipe to a socket
816 * @pipe: pipe to splice from
817 * @out: socket to write to
818 * @ppos: position in @out
819 * @len: number of bytes to splice
820 * @flags: splice modifier flags
823 * Will send @len bytes from the pipe to a network socket. No data copying
827 ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out,
828 loff_t *ppos, size_t len, unsigned int flags)
830 struct socket *sock = sock_from_file(out);
831 struct bio_vec bvec[16];
832 struct msghdr msg = {};
835 bool need_wakeup = false;
840 unsigned int head, tail, mask, bc = 0;
844 * Check for signal early to make process killable when there
845 * are always buffers available
848 if (signal_pending(current))
851 while (pipe_empty(pipe->head, pipe->tail)) {
860 if (flags & SPLICE_F_NONBLOCK)
864 if (signal_pending(current))
868 wakeup_pipe_writers(pipe);
872 pipe_wait_readable(pipe);
877 mask = pipe->ring_size - 1;
879 while (!pipe_empty(head, tail)) {
880 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
888 seg = min_t(size_t, remain, buf->len);
889 seg = min_t(size_t, seg, PAGE_SIZE);
891 ret = pipe_buf_confirm(pipe, buf);
898 bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset);
902 if (bc >= ARRAY_SIZE(bvec))
909 msg.msg_flags = MSG_SPLICE_PAGES;
910 if (flags & SPLICE_F_MORE)
911 msg.msg_flags |= MSG_MORE;
912 if (remain && pipe_occupancy(pipe->head, tail) > 0)
913 msg.msg_flags |= MSG_MORE;
915 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc,
917 ret = sock_sendmsg(sock, &msg);
925 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
926 size_t seg = min_t(size_t, ret, buf->len);
933 pipe_buf_release(pipe, buf);
938 if (tail != pipe->tail) {
948 wakeup_pipe_writers(pipe);
949 return spliced ?: ret;
953 static int warn_unsupported(struct file *file, const char *op)
955 pr_debug_ratelimited(
956 "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
957 op, file, current->pid, current->comm);
962 * Attempt to initiate a splice from pipe to file.
964 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
965 loff_t *ppos, size_t len, unsigned int flags)
967 if (unlikely(!out->f_op->splice_write))
968 return warn_unsupported(out, "write");
969 return out->f_op->splice_write(pipe, out, ppos, len, flags);
973 * Indicate to the caller that there was a premature EOF when reading from the
974 * source and the caller didn't indicate they would be sending more data after
977 static void do_splice_eof(struct splice_desc *sd)
984 * Attempt to initiate a splice from a file to a pipe.
986 static long do_splice_to(struct file *in, loff_t *ppos,
987 struct pipe_inode_info *pipe, size_t len,
990 unsigned int p_space;
993 if (unlikely(!(in->f_mode & FMODE_READ)))
996 /* Don't try to read more the pipe has space for. */
997 p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);
998 len = min_t(size_t, len, p_space << PAGE_SHIFT);
1000 ret = rw_verify_area(READ, in, ppos, len);
1001 if (unlikely(ret < 0))
1004 if (unlikely(len > MAX_RW_COUNT))
1007 if (unlikely(!in->f_op->splice_read))
1008 return warn_unsupported(in, "read");
1009 return in->f_op->splice_read(in, ppos, pipe, len, flags);
1013 * splice_direct_to_actor - splices data directly between two non-pipes
1014 * @in: file to splice from
1015 * @sd: actor information on where to splice to
1016 * @actor: handles the data splicing
1019 * This is a special case helper to splice directly between two
1020 * points, without requiring an explicit pipe. Internally an allocated
1021 * pipe is cached in the process, and reused during the lifetime of
1025 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1026 splice_direct_actor *actor)
1028 struct pipe_inode_info *pipe;
1034 * We require the input to be seekable, as we don't want to randomly
1035 * drop data for eg socket -> socket splicing. Use the piped splicing
1038 if (unlikely(!(in->f_mode & FMODE_LSEEK)))
1042 * neither in nor out is a pipe, setup an internal pipe attached to
1043 * 'out' and transfer the wanted data from 'in' to 'out' through that
1045 pipe = current->splice_pipe;
1046 if (unlikely(!pipe)) {
1047 pipe = alloc_pipe_info();
1052 * We don't have an immediate reader, but we'll read the stuff
1053 * out of the pipe right after the splice_to_pipe(). So set
1054 * PIPE_READERS appropriately.
1058 current->splice_pipe = pipe;
1065 len = sd->total_len;
1067 /* Don't block on output, we have to drain the direct pipe. */
1069 sd->flags &= ~SPLICE_F_NONBLOCK;
1072 * We signal MORE until we've read sufficient data to fulfill the
1073 * request and we keep signalling it if the caller set it.
1075 more = sd->flags & SPLICE_F_MORE;
1076 sd->flags |= SPLICE_F_MORE;
1078 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
1082 loff_t pos = sd->pos, prev_pos = pos;
1084 ret = do_splice_to(in, &pos, pipe, len, flags);
1085 if (unlikely(ret <= 0))
1089 sd->total_len = read_len;
1092 * If we now have sufficient data to fulfill the request then
1093 * we clear SPLICE_F_MORE if it was not set initially.
1095 if (read_len >= len && !more)
1096 sd->flags &= ~SPLICE_F_MORE;
1099 * NOTE: nonblocking mode only applies to the input. We
1100 * must not do the output in nonblocking mode as then we
1101 * could get stuck data in the internal pipe:
1103 ret = actor(pipe, sd);
1104 if (unlikely(ret <= 0)) {
1113 if (ret < read_len) {
1114 sd->pos = prev_pos + ret;
1120 pipe->tail = pipe->head = 0;
1126 * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that
1127 * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a
1128 * "->splice_in()" that returned EOF (ie zero) *and* we have sent at
1129 * least 1 byte *then* we will also do the ->splice_eof() call.
1131 if (ret == 0 && !more && len > 0 && bytes)
1135 * If we did an incomplete transfer we must release
1136 * the pipe buffers in question:
1138 for (i = 0; i < pipe->ring_size; i++) {
1139 struct pipe_buffer *buf = &pipe->bufs[i];
1142 pipe_buf_release(pipe, buf);
1150 EXPORT_SYMBOL(splice_direct_to_actor);
1152 static int direct_splice_actor(struct pipe_inode_info *pipe,
1153 struct splice_desc *sd)
1155 struct file *file = sd->u.file;
1157 return do_splice_from(pipe, file, sd->opos, sd->total_len,
1161 static void direct_file_splice_eof(struct splice_desc *sd)
1163 struct file *file = sd->u.file;
1165 if (file->f_op->splice_eof)
1166 file->f_op->splice_eof(file);
1170 * do_splice_direct - splices data directly between two files
1171 * @in: file to splice from
1172 * @ppos: input file offset
1173 * @out: file to splice to
1174 * @opos: output file offset
1175 * @len: number of bytes to splice
1176 * @flags: splice modifier flags
1179 * For use by do_sendfile(). splice can easily emulate sendfile, but
1180 * doing it in the application would incur an extra system call
1181 * (splice in + splice out, as compared to just sendfile()). So this helper
1182 * can splice directly through a process-private pipe.
1185 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1186 loff_t *opos, size_t len, unsigned int flags)
1188 struct splice_desc sd = {
1194 .splice_eof = direct_file_splice_eof,
1199 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1202 if (unlikely(out->f_flags & O_APPEND))
1205 ret = rw_verify_area(WRITE, out, opos, len);
1206 if (unlikely(ret < 0))
1209 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1215 EXPORT_SYMBOL(do_splice_direct);
1217 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
1220 if (unlikely(!pipe->readers)) {
1221 send_sig(SIGPIPE, current, 0);
1224 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1226 if (flags & SPLICE_F_NONBLOCK)
1228 if (signal_pending(current))
1229 return -ERESTARTSYS;
1230 pipe_wait_writable(pipe);
1234 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1235 struct pipe_inode_info *opipe,
1236 size_t len, unsigned int flags);
1238 long splice_file_to_pipe(struct file *in,
1239 struct pipe_inode_info *opipe,
1241 size_t len, unsigned int flags)
1246 ret = wait_for_space(opipe, flags);
1248 ret = do_splice_to(in, offset, opipe, len, flags);
1251 wakeup_pipe_readers(opipe);
1256 * Determine where to splice to/from.
1258 long do_splice(struct file *in, loff_t *off_in, struct file *out,
1259 loff_t *off_out, size_t len, unsigned int flags)
1261 struct pipe_inode_info *ipipe;
1262 struct pipe_inode_info *opipe;
1266 if (unlikely(!(in->f_mode & FMODE_READ) ||
1267 !(out->f_mode & FMODE_WRITE)))
1270 ipipe = get_pipe_info(in, true);
1271 opipe = get_pipe_info(out, true);
1273 if (ipipe && opipe) {
1274 if (off_in || off_out)
1277 /* Splicing to self would be fun, but... */
1281 if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1282 flags |= SPLICE_F_NONBLOCK;
1284 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1291 if (!(out->f_mode & FMODE_PWRITE))
1295 offset = out->f_pos;
1298 if (unlikely(out->f_flags & O_APPEND))
1301 ret = rw_verify_area(WRITE, out, &offset, len);
1302 if (unlikely(ret < 0))
1305 if (in->f_flags & O_NONBLOCK)
1306 flags |= SPLICE_F_NONBLOCK;
1308 file_start_write(out);
1309 ret = do_splice_from(ipipe, out, &offset, len, flags);
1310 file_end_write(out);
1313 fsnotify_modify(out);
1316 out->f_pos = offset;
1327 if (!(in->f_mode & FMODE_PREAD))
1334 if (out->f_flags & O_NONBLOCK)
1335 flags |= SPLICE_F_NONBLOCK;
1337 ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
1340 fsnotify_access(in);
1353 static long __do_splice(struct file *in, loff_t __user *off_in,
1354 struct file *out, loff_t __user *off_out,
1355 size_t len, unsigned int flags)
1357 struct pipe_inode_info *ipipe;
1358 struct pipe_inode_info *opipe;
1359 loff_t offset, *__off_in = NULL, *__off_out = NULL;
1362 ipipe = get_pipe_info(in, true);
1363 opipe = get_pipe_info(out, true);
1368 pipe_clear_nowait(in);
1373 pipe_clear_nowait(out);
1377 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1379 __off_out = &offset;
1382 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1387 ret = do_splice(in, __off_in, out, __off_out, len, flags);
1391 if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
1393 if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
1399 static int iter_to_pipe(struct iov_iter *from,
1400 struct pipe_inode_info *pipe,
1403 struct pipe_buffer buf = {
1404 .ops = &user_page_pipe_buf_ops,
1410 while (iov_iter_count(from)) {
1411 struct page *pages[16];
1416 left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);
1422 n = DIV_ROUND_UP(left + start, PAGE_SIZE);
1423 for (i = 0; i < n; i++) {
1424 int size = min_t(int, left, PAGE_SIZE - start);
1426 buf.page = pages[i];
1429 ret = add_to_pipe(pipe, &buf);
1430 if (unlikely(ret < 0)) {
1431 iov_iter_revert(from, left);
1432 // this one got dropped by add_to_pipe()
1443 return total ? total : ret;
1446 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1447 struct splice_desc *sd)
1449 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1450 return n == sd->len ? n : -EFAULT;
1454 * For lack of a better implementation, implement vmsplice() to userspace
1455 * as a simple copy of the pipes pages to the user iov.
1457 static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
1460 struct pipe_inode_info *pipe = get_pipe_info(file, true);
1461 struct splice_desc sd = {
1462 .total_len = iov_iter_count(iter),
1471 pipe_clear_nowait(file);
1475 ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1483 * vmsplice splices a user address range into a pipe. It can be thought of
1484 * as splice-from-memory, where the regular splice is splice-from-file (or
1485 * to file). In both cases the output is a pipe, naturally.
1487 static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1490 struct pipe_inode_info *pipe;
1492 unsigned buf_flag = 0;
1494 if (flags & SPLICE_F_GIFT)
1495 buf_flag = PIPE_BUF_FLAG_GIFT;
1497 pipe = get_pipe_info(file, true);
1501 pipe_clear_nowait(file);
1504 ret = wait_for_space(pipe, flags);
1506 ret = iter_to_pipe(iter, pipe, buf_flag);
1509 wakeup_pipe_readers(pipe);
1513 static int vmsplice_type(struct fd f, int *type)
1517 if (f.file->f_mode & FMODE_WRITE) {
1518 *type = ITER_SOURCE;
1519 } else if (f.file->f_mode & FMODE_READ) {
1529 * Note that vmsplice only really supports true splicing _from_ user memory
1530 * to a pipe, not the other way around. Splicing from user memory is a simple
1531 * operation that can be supported without any funky alignment restrictions
1532 * or nasty vm tricks. We simply map in the user memory and fill them into
1533 * a pipe. The reverse isn't quite as easy, though. There are two possible
1534 * solutions for that:
1536 * - memcpy() the data internally, at which point we might as well just
1537 * do a regular read() on the buffer anyway.
1538 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1539 * has restriction limitations on both ends of the pipe).
1541 * Currently we punt and implement it as a normal copy, see pipe_to_user().
1544 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
1545 unsigned long, nr_segs, unsigned int, flags)
1547 struct iovec iovstack[UIO_FASTIOV];
1548 struct iovec *iov = iovstack;
1549 struct iov_iter iter;
1554 if (unlikely(flags & ~SPLICE_F_ALL))
1558 error = vmsplice_type(f, &type);
1562 error = import_iovec(type, uiov, nr_segs,
1563 ARRAY_SIZE(iovstack), &iov, &iter);
1567 if (!iov_iter_count(&iter))
1569 else if (type == ITER_SOURCE)
1570 error = vmsplice_to_pipe(f.file, &iter, flags);
1572 error = vmsplice_to_user(f.file, &iter, flags);
1580 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1581 int, fd_out, loff_t __user *, off_out,
1582 size_t, len, unsigned int, flags)
1590 if (unlikely(flags & ~SPLICE_F_ALL))
1596 out = fdget(fd_out);
1598 error = __do_splice(in.file, off_in, out.file, off_out,
1608 * Make sure there's data to read. Wait for input if we can, otherwise
1609 * return an appropriate error.
1611 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1616 * Check the pipe occupancy without the inode lock first. This function
1617 * is speculative anyways, so missing one is ok.
1619 if (!pipe_empty(pipe->head, pipe->tail))
1625 while (pipe_empty(pipe->head, pipe->tail)) {
1626 if (signal_pending(current)) {
1632 if (flags & SPLICE_F_NONBLOCK) {
1636 pipe_wait_readable(pipe);
1644 * Make sure there's writeable room. Wait for room if we can, otherwise
1645 * return an appropriate error.
1647 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1652 * Check pipe occupancy without the inode lock first. This function
1653 * is speculative anyways, so missing one is ok.
1655 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1661 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
1662 if (!pipe->readers) {
1663 send_sig(SIGPIPE, current, 0);
1667 if (flags & SPLICE_F_NONBLOCK) {
1671 if (signal_pending(current)) {
1675 pipe_wait_writable(pipe);
1683 * Splice contents of ipipe to opipe.
1685 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1686 struct pipe_inode_info *opipe,
1687 size_t len, unsigned int flags)
1689 struct pipe_buffer *ibuf, *obuf;
1690 unsigned int i_head, o_head;
1691 unsigned int i_tail, o_tail;
1692 unsigned int i_mask, o_mask;
1694 bool input_wakeup = false;
1698 ret = ipipe_prep(ipipe, flags);
1702 ret = opipe_prep(opipe, flags);
1707 * Potential ABBA deadlock, work around it by ordering lock
1708 * grabbing by pipe info address. Otherwise two different processes
1709 * could deadlock (one doing tee from A -> B, the other from B -> A).
1711 pipe_double_lock(ipipe, opipe);
1713 i_tail = ipipe->tail;
1714 i_mask = ipipe->ring_size - 1;
1715 o_head = opipe->head;
1716 o_mask = opipe->ring_size - 1;
1721 if (!opipe->readers) {
1722 send_sig(SIGPIPE, current, 0);
1728 i_head = ipipe->head;
1729 o_tail = opipe->tail;
1731 if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1735 * Cannot make any progress, because either the input
1736 * pipe is empty or the output pipe is full.
1738 if (pipe_empty(i_head, i_tail) ||
1739 pipe_full(o_head, o_tail, opipe->max_usage)) {
1740 /* Already processed some buffers, break */
1744 if (flags & SPLICE_F_NONBLOCK) {
1750 * We raced with another reader/writer and haven't
1751 * managed to process any buffers. A zero return
1752 * value means EOF, so retry instead.
1759 ibuf = &ipipe->bufs[i_tail & i_mask];
1760 obuf = &opipe->bufs[o_head & o_mask];
1762 if (len >= ibuf->len) {
1764 * Simply move the whole buffer from ipipe to opipe
1769 ipipe->tail = i_tail;
1770 input_wakeup = true;
1773 opipe->head = o_head;
1776 * Get a reference to this pipe buffer,
1777 * so we can copy the contents over.
1779 if (!pipe_buf_get(ipipe, ibuf)) {
1787 * Don't inherit the gift and merge flags, we need to
1788 * prevent multiple steals of this page.
1790 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1791 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1794 ibuf->offset += len;
1798 opipe->head = o_head;
1808 * If we put data in the output pipe, wakeup any potential readers.
1811 wakeup_pipe_readers(opipe);
1814 wakeup_pipe_writers(ipipe);
1820 * Link contents of ipipe to opipe.
1822 static int link_pipe(struct pipe_inode_info *ipipe,
1823 struct pipe_inode_info *opipe,
1824 size_t len, unsigned int flags)
1826 struct pipe_buffer *ibuf, *obuf;
1827 unsigned int i_head, o_head;
1828 unsigned int i_tail, o_tail;
1829 unsigned int i_mask, o_mask;
1833 * Potential ABBA deadlock, work around it by ordering lock
1834 * grabbing by pipe info address. Otherwise two different processes
1835 * could deadlock (one doing tee from A -> B, the other from B -> A).
1837 pipe_double_lock(ipipe, opipe);
1839 i_tail = ipipe->tail;
1840 i_mask = ipipe->ring_size - 1;
1841 o_head = opipe->head;
1842 o_mask = opipe->ring_size - 1;
1845 if (!opipe->readers) {
1846 send_sig(SIGPIPE, current, 0);
1852 i_head = ipipe->head;
1853 o_tail = opipe->tail;
1856 * If we have iterated all input buffers or run out of
1857 * output room, break.
1859 if (pipe_empty(i_head, i_tail) ||
1860 pipe_full(o_head, o_tail, opipe->max_usage))
1863 ibuf = &ipipe->bufs[i_tail & i_mask];
1864 obuf = &opipe->bufs[o_head & o_mask];
1867 * Get a reference to this pipe buffer,
1868 * so we can copy the contents over.
1870 if (!pipe_buf_get(ipipe, ibuf)) {
1879 * Don't inherit the gift and merge flag, we need to prevent
1880 * multiple steals of this page.
1882 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1883 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1885 if (obuf->len > len)
1891 opipe->head = o_head;
1899 * If we put data in the output pipe, wakeup any potential readers.
1902 wakeup_pipe_readers(opipe);
1908 * This is a tee(1) implementation that works on pipes. It doesn't copy
1909 * any data, it simply references the 'in' pages on the 'out' pipe.
1910 * The 'flags' used are the SPLICE_F_* variants, currently the only
1911 * applicable one is SPLICE_F_NONBLOCK.
1913 long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
1915 struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1916 struct pipe_inode_info *opipe = get_pipe_info(out, true);
1919 if (unlikely(!(in->f_mode & FMODE_READ) ||
1920 !(out->f_mode & FMODE_WRITE)))
1924 * Duplicate the contents of ipipe to opipe without actually
1927 if (ipipe && opipe && ipipe != opipe) {
1928 if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1929 flags |= SPLICE_F_NONBLOCK;
1932 * Keep going, unless we encounter an error. The ipipe/opipe
1933 * ordering doesn't really matter.
1935 ret = ipipe_prep(ipipe, flags);
1937 ret = opipe_prep(opipe, flags);
1939 ret = link_pipe(ipipe, opipe, len, flags);
1946 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1951 if (unlikely(flags & ~SPLICE_F_ALL))
1962 error = do_tee(in.file, out.file, len, flags);