Lines Matching full:pipe
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
9 * that transfers data buffers to or from a pipe buffer.
41 * Attempt to steal a page from a pipe buffer. This should perhaps go into
46 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, in page_cache_pipe_buf_steal() argument
91 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, in page_cache_pipe_buf_release() argument
102 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, in page_cache_pipe_buf_confirm() argument
148 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, in user_page_pipe_buf_steal() argument
155 return generic_pipe_buf_steal(pipe, buf); in user_page_pipe_buf_steal()
166 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) in wakeup_pipe_readers() argument
169 if (waitqueue_active(&pipe->wait)) in wakeup_pipe_readers()
170 wake_up_interruptible(&pipe->wait); in wakeup_pipe_readers()
171 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in wakeup_pipe_readers()
175 * splice_to_pipe - fill passed data into a pipe
176 * @pipe: pipe to fill
182 * function will link that data to the pipe.
185 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, in splice_to_pipe() argument
194 if (unlikely(!pipe->readers)) { in splice_to_pipe()
200 while (pipe->nrbufs < pipe->buffers) { in splice_to_pipe()
201 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); in splice_to_pipe()
202 struct pipe_buffer *buf = pipe->bufs + newbuf; in splice_to_pipe()
211 pipe->nrbufs++; in splice_to_pipe()
230 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in add_to_pipe() argument
234 if (unlikely(!pipe->readers)) { in add_to_pipe()
237 } else if (pipe->nrbufs == pipe->buffers) { in add_to_pipe()
240 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); in add_to_pipe()
241 pipe->bufs[newbuf] = *buf; in add_to_pipe()
242 pipe->nrbufs++; in add_to_pipe()
245 pipe_buf_release(pipe, buf); in add_to_pipe()
254 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) in splice_grow_spd() argument
256 unsigned int buffers = READ_ONCE(pipe->buffers); in splice_grow_spd()
284 * generic_file_splice_read - splice data from file to a pipe
287 * @pipe: pipe to splice to
292 * Will read pages from given file and fill them into a pipe. Can be
297 struct pipe_inode_info *pipe, size_t len, in generic_file_splice_read() argument
304 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len); in generic_file_splice_read()
336 int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe, in generic_pipe_buf_nosteal() argument
342 /* Pipe buffer operations for a socket and similar. */
369 struct pipe_inode_info *pipe, size_t len, in default_file_splice_read() argument
380 if (pipe->nrbufs == pipe->buffers) in default_file_splice_read()
389 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset); in default_file_splice_read()
406 pipe->bufs[to.idx].offset = offset; in default_file_splice_read()
407 pipe->bufs[to.idx].len -= offset; in default_file_splice_read()
437 static int pipe_to_sendpage(struct pipe_inode_info *pipe, in pipe_to_sendpage() argument
449 if (sd->len < sd->total_len && pipe->nrbufs > 1) in pipe_to_sendpage()
456 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) in wakeup_pipe_writers() argument
459 if (waitqueue_active(&pipe->wait)) in wakeup_pipe_writers()
460 wake_up_interruptible(&pipe->wait); in wakeup_pipe_writers()
461 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in wakeup_pipe_writers()
465 * splice_from_pipe_feed - feed available data from a pipe to a file
466 * @pipe: pipe to splice from
471 * This function loops over the pipe and calls @actor to do the
474 * the pipe or if the requested number of bytes (@sd->total_len)
476 * pipe needs to be filled with more data, zero if the required
481 * locking is required around copying the pipe buffers to the
484 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, in splice_from_pipe_feed() argument
489 while (pipe->nrbufs) { in splice_from_pipe_feed()
490 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; in splice_from_pipe_feed()
496 ret = pipe_buf_confirm(pipe, buf); in splice_from_pipe_feed()
503 ret = actor(pipe, buf, sd); in splice_from_pipe_feed()
516 pipe_buf_release(pipe, buf); in splice_from_pipe_feed()
517 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); in splice_from_pipe_feed()
518 pipe->nrbufs--; in splice_from_pipe_feed()
519 if (pipe->files) in splice_from_pipe_feed()
532 * @pipe: pipe to splice from
537 * value (one) if pipe buffers are available. It will return zero
540 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) in splice_from_pipe_next() argument
549 while (!pipe->nrbufs) { in splice_from_pipe_next()
550 if (!pipe->writers) in splice_from_pipe_next()
553 if (!pipe->waiting_writers && sd->num_spliced) in splice_from_pipe_next()
563 wakeup_pipe_writers(pipe); in splice_from_pipe_next()
567 pipe_wait(pipe); in splice_from_pipe_next()
574 * splice_from_pipe_begin - start splicing from pipe
589 * splice_from_pipe_end - finish splicing from pipe
590 * @pipe: pipe to splice from
594 * This function will wake up pipe writers if necessary. It should
598 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) in splice_from_pipe_end() argument
601 wakeup_pipe_writers(pipe); in splice_from_pipe_end()
605 * __splice_from_pipe - splice data from a pipe to given actor
606 * @pipe: pipe to splice from
611 * This function does little more than loop over the pipe and call
617 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, in __splice_from_pipe() argument
625 ret = splice_from_pipe_next(pipe, sd); in __splice_from_pipe()
627 ret = splice_from_pipe_feed(pipe, sd, actor); in __splice_from_pipe()
629 splice_from_pipe_end(pipe, sd); in __splice_from_pipe()
636 * splice_from_pipe - splice data from a pipe to a file
637 * @pipe: pipe to splice from
645 * See __splice_from_pipe. This function locks the pipe inode,
649 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, in splice_from_pipe() argument
661 pipe_lock(pipe); in splice_from_pipe()
662 ret = __splice_from_pipe(pipe, &sd, actor); in splice_from_pipe()
663 pipe_unlock(pipe); in splice_from_pipe()
669 * iter_file_splice_write - splice data from a pipe to a file
670 * @pipe: pipe info
678 * the given pipe inode to the given file.
683 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, in iter_file_splice_write() argument
692 int nbufs = pipe->buffers; in iter_file_splice_write()
700 pipe_lock(pipe); in iter_file_splice_write()
708 ret = splice_from_pipe_next(pipe, &sd); in iter_file_splice_write()
712 if (unlikely(nbufs < pipe->buffers)) { in iter_file_splice_write()
714 nbufs = pipe->buffers; in iter_file_splice_write()
725 for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) { in iter_file_splice_write()
726 struct pipe_buffer *buf = pipe->bufs + idx; in iter_file_splice_write()
732 if (idx == pipe->buffers - 1) in iter_file_splice_write()
735 ret = pipe_buf_confirm(pipe, buf); in iter_file_splice_write()
760 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; in iter_file_splice_write()
764 pipe_buf_release(pipe, buf); in iter_file_splice_write()
765 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); in iter_file_splice_write()
766 pipe->nrbufs--; in iter_file_splice_write()
767 if (pipe->files) in iter_file_splice_write()
778 splice_from_pipe_end(pipe, &sd); in iter_file_splice_write()
780 pipe_unlock(pipe); in iter_file_splice_write()
790 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, in write_pipe_buf() argument
804 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, in default_file_splice_write() argument
810 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); in default_file_splice_write()
818 * generic_splice_sendpage - splice data from a pipe to a socket
819 * @pipe: pipe to splice from
826 * Will send @len bytes from the pipe to a network socket. No data copying
830 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, in generic_splice_sendpage() argument
833 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); in generic_splice_sendpage()
839 * Attempt to initiate a splice from pipe to file.
841 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, in do_splice_from() argument
852 return splice_write(pipe, out, ppos, len, flags); in do_splice_from()
856 * Attempt to initiate a splice from a file to a pipe.
859 struct pipe_inode_info *pipe, size_t len, in do_splice_to() argument
881 return splice_read(in, ppos, pipe, len, flags); in do_splice_to()
892 * points, without requiring an explicit pipe. Internally an allocated
893 * pipe is cached in the process, and reused during the lifetime of
900 struct pipe_inode_info *pipe; in splice_direct_to_actor() local
916 * neither in nor out is a pipe, setup an internal pipe attached to in splice_direct_to_actor()
919 pipe = current->splice_pipe; in splice_direct_to_actor()
920 if (unlikely(!pipe)) { in splice_direct_to_actor()
921 pipe = alloc_pipe_info(); in splice_direct_to_actor()
922 if (!pipe) in splice_direct_to_actor()
927 * out of the pipe right after the splice_to_pipe(). So set in splice_direct_to_actor()
930 pipe->readers = 1; in splice_direct_to_actor()
932 current->splice_pipe = pipe; in splice_direct_to_actor()
944 * Don't block on output, we have to drain the direct pipe. in splice_direct_to_actor()
949 WARN_ON_ONCE(pipe->nrbufs != 0); in splice_direct_to_actor()
956 /* Don't try to read more the pipe has space for. */ in splice_direct_to_actor()
957 pipe_pages = pipe->buffers - pipe->nrbufs; in splice_direct_to_actor()
959 ret = do_splice_to(in, &pos, pipe, read_len, flags); in splice_direct_to_actor()
978 * could get stuck data in the internal pipe: in splice_direct_to_actor()
980 ret = actor(pipe, sd); in splice_direct_to_actor()
997 pipe->nrbufs = pipe->curbuf = 0; in splice_direct_to_actor()
1004 * the pipe buffers in question: in splice_direct_to_actor()
1006 for (i = 0; i < pipe->buffers; i++) { in splice_direct_to_actor()
1007 struct pipe_buffer *buf = pipe->bufs + i; in splice_direct_to_actor()
1010 pipe_buf_release(pipe, buf); in splice_direct_to_actor()
1020 static int direct_splice_actor(struct pipe_inode_info *pipe, in direct_splice_actor() argument
1025 return do_splice_from(pipe, file, sd->opos, sd->total_len, in direct_splice_actor()
1042 * can splice directly through a process-private pipe.
1076 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) in wait_for_space() argument
1079 if (unlikely(!pipe->readers)) { in wait_for_space()
1083 if (pipe->nrbufs != pipe->buffers) in wait_for_space()
1089 pipe->waiting_writers++; in wait_for_space()
1090 pipe_wait(pipe); in wait_for_space()
1091 pipe->waiting_writers--; in wait_for_space()
1182 /* Don't try to read more the pipe has space for. */ in do_splice()
1203 struct pipe_inode_info *pipe, in iter_to_pipe() argument
1232 ret = add_to_pipe(pipe, &buf); in iter_to_pipe()
1248 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, in pipe_to_user() argument
1262 struct pipe_inode_info *pipe = get_pipe_info(file); in vmsplice_to_user() local
1270 if (!pipe) in vmsplice_to_user()
1274 pipe_lock(pipe); in vmsplice_to_user()
1275 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); in vmsplice_to_user()
1276 pipe_unlock(pipe); in vmsplice_to_user()
1283 * vmsplice splices a user address range into a pipe. It can be thought of
1285 * to file). In both cases the output is a pipe, naturally.
1290 struct pipe_inode_info *pipe; in vmsplice_to_pipe() local
1297 pipe = get_pipe_info(file); in vmsplice_to_pipe()
1298 if (!pipe) in vmsplice_to_pipe()
1301 pipe_lock(pipe); in vmsplice_to_pipe()
1302 ret = wait_for_space(pipe, flags); in vmsplice_to_pipe()
1304 ret = iter_to_pipe(iter, pipe, buf_flag); in vmsplice_to_pipe()
1305 pipe_unlock(pipe); in vmsplice_to_pipe()
1307 wakeup_pipe_readers(pipe); in vmsplice_to_pipe()
1328 * to a pipe, not the other way around. Splicing from user memory is a simple
1331 * a pipe. The reverse isn't quite as easy, though. There are two possible
1337 * has restriction limitations on both ends of the pipe).
1443 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) in ipipe_prep() argument
1451 if (pipe->nrbufs) in ipipe_prep()
1455 pipe_lock(pipe); in ipipe_prep()
1457 while (!pipe->nrbufs) { in ipipe_prep()
1462 if (!pipe->writers) in ipipe_prep()
1464 if (!pipe->waiting_writers) { in ipipe_prep()
1470 pipe_wait(pipe); in ipipe_prep()
1473 pipe_unlock(pipe); in ipipe_prep()
1481 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) in opipe_prep() argument
1489 if (pipe->nrbufs < pipe->buffers) in opipe_prep()
1493 pipe_lock(pipe); in opipe_prep()
1495 while (pipe->nrbufs >= pipe->buffers) { in opipe_prep()
1496 if (!pipe->readers) { in opipe_prep()
1509 pipe->waiting_writers++; in opipe_prep()
1510 pipe_wait(pipe); in opipe_prep()
1511 pipe->waiting_writers--; in opipe_prep()
1514 pipe_unlock(pipe); in opipe_prep()
1541 * grabbing by pipe info address. Otherwise two different processes in splice_pipe_to_pipe()
1559 * pipe is empty or the output pipe is full. in splice_pipe_to_pipe()
1597 * Get a reference to this pipe buffer, in splice_pipe_to_pipe()
1628 * If we put data in the output pipe, wakeup any potential readers. in splice_pipe_to_pipe()
1651 * grabbing by pipe info address. Otherwise two different processes in link_pipe()
1675 * Get a reference to this pipe buffer, in link_pipe()
1715 * If we put data in the output pipe, wakeup any potential readers. in link_pipe()
1725 * any data, it simply references the 'in' pages on the 'out' pipe.