372 	size_t off;  in copy_page_to_iter_pipe()  local
384 	off = i->iov_offset;  in copy_page_to_iter_pipe()
387 	if (off) {  in copy_page_to_iter_pipe()
388 		if (offset == off && buf->page == page) {  in copy_page_to_iter_pipe()
485 	size_t off = i->iov_offset;  in data_start()  local
487 	if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {  in data_start()
489 		off = 0;  in data_start()
492 	*offp = off;  in data_start()
499 	size_t off;  in push_pipe()  local
509 	data_start(i, &idx, &off);  in push_pipe()
511 	*offp = off;  in push_pipe()
512 	if (off) {  in push_pipe()
513 		left -= PAGE_SIZE - off;  in push_pipe()
544 	size_t n, off;  in copy_pipe_to_iter()  local
550 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter()
553 	for ( ; n; idx = next_idx(idx, pipe), off = 0) {  in copy_pipe_to_iter()
554 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);  in copy_pipe_to_iter()
555 		memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);  in copy_pipe_to_iter()
557 		i->iov_offset = off + chunk;  in copy_pipe_to_iter()
566 			      __wsum sum, size_t off)  in csum_and_memcpy()  argument
569 	return csum_block_add(sum, next, off);  in csum_and_memcpy()
577 	size_t off = 0;  in csum_and_copy_to_pipe_iter()  local
590 		sum = csum_and_memcpy(p + r, addr, chunk, sum, off);  in csum_and_copy_to_pipe_iter()
595 		off += chunk;  in csum_and_copy_to_pipe_iter()
648 	size_t n, off, xfer = 0;  in copy_pipe_to_iter_mcsafe()  local
654 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter_mcsafe()
657 	for ( ; n; idx = next_idx(idx, pipe), off = 0) {  in copy_pipe_to_iter_mcsafe()
658 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);  in copy_pipe_to_iter_mcsafe()
661 		rem = memcpy_mcsafe_to_page(pipe->bufs[idx].page, off, addr,  in copy_pipe_to_iter_mcsafe()
664 		i->iov_offset = off + chunk - rem;  in copy_pipe_to_iter_mcsafe()
928 	size_t n, off;  in pipe_zero()  local
934 	bytes = n = push_pipe(i, bytes, &idx, &off);  in pipe_zero()
938 	for ( ; n; idx = next_idx(idx, pipe), off = 0) {  in pipe_zero()
939 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);  in pipe_zero()
940 		memzero_page(pipe->bufs[idx].page, off, chunk);  in pipe_zero()
942 		i->iov_offset = off + chunk;  in pipe_zero()
991 		size_t off = i->iov_offset;  in pipe_truncate()  local
994 		if (off) {  in pipe_truncate()
995 			pipe->bufs[idx].len = off - pipe->bufs[idx].offset;  in pipe_truncate()
1014 		size_t off = i->iov_offset, left = size;  in pipe_advance()  local
1016 		if (off) /* make it relative to the beginning of buffer */  in pipe_advance()
1017 			left += off - pipe->bufs[idx].offset;  in pipe_advance()
1057 		size_t off = i->iov_offset;  in iov_iter_revert()  local
1059 			size_t n = off - pipe->bufs[idx].offset;  in iov_iter_revert()
1061 				off -= unroll;  in iov_iter_revert()
1066 				off = 0;  in iov_iter_revert()
1071 			off = pipe->bufs[idx].offset + pipe->bufs[idx].len;  in iov_iter_revert()
1073 		i->iov_offset = off;  in iov_iter_revert()
1409 	size_t off = 0;  in csum_and_copy_from_iter()  local
1421 			sum = csum_block_add(sum, next, off);  in csum_and_copy_from_iter()
1422 			off += v.iov_len;  in csum_and_copy_from_iter()
1429 				      sum, off);  in csum_and_copy_from_iter()
1431 		off += v.bv_len;  in csum_and_copy_from_iter()
1435 				      sum, off);  in csum_and_copy_from_iter()
1436 		off += v.iov_len;  in csum_and_copy_from_iter()
1449 	size_t off = 0;  in csum_and_copy_from_iter_full()  local
1464 		sum = csum_block_add(sum, next, off);  in csum_and_copy_from_iter_full()
1465 		off += v.iov_len;  in csum_and_copy_from_iter_full()
1471 				      sum, off);  in csum_and_copy_from_iter_full()
1473 		off += v.bv_len;  in csum_and_copy_from_iter_full()
1477 				      sum, off);  in csum_and_copy_from_iter_full()
1478 		off += v.iov_len;  in csum_and_copy_from_iter_full()
1493 	size_t off = 0;  in csum_and_copy_to_iter()  local
1509 			sum = csum_block_add(sum, next, off);  in csum_and_copy_to_iter()
1510 			off += v.iov_len;  in csum_and_copy_to_iter()
1517 				      v.bv_len, sum, off);  in csum_and_copy_to_iter()
1519 		off += v.bv_len;  in csum_and_copy_to_iter()
1523 				     v.iov_len, sum, off);  in csum_and_copy_to_iter()
1524 		off += v.iov_len;  in csum_and_copy_to_iter()
1563 		size_t off;  in iov_iter_npages()  local
1569 		data_start(i, &idx, &off);  in iov_iter_npages()