• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/read_write.c
4  *
5  *  Copyright (C) 1991, 1992  Linus Torvalds
6  */
7 
8 #include <linux/slab.h>
9 #include <linux/stat.h>
10 #include <linux/sched/xacct.h>
11 #include <linux/fcntl.h>
12 #include <linux/file.h>
13 #include <linux/uio.h>
14 #include <linux/fsnotify.h>
15 #include <linux/security.h>
16 #include <linux/export.h>
17 #include <linux/syscalls.h>
18 #include <linux/pagemap.h>
19 #include <linux/splice.h>
20 #include <linux/compat.h>
21 #include <linux/mount.h>
22 #include <linux/fs.h>
23 #include "internal.h"
24 
25 #include <linux/uaccess.h>
26 #include <asm/unistd.h>
27 
28 const struct file_operations generic_ro_fops = {
29 	.llseek		= generic_file_llseek,
30 	.read_iter	= generic_file_read_iter,
31 	.mmap		= generic_file_readonly_mmap,
32 	.splice_read	= generic_file_splice_read,
33 };
34 
35 EXPORT_SYMBOL(generic_ro_fops);
36 
37 /**
38  * vfs_setpos - update the file offset for lseek
39  * @file:	file structure in question
40  * @offset:	file offset to seek to
41  * @maxsize:	maximum file size
42  *
43  * This is a low-level filesystem helper for updating the file offset to
44  * the value specified by @offset if the given offset is valid and it is
45  * not equal to the current file offset.
46  *
47  * Return the specified offset on success and -EINVAL on invalid offset.
48  */
vfs_setpos(struct file * file,loff_t offset,loff_t maxsize)49 loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
50 {
51 	if (offset < 0 && !unsigned_offsets(file))
52 		return -EINVAL;
53 	if (offset > maxsize)
54 		return -EINVAL;
55 
56 	if (offset != file->f_pos) {
57 		file->f_pos = offset;
58 		file->f_version = 0;
59 	}
60 	return offset;
61 }
62 EXPORT_SYMBOL(vfs_setpos);
63 
64 /**
65  * generic_file_llseek_size - generic llseek implementation for regular files
66  * @file:	file structure to seek on
67  * @offset:	file offset to seek to
68  * @whence:	type of seek
69  * @size:	max size of this file in file system
70  * @eof:	offset used for SEEK_END position
71  *
72  * This is a variant of generic_file_llseek that allows passing in a custom
73  * maximum file size and a custom EOF position, for e.g. hashed directories
74  *
75  * Synchronization:
76  * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
77  * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
78  * read/writes behave like SEEK_SET against seeks.
79  */
80 loff_t
generic_file_llseek_size(struct file * file,loff_t offset,int whence,loff_t maxsize,loff_t eof)81 generic_file_llseek_size(struct file *file, loff_t offset, int whence,
82 		loff_t maxsize, loff_t eof)
83 {
84 	switch (whence) {
85 	case SEEK_END:
86 		offset += eof;
87 		break;
88 	case SEEK_CUR:
89 		/*
90 		 * Here we special-case the lseek(fd, 0, SEEK_CUR)
91 		 * position-querying operation.  Avoid rewriting the "same"
92 		 * f_pos value back to the file because a concurrent read(),
93 		 * write() or lseek() might have altered it
94 		 */
95 		if (offset == 0)
96 			return file->f_pos;
97 		/*
98 		 * f_lock protects against read/modify/write race with other
99 		 * SEEK_CURs. Note that parallel writes and reads behave
100 		 * like SEEK_SET.
101 		 */
102 		spin_lock(&file->f_lock);
103 		offset = vfs_setpos(file, file->f_pos + offset, maxsize);
104 		spin_unlock(&file->f_lock);
105 		return offset;
106 	case SEEK_DATA:
107 		/*
108 		 * In the generic case the entire file is data, so as long as
109 		 * offset isn't at the end of the file then the offset is data.
110 		 */
111 		if ((unsigned long long)offset >= eof)
112 			return -ENXIO;
113 		break;
114 	case SEEK_HOLE:
115 		/*
116 		 * There is a virtual hole at the end of the file, so as long as
117 		 * offset isn't i_size or larger, return i_size.
118 		 */
119 		if ((unsigned long long)offset >= eof)
120 			return -ENXIO;
121 		offset = eof;
122 		break;
123 	}
124 
125 	return vfs_setpos(file, offset, maxsize);
126 }
127 EXPORT_SYMBOL(generic_file_llseek_size);
128 
129 /**
130  * generic_file_llseek - generic llseek implementation for regular files
131  * @file:	file structure to seek on
132  * @offset:	file offset to seek to
133  * @whence:	type of seek
134  *
135  * This is a generic implemenation of ->llseek useable for all normal local
136  * filesystems.  It just updates the file offset to the value specified by
137  * @offset and @whence.
138  */
generic_file_llseek(struct file * file,loff_t offset,int whence)139 loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
140 {
141 	struct inode *inode = file->f_mapping->host;
142 
143 	return generic_file_llseek_size(file, offset, whence,
144 					inode->i_sb->s_maxbytes,
145 					i_size_read(inode));
146 }
147 EXPORT_SYMBOL(generic_file_llseek);
148 
149 /**
150  * fixed_size_llseek - llseek implementation for fixed-sized devices
151  * @file:	file structure to seek on
152  * @offset:	file offset to seek to
153  * @whence:	type of seek
154  * @size:	size of the file
155  *
156  */
fixed_size_llseek(struct file * file,loff_t offset,int whence,loff_t size)157 loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
158 {
159 	switch (whence) {
160 	case SEEK_SET: case SEEK_CUR: case SEEK_END:
161 		return generic_file_llseek_size(file, offset, whence,
162 						size, size);
163 	default:
164 		return -EINVAL;
165 	}
166 }
167 EXPORT_SYMBOL(fixed_size_llseek);
168 
169 /**
170  * no_seek_end_llseek - llseek implementation for fixed-sized devices
171  * @file:	file structure to seek on
172  * @offset:	file offset to seek to
173  * @whence:	type of seek
174  *
175  */
no_seek_end_llseek(struct file * file,loff_t offset,int whence)176 loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
177 {
178 	switch (whence) {
179 	case SEEK_SET: case SEEK_CUR:
180 		return generic_file_llseek_size(file, offset, whence,
181 						OFFSET_MAX, 0);
182 	default:
183 		return -EINVAL;
184 	}
185 }
186 EXPORT_SYMBOL(no_seek_end_llseek);
187 
188 /**
189  * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
190  * @file:	file structure to seek on
191  * @offset:	file offset to seek to
192  * @whence:	type of seek
193  * @size:	maximal offset allowed
194  *
195  */
no_seek_end_llseek_size(struct file * file,loff_t offset,int whence,loff_t size)196 loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
197 {
198 	switch (whence) {
199 	case SEEK_SET: case SEEK_CUR:
200 		return generic_file_llseek_size(file, offset, whence,
201 						size, 0);
202 	default:
203 		return -EINVAL;
204 	}
205 }
206 EXPORT_SYMBOL(no_seek_end_llseek_size);
207 
208 /**
209  * noop_llseek - No Operation Performed llseek implementation
210  * @file:	file structure to seek on
211  * @offset:	file offset to seek to
212  * @whence:	type of seek
213  *
214  * This is an implementation of ->llseek useable for the rare special case when
215  * userspace expects the seek to succeed but the (device) file is actually not
216  * able to perform the seek. In this case you use noop_llseek() instead of
217  * falling back to the default implementation of ->llseek.
218  */
noop_llseek(struct file * file,loff_t offset,int whence)219 loff_t noop_llseek(struct file *file, loff_t offset, int whence)
220 {
221 	return file->f_pos;
222 }
223 EXPORT_SYMBOL(noop_llseek);
224 
no_llseek(struct file * file,loff_t offset,int whence)225 loff_t no_llseek(struct file *file, loff_t offset, int whence)
226 {
227 	return -ESPIPE;
228 }
229 EXPORT_SYMBOL(no_llseek);
230 
default_llseek(struct file * file,loff_t offset,int whence)231 loff_t default_llseek(struct file *file, loff_t offset, int whence)
232 {
233 	struct inode *inode = file_inode(file);
234 	loff_t retval;
235 
236 	inode_lock(inode);
237 	switch (whence) {
238 		case SEEK_END:
239 			offset += i_size_read(inode);
240 			break;
241 		case SEEK_CUR:
242 			if (offset == 0) {
243 				retval = file->f_pos;
244 				goto out;
245 			}
246 			offset += file->f_pos;
247 			break;
248 		case SEEK_DATA:
249 			/*
250 			 * In the generic case the entire file is data, so as
251 			 * long as offset isn't at the end of the file then the
252 			 * offset is data.
253 			 */
254 			if (offset >= inode->i_size) {
255 				retval = -ENXIO;
256 				goto out;
257 			}
258 			break;
259 		case SEEK_HOLE:
260 			/*
261 			 * There is a virtual hole at the end of the file, so
262 			 * as long as offset isn't i_size or larger, return
263 			 * i_size.
264 			 */
265 			if (offset >= inode->i_size) {
266 				retval = -ENXIO;
267 				goto out;
268 			}
269 			offset = inode->i_size;
270 			break;
271 	}
272 	retval = -EINVAL;
273 	if (offset >= 0 || unsigned_offsets(file)) {
274 		if (offset != file->f_pos) {
275 			file->f_pos = offset;
276 			file->f_version = 0;
277 		}
278 		retval = offset;
279 	}
280 out:
281 	inode_unlock(inode);
282 	return retval;
283 }
284 EXPORT_SYMBOL(default_llseek);
285 
vfs_llseek(struct file * file,loff_t offset,int whence)286 loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
287 {
288 	loff_t (*fn)(struct file *, loff_t, int);
289 
290 	fn = no_llseek;
291 	if (file->f_mode & FMODE_LSEEK) {
292 		if (file->f_op->llseek)
293 			fn = file->f_op->llseek;
294 	}
295 	return fn(file, offset, whence);
296 }
297 EXPORT_SYMBOL(vfs_llseek);
298 
ksys_lseek(unsigned int fd,off_t offset,unsigned int whence)299 static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
300 {
301 	off_t retval;
302 	struct fd f = fdget_pos(fd);
303 	if (!f.file)
304 		return -EBADF;
305 
306 	retval = -EINVAL;
307 	if (whence <= SEEK_MAX) {
308 		loff_t res = vfs_llseek(f.file, offset, whence);
309 		retval = res;
310 		if (res != (loff_t)retval)
311 			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
312 	}
313 	fdput_pos(f);
314 	return retval;
315 }
316 
SYSCALL_DEFINE3(lseek,unsigned int,fd,off_t,offset,unsigned int,whence)317 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
318 {
319 	return ksys_lseek(fd, offset, whence);
320 }
321 
322 #ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(lseek,unsigned int,fd,compat_off_t,offset,unsigned int,whence)323 COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
324 {
325 	return ksys_lseek(fd, offset, whence);
326 }
327 #endif
328 
329 #if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
330 	defined(__ARCH_WANT_SYS_LLSEEK)
SYSCALL_DEFINE5(llseek,unsigned int,fd,unsigned long,offset_high,unsigned long,offset_low,loff_t __user *,result,unsigned int,whence)331 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
332 		unsigned long, offset_low, loff_t __user *, result,
333 		unsigned int, whence)
334 {
335 	int retval;
336 	struct fd f = fdget_pos(fd);
337 	loff_t offset;
338 
339 	if (!f.file)
340 		return -EBADF;
341 
342 	retval = -EINVAL;
343 	if (whence > SEEK_MAX)
344 		goto out_putf;
345 
346 	offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
347 			whence);
348 
349 	retval = (int)offset;
350 	if (offset >= 0) {
351 		retval = -EFAULT;
352 		if (!copy_to_user(result, &offset, sizeof(offset)))
353 			retval = 0;
354 	}
355 out_putf:
356 	fdput_pos(f);
357 	return retval;
358 }
359 #endif
360 
rw_verify_area(int read_write,struct file * file,const loff_t * ppos,size_t count)361 int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
362 {
363 	struct inode *inode;
364 	int retval = -EINVAL;
365 
366 	inode = file_inode(file);
367 	if (unlikely((ssize_t) count < 0))
368 		return retval;
369 
370 	/*
371 	 * ranged mandatory locking does not apply to streams - it makes sense
372 	 * only for files where position has a meaning.
373 	 */
374 	if (ppos) {
375 		loff_t pos = *ppos;
376 
377 		if (unlikely(pos < 0)) {
378 			if (!unsigned_offsets(file))
379 				return retval;
380 			if (count >= -pos) /* both values are in 0..LLONG_MAX */
381 				return -EOVERFLOW;
382 		} else if (unlikely((loff_t) (pos + count) < 0)) {
383 			if (!unsigned_offsets(file))
384 				return retval;
385 		}
386 
387 		if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
388 			retval = locks_mandatory_area(inode, file, pos, pos + count - 1,
389 					read_write == READ ? F_RDLCK : F_WRLCK);
390 			if (retval < 0)
391 				return retval;
392 		}
393 	}
394 
395 	return security_file_permission(file,
396 				read_write == READ ? MAY_READ : MAY_WRITE);
397 }
398 
new_sync_read(struct file * filp,char __user * buf,size_t len,loff_t * ppos)399 static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
400 {
401 	struct iovec iov = { .iov_base = buf, .iov_len = len };
402 	struct kiocb kiocb;
403 	struct iov_iter iter;
404 	ssize_t ret;
405 
406 	init_sync_kiocb(&kiocb, filp);
407 	kiocb.ki_pos = (ppos ? *ppos : 0);
408 	iov_iter_init(&iter, READ, &iov, 1, len);
409 
410 	ret = call_read_iter(filp, &kiocb, &iter);
411 	BUG_ON(ret == -EIOCBQUEUED);
412 	if (ppos)
413 		*ppos = kiocb.ki_pos;
414 	return ret;
415 }
416 
warn_unsupported(struct file * file,const char * op)417 static int warn_unsupported(struct file *file, const char *op)
418 {
419 	pr_warn_ratelimited(
420 		"kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
421 		op, file, current->pid, current->comm);
422 	return -EINVAL;
423 }
424 
__kernel_read(struct file * file,void * buf,size_t count,loff_t * pos)425 ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
426 {
427 	struct kvec iov = {
428 		.iov_base	= buf,
429 		.iov_len	= min_t(size_t, count, MAX_RW_COUNT),
430 	};
431 	struct kiocb kiocb;
432 	struct iov_iter iter;
433 	ssize_t ret;
434 
435 	if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
436 		return -EINVAL;
437 	if (!(file->f_mode & FMODE_CAN_READ))
438 		return -EINVAL;
439 	/*
440 	 * Also fail if ->read_iter and ->read are both wired up as that
441 	 * implies very convoluted semantics.
442 	 */
443 	if (unlikely(!file->f_op->read_iter || file->f_op->read))
444 		return warn_unsupported(file, "read");
445 
446 	init_sync_kiocb(&kiocb, file);
447 	kiocb.ki_pos = pos ? *pos : 0;
448 	iov_iter_kvec(&iter, READ, &iov, 1, iov.iov_len);
449 	ret = file->f_op->read_iter(&kiocb, &iter);
450 	if (ret > 0) {
451 		if (pos)
452 			*pos = kiocb.ki_pos;
453 		fsnotify_access(file);
454 		add_rchar(current, ret);
455 	}
456 	inc_syscr(current);
457 	return ret;
458 }
459 
kernel_read(struct file * file,void * buf,size_t count,loff_t * pos)460 ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
461 {
462 	ssize_t ret;
463 
464 	ret = rw_verify_area(READ, file, pos, count);
465 	if (ret)
466 		return ret;
467 	return __kernel_read(file, buf, count, pos);
468 }
469 EXPORT_SYMBOL(kernel_read);
470 
vfs_read(struct file * file,char __user * buf,size_t count,loff_t * pos)471 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
472 {
473 	ssize_t ret;
474 
475 	if (!(file->f_mode & FMODE_READ))
476 		return -EBADF;
477 	if (!(file->f_mode & FMODE_CAN_READ))
478 		return -EINVAL;
479 	if (unlikely(!access_ok(buf, count)))
480 		return -EFAULT;
481 
482 	ret = rw_verify_area(READ, file, pos, count);
483 	if (ret)
484 		return ret;
485 	if (count > MAX_RW_COUNT)
486 		count =  MAX_RW_COUNT;
487 
488 	if (file->f_op->read)
489 		ret = file->f_op->read(file, buf, count, pos);
490 	else if (file->f_op->read_iter)
491 		ret = new_sync_read(file, buf, count, pos);
492 	else
493 		ret = -EINVAL;
494 	if (ret > 0) {
495 		fsnotify_access(file);
496 		add_rchar(current, ret);
497 	}
498 	inc_syscr(current);
499 	return ret;
500 }
501 
new_sync_write(struct file * filp,const char __user * buf,size_t len,loff_t * ppos)502 static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
503 {
504 	struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
505 	struct kiocb kiocb;
506 	struct iov_iter iter;
507 	ssize_t ret;
508 
509 	init_sync_kiocb(&kiocb, filp);
510 	kiocb.ki_pos = (ppos ? *ppos : 0);
511 	iov_iter_init(&iter, WRITE, &iov, 1, len);
512 
513 	ret = call_write_iter(filp, &kiocb, &iter);
514 	BUG_ON(ret == -EIOCBQUEUED);
515 	if (ret > 0 && ppos)
516 		*ppos = kiocb.ki_pos;
517 	return ret;
518 }
519 
520 /* caller is responsible for file_start_write/file_end_write */
__kernel_write(struct file * file,const void * buf,size_t count,loff_t * pos)521 ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
522 {
523 	struct kvec iov = {
524 		.iov_base	= (void *)buf,
525 		.iov_len	= min_t(size_t, count, MAX_RW_COUNT),
526 	};
527 	struct kiocb kiocb;
528 	struct iov_iter iter;
529 	ssize_t ret;
530 
531 	if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
532 		return -EBADF;
533 	if (!(file->f_mode & FMODE_CAN_WRITE))
534 		return -EINVAL;
535 	/*
536 	 * Also fail if ->write_iter and ->write are both wired up as that
537 	 * implies very convoluted semantics.
538 	 */
539 	if (unlikely(!file->f_op->write_iter || file->f_op->write))
540 		return warn_unsupported(file, "write");
541 
542 	init_sync_kiocb(&kiocb, file);
543 	kiocb.ki_pos = pos ? *pos : 0;
544 	iov_iter_kvec(&iter, WRITE, &iov, 1, iov.iov_len);
545 	ret = file->f_op->write_iter(&kiocb, &iter);
546 	if (ret > 0) {
547 		if (pos)
548 			*pos = kiocb.ki_pos;
549 		fsnotify_modify(file);
550 		add_wchar(current, ret);
551 	}
552 	inc_syscw(current);
553 	return ret;
554 }
555 /*
556  * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
557  * but autofs is one of the few internal kernel users that actually
558  * wants this _and_ can be built as a module. So we need to export
559  * this symbol for autofs, even though it really isn't appropriate
560  * for any other kernel modules.
561  */
562 EXPORT_SYMBOL_GPL(__kernel_write);
563 
kernel_write(struct file * file,const void * buf,size_t count,loff_t * pos)564 ssize_t kernel_write(struct file *file, const void *buf, size_t count,
565 			    loff_t *pos)
566 {
567 	ssize_t ret;
568 
569 	ret = rw_verify_area(WRITE, file, pos, count);
570 	if (ret)
571 		return ret;
572 
573 	file_start_write(file);
574 	ret =  __kernel_write(file, buf, count, pos);
575 	file_end_write(file);
576 	return ret;
577 }
578 EXPORT_SYMBOL(kernel_write);
579 
vfs_write(struct file * file,const char __user * buf,size_t count,loff_t * pos)580 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
581 {
582 	ssize_t ret;
583 
584 	if (!(file->f_mode & FMODE_WRITE))
585 		return -EBADF;
586 	if (!(file->f_mode & FMODE_CAN_WRITE))
587 		return -EINVAL;
588 	if (unlikely(!access_ok(buf, count)))
589 		return -EFAULT;
590 
591 	ret = rw_verify_area(WRITE, file, pos, count);
592 	if (ret)
593 		return ret;
594 	if (count > MAX_RW_COUNT)
595 		count =  MAX_RW_COUNT;
596 	file_start_write(file);
597 	if (file->f_op->write)
598 		ret = file->f_op->write(file, buf, count, pos);
599 	else if (file->f_op->write_iter)
600 		ret = new_sync_write(file, buf, count, pos);
601 	else
602 		ret = -EINVAL;
603 	if (ret > 0) {
604 		fsnotify_modify(file);
605 		add_wchar(current, ret);
606 	}
607 	inc_syscw(current);
608 	file_end_write(file);
609 	return ret;
610 }
611 
612 /* file_ppos returns &file->f_pos or NULL if file is stream */
file_ppos(struct file * file)613 static inline loff_t *file_ppos(struct file *file)
614 {
615 	return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
616 }
617 
ksys_read(unsigned int fd,char __user * buf,size_t count)618 ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
619 {
620 	struct fd f = fdget_pos(fd);
621 	ssize_t ret = -EBADF;
622 
623 	if (f.file) {
624 		loff_t pos, *ppos = file_ppos(f.file);
625 		if (ppos) {
626 			pos = *ppos;
627 			ppos = &pos;
628 		}
629 		ret = vfs_read(f.file, buf, count, ppos);
630 		if (ret >= 0 && ppos)
631 			f.file->f_pos = pos;
632 		fdput_pos(f);
633 	}
634 	return ret;
635 }
636 
SYSCALL_DEFINE3(read,unsigned int,fd,char __user *,buf,size_t,count)637 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
638 {
639 	return ksys_read(fd, buf, count);
640 }
641 
ksys_write(unsigned int fd,const char __user * buf,size_t count)642 ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
643 {
644 	struct fd f = fdget_pos(fd);
645 	ssize_t ret = -EBADF;
646 
647 	if (f.file) {
648 		loff_t pos, *ppos = file_ppos(f.file);
649 		if (ppos) {
650 			pos = *ppos;
651 			ppos = &pos;
652 		}
653 		ret = vfs_write(f.file, buf, count, ppos);
654 		if (ret >= 0 && ppos)
655 			f.file->f_pos = pos;
656 		fdput_pos(f);
657 	}
658 
659 	return ret;
660 }
661 
SYSCALL_DEFINE3(write,unsigned int,fd,const char __user *,buf,size_t,count)662 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
663 		size_t, count)
664 {
665 	return ksys_write(fd, buf, count);
666 }
667 
ksys_pread64(unsigned int fd,char __user * buf,size_t count,loff_t pos)668 ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
669 		     loff_t pos)
670 {
671 	struct fd f;
672 	ssize_t ret = -EBADF;
673 
674 	if (pos < 0)
675 		return -EINVAL;
676 
677 	f = fdget(fd);
678 	if (f.file) {
679 		ret = -ESPIPE;
680 		if (f.file->f_mode & FMODE_PREAD)
681 			ret = vfs_read(f.file, buf, count, &pos);
682 		fdput(f);
683 	}
684 
685 	return ret;
686 }
687 
SYSCALL_DEFINE4(pread64,unsigned int,fd,char __user *,buf,size_t,count,loff_t,pos)688 SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
689 			size_t, count, loff_t, pos)
690 {
691 	return ksys_pread64(fd, buf, count, pos);
692 }
693 
ksys_pwrite64(unsigned int fd,const char __user * buf,size_t count,loff_t pos)694 ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
695 		      size_t count, loff_t pos)
696 {
697 	struct fd f;
698 	ssize_t ret = -EBADF;
699 
700 	if (pos < 0)
701 		return -EINVAL;
702 
703 	f = fdget(fd);
704 	if (f.file) {
705 		ret = -ESPIPE;
706 		if (f.file->f_mode & FMODE_PWRITE)
707 			ret = vfs_write(f.file, buf, count, &pos);
708 		fdput(f);
709 	}
710 
711 	return ret;
712 }
713 
SYSCALL_DEFINE4(pwrite64,unsigned int,fd,const char __user *,buf,size_t,count,loff_t,pos)714 SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
715 			 size_t, count, loff_t, pos)
716 {
717 	return ksys_pwrite64(fd, buf, count, pos);
718 }
719 
do_iter_readv_writev(struct file * filp,struct iov_iter * iter,loff_t * ppos,int type,rwf_t flags)720 static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
721 		loff_t *ppos, int type, rwf_t flags)
722 {
723 	struct kiocb kiocb;
724 	ssize_t ret;
725 
726 	init_sync_kiocb(&kiocb, filp);
727 	ret = kiocb_set_rw_flags(&kiocb, flags);
728 	if (ret)
729 		return ret;
730 	kiocb.ki_pos = (ppos ? *ppos : 0);
731 
732 	if (type == READ)
733 		ret = call_read_iter(filp, &kiocb, iter);
734 	else
735 		ret = call_write_iter(filp, &kiocb, iter);
736 	BUG_ON(ret == -EIOCBQUEUED);
737 	if (ppos)
738 		*ppos = kiocb.ki_pos;
739 	return ret;
740 }
741 
742 /* Do it by hand, with file-ops */
do_loop_readv_writev(struct file * filp,struct iov_iter * iter,loff_t * ppos,int type,rwf_t flags)743 static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
744 		loff_t *ppos, int type, rwf_t flags)
745 {
746 	ssize_t ret = 0;
747 
748 	if (flags & ~RWF_HIPRI)
749 		return -EOPNOTSUPP;
750 
751 	while (iov_iter_count(iter)) {
752 		struct iovec iovec = iov_iter_iovec(iter);
753 		ssize_t nr;
754 
755 		if (type == READ) {
756 			nr = filp->f_op->read(filp, iovec.iov_base,
757 					      iovec.iov_len, ppos);
758 		} else {
759 			nr = filp->f_op->write(filp, iovec.iov_base,
760 					       iovec.iov_len, ppos);
761 		}
762 
763 		if (nr < 0) {
764 			if (!ret)
765 				ret = nr;
766 			break;
767 		}
768 		ret += nr;
769 		if (nr != iovec.iov_len)
770 			break;
771 		iov_iter_advance(iter, nr);
772 	}
773 
774 	return ret;
775 }
776 
do_iter_read(struct file * file,struct iov_iter * iter,loff_t * pos,rwf_t flags)777 static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
778 		loff_t *pos, rwf_t flags)
779 {
780 	size_t tot_len;
781 	ssize_t ret = 0;
782 
783 	if (!(file->f_mode & FMODE_READ))
784 		return -EBADF;
785 	if (!(file->f_mode & FMODE_CAN_READ))
786 		return -EINVAL;
787 
788 	tot_len = iov_iter_count(iter);
789 	if (!tot_len)
790 		goto out;
791 	ret = rw_verify_area(READ, file, pos, tot_len);
792 	if (ret < 0)
793 		return ret;
794 
795 	if (file->f_op->read_iter)
796 		ret = do_iter_readv_writev(file, iter, pos, READ, flags);
797 	else
798 		ret = do_loop_readv_writev(file, iter, pos, READ, flags);
799 out:
800 	if (ret >= 0)
801 		fsnotify_access(file);
802 	return ret;
803 }
804 
vfs_iocb_iter_read(struct file * file,struct kiocb * iocb,struct iov_iter * iter)805 ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
806 			   struct iov_iter *iter)
807 {
808 	size_t tot_len;
809 	ssize_t ret = 0;
810 
811 	if (!file->f_op->read_iter)
812 		return -EINVAL;
813 	if (!(file->f_mode & FMODE_READ))
814 		return -EBADF;
815 	if (!(file->f_mode & FMODE_CAN_READ))
816 		return -EINVAL;
817 
818 	tot_len = iov_iter_count(iter);
819 	if (!tot_len)
820 		goto out;
821 	ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
822 	if (ret < 0)
823 		return ret;
824 
825 	ret = call_read_iter(file, iocb, iter);
826 out:
827 	if (ret >= 0)
828 		fsnotify_access(file);
829 	return ret;
830 }
831 EXPORT_SYMBOL(vfs_iocb_iter_read);
832 
vfs_iter_read(struct file * file,struct iov_iter * iter,loff_t * ppos,rwf_t flags)833 ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
834 		rwf_t flags)
835 {
836 	if (!file->f_op->read_iter)
837 		return -EINVAL;
838 	return do_iter_read(file, iter, ppos, flags);
839 }
840 EXPORT_SYMBOL(vfs_iter_read);
841 
do_iter_write(struct file * file,struct iov_iter * iter,loff_t * pos,rwf_t flags)842 static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
843 		loff_t *pos, rwf_t flags)
844 {
845 	size_t tot_len;
846 	ssize_t ret = 0;
847 
848 	if (!(file->f_mode & FMODE_WRITE))
849 		return -EBADF;
850 	if (!(file->f_mode & FMODE_CAN_WRITE))
851 		return -EINVAL;
852 
853 	tot_len = iov_iter_count(iter);
854 	if (!tot_len)
855 		return 0;
856 	ret = rw_verify_area(WRITE, file, pos, tot_len);
857 	if (ret < 0)
858 		return ret;
859 
860 	if (file->f_op->write_iter)
861 		ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
862 	else
863 		ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
864 	if (ret > 0)
865 		fsnotify_modify(file);
866 	return ret;
867 }
868 
vfs_iocb_iter_write(struct file * file,struct kiocb * iocb,struct iov_iter * iter)869 ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
870 			    struct iov_iter *iter)
871 {
872 	size_t tot_len;
873 	ssize_t ret = 0;
874 
875 	if (!file->f_op->write_iter)
876 		return -EINVAL;
877 	if (!(file->f_mode & FMODE_WRITE))
878 		return -EBADF;
879 	if (!(file->f_mode & FMODE_CAN_WRITE))
880 		return -EINVAL;
881 
882 	tot_len = iov_iter_count(iter);
883 	if (!tot_len)
884 		return 0;
885 	ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
886 	if (ret < 0)
887 		return ret;
888 
889 	ret = call_write_iter(file, iocb, iter);
890 	if (ret > 0)
891 		fsnotify_modify(file);
892 
893 	return ret;
894 }
895 EXPORT_SYMBOL(vfs_iocb_iter_write);
896 
vfs_iter_write(struct file * file,struct iov_iter * iter,loff_t * ppos,rwf_t flags)897 ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
898 		rwf_t flags)
899 {
900 	if (!file->f_op->write_iter)
901 		return -EINVAL;
902 	return do_iter_write(file, iter, ppos, flags);
903 }
904 EXPORT_SYMBOL(vfs_iter_write);
905 
vfs_readv(struct file * file,const struct iovec __user * vec,unsigned long vlen,loff_t * pos,rwf_t flags)906 static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
907 		  unsigned long vlen, loff_t *pos, rwf_t flags)
908 {
909 	struct iovec iovstack[UIO_FASTIOV];
910 	struct iovec *iov = iovstack;
911 	struct iov_iter iter;
912 	ssize_t ret;
913 
914 	ret = import_iovec(READ, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
915 	if (ret >= 0) {
916 		ret = do_iter_read(file, &iter, pos, flags);
917 		kfree(iov);
918 	}
919 
920 	return ret;
921 }
922 
vfs_writev(struct file * file,const struct iovec __user * vec,unsigned long vlen,loff_t * pos,rwf_t flags)923 static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
924 		   unsigned long vlen, loff_t *pos, rwf_t flags)
925 {
926 	struct iovec iovstack[UIO_FASTIOV];
927 	struct iovec *iov = iovstack;
928 	struct iov_iter iter;
929 	ssize_t ret;
930 
931 	ret = import_iovec(WRITE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
932 	if (ret >= 0) {
933 		file_start_write(file);
934 		ret = do_iter_write(file, &iter, pos, flags);
935 		file_end_write(file);
936 		kfree(iov);
937 	}
938 	return ret;
939 }
940 
do_readv(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,rwf_t flags)941 static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
942 			unsigned long vlen, rwf_t flags)
943 {
944 	struct fd f = fdget_pos(fd);
945 	ssize_t ret = -EBADF;
946 
947 	if (f.file) {
948 		loff_t pos, *ppos = file_ppos(f.file);
949 		if (ppos) {
950 			pos = *ppos;
951 			ppos = &pos;
952 		}
953 		ret = vfs_readv(f.file, vec, vlen, ppos, flags);
954 		if (ret >= 0 && ppos)
955 			f.file->f_pos = pos;
956 		fdput_pos(f);
957 	}
958 
959 	if (ret > 0)
960 		add_rchar(current, ret);
961 	inc_syscr(current);
962 	return ret;
963 }
964 
do_writev(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,rwf_t flags)965 static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
966 			 unsigned long vlen, rwf_t flags)
967 {
968 	struct fd f = fdget_pos(fd);
969 	ssize_t ret = -EBADF;
970 
971 	if (f.file) {
972 		loff_t pos, *ppos = file_ppos(f.file);
973 		if (ppos) {
974 			pos = *ppos;
975 			ppos = &pos;
976 		}
977 		ret = vfs_writev(f.file, vec, vlen, ppos, flags);
978 		if (ret >= 0 && ppos)
979 			f.file->f_pos = pos;
980 		fdput_pos(f);
981 	}
982 
983 	if (ret > 0)
984 		add_wchar(current, ret);
985 	inc_syscw(current);
986 	return ret;
987 }
988 
pos_from_hilo(unsigned long high,unsigned long low)989 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
990 {
991 #define HALF_LONG_BITS (BITS_PER_LONG / 2)
992 	return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
993 }
994 
do_preadv(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,loff_t pos,rwf_t flags)995 static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
996 			 unsigned long vlen, loff_t pos, rwf_t flags)
997 {
998 	struct fd f;
999 	ssize_t ret = -EBADF;
1000 
1001 	if (pos < 0)
1002 		return -EINVAL;
1003 
1004 	f = fdget(fd);
1005 	if (f.file) {
1006 		ret = -ESPIPE;
1007 		if (f.file->f_mode & FMODE_PREAD)
1008 			ret = vfs_readv(f.file, vec, vlen, &pos, flags);
1009 		fdput(f);
1010 	}
1011 
1012 	if (ret > 0)
1013 		add_rchar(current, ret);
1014 	inc_syscr(current);
1015 	return ret;
1016 }
1017 
do_pwritev(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,loff_t pos,rwf_t flags)1018 static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1019 			  unsigned long vlen, loff_t pos, rwf_t flags)
1020 {
1021 	struct fd f;
1022 	ssize_t ret = -EBADF;
1023 
1024 	if (pos < 0)
1025 		return -EINVAL;
1026 
1027 	f = fdget(fd);
1028 	if (f.file) {
1029 		ret = -ESPIPE;
1030 		if (f.file->f_mode & FMODE_PWRITE)
1031 			ret = vfs_writev(f.file, vec, vlen, &pos, flags);
1032 		fdput(f);
1033 	}
1034 
1035 	if (ret > 0)
1036 		add_wchar(current, ret);
1037 	inc_syscw(current);
1038 	return ret;
1039 }
1040 
SYSCALL_DEFINE3(readv,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen)1041 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1042 		unsigned long, vlen)
1043 {
1044 	return do_readv(fd, vec, vlen, 0);
1045 }
1046 
SYSCALL_DEFINE3(writev,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen)1047 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1048 		unsigned long, vlen)
1049 {
1050 	return do_writev(fd, vec, vlen, 0);
1051 }
1052 
SYSCALL_DEFINE5(preadv,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h)1053 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1054 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1055 {
1056 	loff_t pos = pos_from_hilo(pos_h, pos_l);
1057 
1058 	return do_preadv(fd, vec, vlen, pos, 0);
1059 }
1060 
SYSCALL_DEFINE6(preadv2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h,rwf_t,flags)1061 SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1062 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1063 		rwf_t, flags)
1064 {
1065 	loff_t pos = pos_from_hilo(pos_h, pos_l);
1066 
1067 	if (pos == -1)
1068 		return do_readv(fd, vec, vlen, flags);
1069 
1070 	return do_preadv(fd, vec, vlen, pos, flags);
1071 }
1072 
SYSCALL_DEFINE5(pwritev,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h)1073 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1074 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1075 {
1076 	loff_t pos = pos_from_hilo(pos_h, pos_l);
1077 
1078 	return do_pwritev(fd, vec, vlen, pos, 0);
1079 }
1080 
SYSCALL_DEFINE6(pwritev2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h,rwf_t,flags)1081 SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1082 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1083 		rwf_t, flags)
1084 {
1085 	loff_t pos = pos_from_hilo(pos_h, pos_l);
1086 
1087 	if (pos == -1)
1088 		return do_writev(fd, vec, vlen, flags);
1089 
1090 	return do_pwritev(fd, vec, vlen, pos, flags);
1091 }
1092 
1093 /*
1094  * Various compat syscalls.  Note that they all pretend to take a native
1095  * iovec - import_iovec will properly treat those as compat_iovecs based on
1096  * in_compat_syscall().
1097  */
1098 #ifdef CONFIG_COMPAT
1099 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
COMPAT_SYSCALL_DEFINE4(preadv64,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos)1100 COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1101 		const struct iovec __user *, vec,
1102 		unsigned long, vlen, loff_t, pos)
1103 {
1104 	return do_preadv(fd, vec, vlen, pos, 0);
1105 }
1106 #endif
1107 
COMPAT_SYSCALL_DEFINE5(preadv,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high)1108 COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1109 		const struct iovec __user *, vec,
1110 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1111 {
1112 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1113 
1114 	return do_preadv(fd, vec, vlen, pos, 0);
1115 }
1116 
1117 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
COMPAT_SYSCALL_DEFINE5(preadv64v2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos,rwf_t,flags)1118 COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1119 		const struct iovec __user *, vec,
1120 		unsigned long, vlen, loff_t, pos, rwf_t, flags)
1121 {
1122 	if (pos == -1)
1123 		return do_readv(fd, vec, vlen, flags);
1124 	return do_preadv(fd, vec, vlen, pos, flags);
1125 }
1126 #endif
1127 
COMPAT_SYSCALL_DEFINE6(preadv2,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high,rwf_t,flags)1128 COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1129 		const struct iovec __user *, vec,
1130 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1131 		rwf_t, flags)
1132 {
1133 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1134 
1135 	if (pos == -1)
1136 		return do_readv(fd, vec, vlen, flags);
1137 	return do_preadv(fd, vec, vlen, pos, flags);
1138 }
1139 
1140 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
COMPAT_SYSCALL_DEFINE4(pwritev64,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos)1141 COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1142 		const struct iovec __user *, vec,
1143 		unsigned long, vlen, loff_t, pos)
1144 {
1145 	return do_pwritev(fd, vec, vlen, pos, 0);
1146 }
1147 #endif
1148 
COMPAT_SYSCALL_DEFINE5(pwritev,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high)1149 COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1150 		const struct iovec __user *,vec,
1151 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1152 {
1153 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1154 
1155 	return do_pwritev(fd, vec, vlen, pos, 0);
1156 }
1157 
1158 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
COMPAT_SYSCALL_DEFINE5(pwritev64v2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos,rwf_t,flags)1159 COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1160 		const struct iovec __user *, vec,
1161 		unsigned long, vlen, loff_t, pos, rwf_t, flags)
1162 {
1163 	if (pos == -1)
1164 		return do_writev(fd, vec, vlen, flags);
1165 	return do_pwritev(fd, vec, vlen, pos, flags);
1166 }
1167 #endif
1168 
COMPAT_SYSCALL_DEFINE6(pwritev2,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high,rwf_t,flags)1169 COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1170 		const struct iovec __user *,vec,
1171 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1172 {
1173 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1174 
1175 	if (pos == -1)
1176 		return do_writev(fd, vec, vlen, flags);
1177 	return do_pwritev(fd, vec, vlen, pos, flags);
1178 }
1179 #endif /* CONFIG_COMPAT */
1180 
do_sendfile(int out_fd,int in_fd,loff_t * ppos,size_t count,loff_t max)1181 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1182 		  	   size_t count, loff_t max)
1183 {
1184 	struct fd in, out;
1185 	struct inode *in_inode, *out_inode;
1186 	loff_t pos;
1187 	loff_t out_pos;
1188 	ssize_t retval;
1189 	int fl;
1190 
1191 	/*
1192 	 * Get input file, and verify that it is ok..
1193 	 */
1194 	retval = -EBADF;
1195 	in = fdget(in_fd);
1196 	if (!in.file)
1197 		goto out;
1198 	if (!(in.file->f_mode & FMODE_READ))
1199 		goto fput_in;
1200 	retval = -ESPIPE;
1201 	if (!ppos) {
1202 		pos = in.file->f_pos;
1203 	} else {
1204 		pos = *ppos;
1205 		if (!(in.file->f_mode & FMODE_PREAD))
1206 			goto fput_in;
1207 	}
1208 	retval = rw_verify_area(READ, in.file, &pos, count);
1209 	if (retval < 0)
1210 		goto fput_in;
1211 	if (count > MAX_RW_COUNT)
1212 		count =  MAX_RW_COUNT;
1213 
1214 	/*
1215 	 * Get output file, and verify that it is ok..
1216 	 */
1217 	retval = -EBADF;
1218 	out = fdget(out_fd);
1219 	if (!out.file)
1220 		goto fput_in;
1221 	if (!(out.file->f_mode & FMODE_WRITE))
1222 		goto fput_out;
1223 	in_inode = file_inode(in.file);
1224 	out_inode = file_inode(out.file);
1225 	out_pos = out.file->f_pos;
1226 	retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1227 	if (retval < 0)
1228 		goto fput_out;
1229 
1230 	if (!max)
1231 		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1232 
1233 	if (unlikely(pos + count > max)) {
1234 		retval = -EOVERFLOW;
1235 		if (pos >= max)
1236 			goto fput_out;
1237 		count = max - pos;
1238 	}
1239 
1240 	fl = 0;
1241 #if 0
1242 	/*
1243 	 * We need to debate whether we can enable this or not. The
1244 	 * man page documents EAGAIN return for the output at least,
1245 	 * and the application is arguably buggy if it doesn't expect
1246 	 * EAGAIN on a non-blocking file descriptor.
1247 	 */
1248 	if (in.file->f_flags & O_NONBLOCK)
1249 		fl = SPLICE_F_NONBLOCK;
1250 #endif
1251 	file_start_write(out.file);
1252 	retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
1253 	file_end_write(out.file);
1254 
1255 	if (retval > 0) {
1256 		add_rchar(current, retval);
1257 		add_wchar(current, retval);
1258 		fsnotify_access(in.file);
1259 		fsnotify_modify(out.file);
1260 		out.file->f_pos = out_pos;
1261 		if (ppos)
1262 			*ppos = pos;
1263 		else
1264 			in.file->f_pos = pos;
1265 	}
1266 
1267 	inc_syscr(current);
1268 	inc_syscw(current);
1269 	if (pos > max)
1270 		retval = -EOVERFLOW;
1271 
1272 fput_out:
1273 	fdput(out);
1274 fput_in:
1275 	fdput(in);
1276 out:
1277 	return retval;
1278 }
1279 
SYSCALL_DEFINE4(sendfile,int,out_fd,int,in_fd,off_t __user *,offset,size_t,count)1280 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1281 {
1282 	loff_t pos;
1283 	off_t off;
1284 	ssize_t ret;
1285 
1286 	if (offset) {
1287 		if (unlikely(get_user(off, offset)))
1288 			return -EFAULT;
1289 		pos = off;
1290 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1291 		if (unlikely(put_user(pos, offset)))
1292 			return -EFAULT;
1293 		return ret;
1294 	}
1295 
1296 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1297 }
1298 
SYSCALL_DEFINE4(sendfile64,int,out_fd,int,in_fd,loff_t __user *,offset,size_t,count)1299 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1300 {
1301 	loff_t pos;
1302 	ssize_t ret;
1303 
1304 	if (offset) {
1305 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1306 			return -EFAULT;
1307 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1308 		if (unlikely(put_user(pos, offset)))
1309 			return -EFAULT;
1310 		return ret;
1311 	}
1312 
1313 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1314 }
1315 
1316 #ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(sendfile,int,out_fd,int,in_fd,compat_off_t __user *,offset,compat_size_t,count)1317 COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1318 		compat_off_t __user *, offset, compat_size_t, count)
1319 {
1320 	loff_t pos;
1321 	off_t off;
1322 	ssize_t ret;
1323 
1324 	if (offset) {
1325 		if (unlikely(get_user(off, offset)))
1326 			return -EFAULT;
1327 		pos = off;
1328 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1329 		if (unlikely(put_user(pos, offset)))
1330 			return -EFAULT;
1331 		return ret;
1332 	}
1333 
1334 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1335 }
1336 
COMPAT_SYSCALL_DEFINE4(sendfile64,int,out_fd,int,in_fd,compat_loff_t __user *,offset,compat_size_t,count)1337 COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1338 		compat_loff_t __user *, offset, compat_size_t, count)
1339 {
1340 	loff_t pos;
1341 	ssize_t ret;
1342 
1343 	if (offset) {
1344 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1345 			return -EFAULT;
1346 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1347 		if (unlikely(put_user(pos, offset)))
1348 			return -EFAULT;
1349 		return ret;
1350 	}
1351 
1352 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1353 }
1354 #endif
1355 
1356 /**
1357  * generic_copy_file_range - copy data between two files
1358  * @file_in:	file structure to read from
1359  * @pos_in:	file offset to read from
1360  * @file_out:	file structure to write data to
1361  * @pos_out:	file offset to write data to
1362  * @len:	amount of data to copy
1363  * @flags:	copy flags
1364  *
1365  * This is a generic filesystem helper to copy data from one file to another.
1366  * It has no constraints on the source or destination file owners - the files
1367  * can belong to different superblocks and different filesystem types. Short
1368  * copies are allowed.
1369  *
1370  * This should be called from the @file_out filesystem, as per the
1371  * ->copy_file_range() method.
1372  *
1373  * Returns the number of bytes copied or a negative error indicating the
1374  * failure.
1375  */
1376 
generic_copy_file_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t len,unsigned int flags)1377 ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
1378 				struct file *file_out, loff_t pos_out,
1379 				size_t len, unsigned int flags)
1380 {
1381 	return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1382 				len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1383 }
1384 EXPORT_SYMBOL(generic_copy_file_range);
1385 
do_copy_file_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t len,unsigned int flags)1386 static ssize_t do_copy_file_range(struct file *file_in, loff_t pos_in,
1387 				  struct file *file_out, loff_t pos_out,
1388 				  size_t len, unsigned int flags)
1389 {
1390 	/*
1391 	 * Although we now allow filesystems to handle cross sb copy, passing
1392 	 * a file of the wrong filesystem type to filesystem driver can result
1393 	 * in an attempt to dereference the wrong type of ->private_data, so
1394 	 * avoid doing that until we really have a good reason.  NFS defines
1395 	 * several different file_system_type structures, but they all end up
1396 	 * using the same ->copy_file_range() function pointer.
1397 	 */
1398 	if (file_out->f_op->copy_file_range &&
1399 	    file_out->f_op->copy_file_range == file_in->f_op->copy_file_range)
1400 		return file_out->f_op->copy_file_range(file_in, pos_in,
1401 						       file_out, pos_out,
1402 						       len, flags);
1403 
1404 	return generic_copy_file_range(file_in, pos_in, file_out, pos_out, len,
1405 				       flags);
1406 }
1407 
1408 /*
1409  * Performs necessary checks before doing a file copy
1410  *
1411  * Can adjust amount of bytes to copy via @req_count argument.
1412  * Returns appropriate error code that caller should return or
1413  * zero in case the copy should be allowed.
1414  */
generic_copy_file_checks(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t * req_count,unsigned int flags)1415 static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
1416 				    struct file *file_out, loff_t pos_out,
1417 				    size_t *req_count, unsigned int flags)
1418 {
1419 	struct inode *inode_in = file_inode(file_in);
1420 	struct inode *inode_out = file_inode(file_out);
1421 	uint64_t count = *req_count;
1422 	loff_t size_in;
1423 	int ret;
1424 
1425 	ret = generic_file_rw_checks(file_in, file_out);
1426 	if (ret)
1427 		return ret;
1428 
1429 	/* Don't touch certain kinds of inodes */
1430 	if (IS_IMMUTABLE(inode_out))
1431 		return -EPERM;
1432 
1433 	if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1434 		return -ETXTBSY;
1435 
1436 	/* Ensure offsets don't wrap. */
1437 	if (pos_in + count < pos_in || pos_out + count < pos_out)
1438 		return -EOVERFLOW;
1439 
1440 	/* Shorten the copy to EOF */
1441 	size_in = i_size_read(inode_in);
1442 	if (pos_in >= size_in)
1443 		count = 0;
1444 	else
1445 		count = min(count, size_in - (uint64_t)pos_in);
1446 
1447 	ret = generic_write_check_limits(file_out, pos_out, &count);
1448 	if (ret)
1449 		return ret;
1450 
1451 	/* Don't allow overlapped copying within the same file. */
1452 	if (inode_in == inode_out &&
1453 	    pos_out + count > pos_in &&
1454 	    pos_out < pos_in + count)
1455 		return -EINVAL;
1456 
1457 	*req_count = count;
1458 	return 0;
1459 }
1460 
1461 /*
1462  * copy_file_range() differs from regular file read and write in that it
1463  * specifically allows return partial success.  When it does so is up to
1464  * the copy_file_range method.
1465  */
vfs_copy_file_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t len,unsigned int flags)1466 ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1467 			    struct file *file_out, loff_t pos_out,
1468 			    size_t len, unsigned int flags)
1469 {
1470 	ssize_t ret;
1471 
1472 	if (flags != 0)
1473 		return -EINVAL;
1474 
1475 	ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
1476 				       flags);
1477 	if (unlikely(ret))
1478 		return ret;
1479 
1480 	ret = rw_verify_area(READ, file_in, &pos_in, len);
1481 	if (unlikely(ret))
1482 		return ret;
1483 
1484 	ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1485 	if (unlikely(ret))
1486 		return ret;
1487 
1488 	if (len == 0)
1489 		return 0;
1490 
1491 	file_start_write(file_out);
1492 
1493 	/*
1494 	 * Try cloning first, this is supported by more file systems, and
1495 	 * more efficient if both clone and copy are supported (e.g. NFS).
1496 	 */
1497 	if (file_in->f_op->remap_file_range &&
1498 	    file_inode(file_in)->i_sb == file_inode(file_out)->i_sb) {
1499 		loff_t cloned;
1500 
1501 		cloned = file_in->f_op->remap_file_range(file_in, pos_in,
1502 				file_out, pos_out,
1503 				min_t(loff_t, MAX_RW_COUNT, len),
1504 				REMAP_FILE_CAN_SHORTEN);
1505 		if (cloned > 0) {
1506 			ret = cloned;
1507 			goto done;
1508 		}
1509 	}
1510 
1511 	ret = do_copy_file_range(file_in, pos_in, file_out, pos_out, len,
1512 				flags);
1513 	WARN_ON_ONCE(ret == -EOPNOTSUPP);
1514 done:
1515 	if (ret > 0) {
1516 		fsnotify_access(file_in);
1517 		add_rchar(current, ret);
1518 		fsnotify_modify(file_out);
1519 		add_wchar(current, ret);
1520 	}
1521 
1522 	inc_syscr(current);
1523 	inc_syscw(current);
1524 
1525 	file_end_write(file_out);
1526 
1527 	return ret;
1528 }
1529 EXPORT_SYMBOL(vfs_copy_file_range);
1530 
SYSCALL_DEFINE6(copy_file_range,int,fd_in,loff_t __user *,off_in,int,fd_out,loff_t __user *,off_out,size_t,len,unsigned int,flags)1531 SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1532 		int, fd_out, loff_t __user *, off_out,
1533 		size_t, len, unsigned int, flags)
1534 {
1535 	loff_t pos_in;
1536 	loff_t pos_out;
1537 	struct fd f_in;
1538 	struct fd f_out;
1539 	ssize_t ret = -EBADF;
1540 
1541 	f_in = fdget(fd_in);
1542 	if (!f_in.file)
1543 		goto out2;
1544 
1545 	f_out = fdget(fd_out);
1546 	if (!f_out.file)
1547 		goto out1;
1548 
1549 	ret = -EFAULT;
1550 	if (off_in) {
1551 		if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1552 			goto out;
1553 	} else {
1554 		pos_in = f_in.file->f_pos;
1555 	}
1556 
1557 	if (off_out) {
1558 		if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1559 			goto out;
1560 	} else {
1561 		pos_out = f_out.file->f_pos;
1562 	}
1563 
1564 	ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
1565 				  flags);
1566 	if (ret > 0) {
1567 		pos_in += ret;
1568 		pos_out += ret;
1569 
1570 		if (off_in) {
1571 			if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1572 				ret = -EFAULT;
1573 		} else {
1574 			f_in.file->f_pos = pos_in;
1575 		}
1576 
1577 		if (off_out) {
1578 			if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1579 				ret = -EFAULT;
1580 		} else {
1581 			f_out.file->f_pos = pos_out;
1582 		}
1583 	}
1584 
1585 out:
1586 	fdput(f_out);
1587 out1:
1588 	fdput(f_in);
1589 out2:
1590 	return ret;
1591 }
1592 
1593 /*
1594  * Don't operate on ranges the page cache doesn't support, and don't exceed the
1595  * LFS limits.  If pos is under the limit it becomes a short access.  If it
1596  * exceeds the limit we return -EFBIG.
1597  */
generic_write_check_limits(struct file * file,loff_t pos,loff_t * count)1598 int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
1599 {
1600 	struct inode *inode = file->f_mapping->host;
1601 	loff_t max_size = inode->i_sb->s_maxbytes;
1602 	loff_t limit = rlimit(RLIMIT_FSIZE);
1603 
1604 	if (limit != RLIM_INFINITY) {
1605 		if (pos >= limit) {
1606 			send_sig(SIGXFSZ, current, 0);
1607 			return -EFBIG;
1608 		}
1609 		*count = min(*count, limit - pos);
1610 	}
1611 
1612 	if (!(file->f_flags & O_LARGEFILE))
1613 		max_size = MAX_NON_LFS;
1614 
1615 	if (unlikely(pos >= max_size))
1616 		return -EFBIG;
1617 
1618 	*count = min(*count, max_size - pos);
1619 
1620 	return 0;
1621 }
1622 
1623 /*
1624  * Performs necessary checks before doing a write
1625  *
1626  * Can adjust writing position or amount of bytes to write.
1627  * Returns appropriate error code that caller should return or
1628  * zero in case that write should be allowed.
1629  */
generic_write_checks(struct kiocb * iocb,struct iov_iter * from)1630 ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1631 {
1632 	struct file *file = iocb->ki_filp;
1633 	struct inode *inode = file->f_mapping->host;
1634 	loff_t count;
1635 	int ret;
1636 
1637 	if (IS_SWAPFILE(inode))
1638 		return -ETXTBSY;
1639 
1640 	if (!iov_iter_count(from))
1641 		return 0;
1642 
1643 	/* FIXME: this is for backwards compatibility with 2.4 */
1644 	if (iocb->ki_flags & IOCB_APPEND)
1645 		iocb->ki_pos = i_size_read(inode);
1646 
1647 	if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT))
1648 		return -EINVAL;
1649 
1650 	count = iov_iter_count(from);
1651 	ret = generic_write_check_limits(file, iocb->ki_pos, &count);
1652 	if (ret)
1653 		return ret;
1654 
1655 	iov_iter_truncate(from, count);
1656 	return iov_iter_count(from);
1657 }
1658 EXPORT_SYMBOL(generic_write_checks);
1659 
1660 /*
1661  * Performs common checks before doing a file copy/clone
1662  * from @file_in to @file_out.
1663  */
generic_file_rw_checks(struct file * file_in,struct file * file_out)1664 int generic_file_rw_checks(struct file *file_in, struct file *file_out)
1665 {
1666 	struct inode *inode_in = file_inode(file_in);
1667 	struct inode *inode_out = file_inode(file_out);
1668 
1669 	/* Don't copy dirs, pipes, sockets... */
1670 	if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1671 		return -EISDIR;
1672 	if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1673 		return -EINVAL;
1674 
1675 	if (!(file_in->f_mode & FMODE_READ) ||
1676 	    !(file_out->f_mode & FMODE_WRITE) ||
1677 	    (file_out->f_flags & O_APPEND))
1678 		return -EBADF;
1679 
1680 	return 0;
1681 }
1682