1 /*
2 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18 #include "xfs.h"
19 #include "xfs_fs.h"
20 #include "xfs_shared.h"
21 #include "xfs_format.h"
22 #include "xfs_log_format.h"
23 #include "xfs_trans_resv.h"
24 #include "xfs_mount.h"
25 #include "xfs_da_format.h"
26 #include "xfs_da_btree.h"
27 #include "xfs_inode.h"
28 #include "xfs_trans.h"
29 #include "xfs_inode_item.h"
30 #include "xfs_bmap.h"
31 #include "xfs_bmap_util.h"
32 #include "xfs_error.h"
33 #include "xfs_dir2.h"
34 #include "xfs_dir2_priv.h"
35 #include "xfs_ioctl.h"
36 #include "xfs_trace.h"
37 #include "xfs_log.h"
38 #include "xfs_icache.h"
39 #include "xfs_pnfs.h"
40 #include "xfs_iomap.h"
41 #include "xfs_reflink.h"
42
43 #include <linux/dcache.h>
44 #include <linux/falloc.h>
45 #include <linux/pagevec.h>
46 #include <linux/backing-dev.h>
47
48 static const struct vm_operations_struct xfs_file_vm_ops;
49
50 /*
51 * Locking primitives for read and write IO paths to ensure we consistently use
52 * and order the inode->i_mutex, ip->i_lock and ip->i_iolock.
53 */
54 static inline void
xfs_rw_ilock(struct xfs_inode * ip,int type)55 xfs_rw_ilock(
56 struct xfs_inode *ip,
57 int type)
58 {
59 if (type & XFS_IOLOCK_EXCL)
60 inode_lock(VFS_I(ip));
61 xfs_ilock(ip, type);
62 }
63
64 static inline void
xfs_rw_iunlock(struct xfs_inode * ip,int type)65 xfs_rw_iunlock(
66 struct xfs_inode *ip,
67 int type)
68 {
69 xfs_iunlock(ip, type);
70 if (type & XFS_IOLOCK_EXCL)
71 inode_unlock(VFS_I(ip));
72 }
73
74 static inline void
xfs_rw_ilock_demote(struct xfs_inode * ip,int type)75 xfs_rw_ilock_demote(
76 struct xfs_inode *ip,
77 int type)
78 {
79 xfs_ilock_demote(ip, type);
80 if (type & XFS_IOLOCK_EXCL)
81 inode_unlock(VFS_I(ip));
82 }
83
84 /*
85 * Clear the specified ranges to zero through either the pagecache or DAX.
86 * Holes and unwritten extents will be left as-is as they already are zeroed.
87 */
88 int
xfs_zero_range(struct xfs_inode * ip,xfs_off_t pos,xfs_off_t count,bool * did_zero)89 xfs_zero_range(
90 struct xfs_inode *ip,
91 xfs_off_t pos,
92 xfs_off_t count,
93 bool *did_zero)
94 {
95 return iomap_zero_range(VFS_I(ip), pos, count, did_zero, &xfs_iomap_ops);
96 }
97
98 int
xfs_update_prealloc_flags(struct xfs_inode * ip,enum xfs_prealloc_flags flags)99 xfs_update_prealloc_flags(
100 struct xfs_inode *ip,
101 enum xfs_prealloc_flags flags)
102 {
103 struct xfs_trans *tp;
104 int error;
105
106 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_writeid,
107 0, 0, 0, &tp);
108 if (error)
109 return error;
110
111 xfs_ilock(ip, XFS_ILOCK_EXCL);
112 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
113
114 if (!(flags & XFS_PREALLOC_INVISIBLE)) {
115 VFS_I(ip)->i_mode &= ~S_ISUID;
116 if (VFS_I(ip)->i_mode & S_IXGRP)
117 VFS_I(ip)->i_mode &= ~S_ISGID;
118 xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
119 }
120
121 if (flags & XFS_PREALLOC_SET)
122 ip->i_d.di_flags |= XFS_DIFLAG_PREALLOC;
123 if (flags & XFS_PREALLOC_CLEAR)
124 ip->i_d.di_flags &= ~XFS_DIFLAG_PREALLOC;
125
126 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
127 if (flags & XFS_PREALLOC_SYNC)
128 xfs_trans_set_sync(tp);
129 return xfs_trans_commit(tp);
130 }
131
132 /*
133 * Fsync operations on directories are much simpler than on regular files,
134 * as there is no file data to flush, and thus also no need for explicit
135 * cache flush operations, and there are no non-transaction metadata updates
136 * on directories either.
137 */
138 STATIC int
xfs_dir_fsync(struct file * file,loff_t start,loff_t end,int datasync)139 xfs_dir_fsync(
140 struct file *file,
141 loff_t start,
142 loff_t end,
143 int datasync)
144 {
145 struct xfs_inode *ip = XFS_I(file->f_mapping->host);
146 struct xfs_mount *mp = ip->i_mount;
147 xfs_lsn_t lsn = 0;
148
149 trace_xfs_dir_fsync(ip);
150
151 xfs_ilock(ip, XFS_ILOCK_SHARED);
152 if (xfs_ipincount(ip))
153 lsn = ip->i_itemp->ili_last_lsn;
154 xfs_iunlock(ip, XFS_ILOCK_SHARED);
155
156 if (!lsn)
157 return 0;
158 return _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, NULL);
159 }
160
161 STATIC int
xfs_file_fsync(struct file * file,loff_t start,loff_t end,int datasync)162 xfs_file_fsync(
163 struct file *file,
164 loff_t start,
165 loff_t end,
166 int datasync)
167 {
168 struct inode *inode = file->f_mapping->host;
169 struct xfs_inode *ip = XFS_I(inode);
170 struct xfs_mount *mp = ip->i_mount;
171 int error = 0;
172 int log_flushed = 0;
173 xfs_lsn_t lsn = 0;
174
175 trace_xfs_file_fsync(ip);
176
177 error = filemap_write_and_wait_range(inode->i_mapping, start, end);
178 if (error)
179 return error;
180
181 if (XFS_FORCED_SHUTDOWN(mp))
182 return -EIO;
183
184 xfs_iflags_clear(ip, XFS_ITRUNCATED);
185
186 if (mp->m_flags & XFS_MOUNT_BARRIER) {
187 /*
188 * If we have an RT and/or log subvolume we need to make sure
189 * to flush the write cache the device used for file data
190 * first. This is to ensure newly written file data make
191 * it to disk before logging the new inode size in case of
192 * an extending write.
193 */
194 if (XFS_IS_REALTIME_INODE(ip))
195 xfs_blkdev_issue_flush(mp->m_rtdev_targp);
196 else if (mp->m_logdev_targp != mp->m_ddev_targp)
197 xfs_blkdev_issue_flush(mp->m_ddev_targp);
198 }
199
200 /*
201 * All metadata updates are logged, which means that we just have to
202 * flush the log up to the latest LSN that touched the inode. If we have
203 * concurrent fsync/fdatasync() calls, we need them to all block on the
204 * log force before we clear the ili_fsync_fields field. This ensures
205 * that we don't get a racing sync operation that does not wait for the
206 * metadata to hit the journal before returning. If we race with
207 * clearing the ili_fsync_fields, then all that will happen is the log
208 * force will do nothing as the lsn will already be on disk. We can't
209 * race with setting ili_fsync_fields because that is done under
210 * XFS_ILOCK_EXCL, and that can't happen because we hold the lock shared
211 * until after the ili_fsync_fields is cleared.
212 */
213 xfs_ilock(ip, XFS_ILOCK_SHARED);
214 if (xfs_ipincount(ip)) {
215 if (!datasync ||
216 (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
217 lsn = ip->i_itemp->ili_last_lsn;
218 }
219
220 if (lsn) {
221 error = _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed);
222 ip->i_itemp->ili_fsync_fields = 0;
223 }
224 xfs_iunlock(ip, XFS_ILOCK_SHARED);
225
226 /*
227 * If we only have a single device, and the log force about was
228 * a no-op we might have to flush the data device cache here.
229 * This can only happen for fdatasync/O_DSYNC if we were overwriting
230 * an already allocated file and thus do not have any metadata to
231 * commit.
232 */
233 if ((mp->m_flags & XFS_MOUNT_BARRIER) &&
234 mp->m_logdev_targp == mp->m_ddev_targp &&
235 !XFS_IS_REALTIME_INODE(ip) &&
236 !log_flushed)
237 xfs_blkdev_issue_flush(mp->m_ddev_targp);
238
239 return error;
240 }
241
242 STATIC ssize_t
xfs_file_dio_aio_read(struct kiocb * iocb,struct iov_iter * to)243 xfs_file_dio_aio_read(
244 struct kiocb *iocb,
245 struct iov_iter *to)
246 {
247 struct address_space *mapping = iocb->ki_filp->f_mapping;
248 struct inode *inode = mapping->host;
249 struct xfs_inode *ip = XFS_I(inode);
250 loff_t isize = i_size_read(inode);
251 size_t count = iov_iter_count(to);
252 loff_t end = iocb->ki_pos + count - 1;
253 struct iov_iter data;
254 struct xfs_buftarg *target;
255 ssize_t ret = 0;
256
257 trace_xfs_file_direct_read(ip, count, iocb->ki_pos);
258
259 if (!count)
260 return 0; /* skip atime */
261
262 if (XFS_IS_REALTIME_INODE(ip))
263 target = ip->i_mount->m_rtdev_targp;
264 else
265 target = ip->i_mount->m_ddev_targp;
266
267 /* DIO must be aligned to device logical sector size */
268 if ((iocb->ki_pos | count) & target->bt_logical_sectormask) {
269 if (iocb->ki_pos == isize)
270 return 0;
271 return -EINVAL;
272 }
273
274 file_accessed(iocb->ki_filp);
275
276 xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
277 if (mapping->nrpages) {
278 ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end);
279 if (ret)
280 goto out_unlock;
281
282 /*
283 * Invalidate whole pages. This can return an error if we fail
284 * to invalidate a page, but this should never happen on XFS.
285 * Warn if it does fail.
286 */
287 ret = invalidate_inode_pages2_range(mapping,
288 iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
289 WARN_ON_ONCE(ret);
290 ret = 0;
291 }
292
293 data = *to;
294 ret = __blockdev_direct_IO(iocb, inode, target->bt_bdev, &data,
295 xfs_get_blocks_direct, NULL, NULL, 0);
296 if (ret >= 0) {
297 iocb->ki_pos += ret;
298 iov_iter_advance(to, ret);
299 }
300
301 out_unlock:
302 xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
303 return ret;
304 }
305
306 static noinline ssize_t
xfs_file_dax_read(struct kiocb * iocb,struct iov_iter * to)307 xfs_file_dax_read(
308 struct kiocb *iocb,
309 struct iov_iter *to)
310 {
311 struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host);
312 size_t count = iov_iter_count(to);
313 ssize_t ret = 0;
314
315 trace_xfs_file_dax_read(ip, count, iocb->ki_pos);
316
317 if (!count)
318 return 0; /* skip atime */
319
320 xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
321 ret = iomap_dax_rw(iocb, to, &xfs_iomap_ops);
322 xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
323
324 file_accessed(iocb->ki_filp);
325 return ret;
326 }
327
328 STATIC ssize_t
xfs_file_buffered_aio_read(struct kiocb * iocb,struct iov_iter * to)329 xfs_file_buffered_aio_read(
330 struct kiocb *iocb,
331 struct iov_iter *to)
332 {
333 struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
334 ssize_t ret;
335
336 trace_xfs_file_buffered_read(ip, iov_iter_count(to), iocb->ki_pos);
337
338 xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
339 ret = generic_file_read_iter(iocb, to);
340 xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
341
342 return ret;
343 }
344
345 STATIC ssize_t
xfs_file_read_iter(struct kiocb * iocb,struct iov_iter * to)346 xfs_file_read_iter(
347 struct kiocb *iocb,
348 struct iov_iter *to)
349 {
350 struct inode *inode = file_inode(iocb->ki_filp);
351 struct xfs_mount *mp = XFS_I(inode)->i_mount;
352 ssize_t ret = 0;
353
354 XFS_STATS_INC(mp, xs_read_calls);
355
356 if (XFS_FORCED_SHUTDOWN(mp))
357 return -EIO;
358
359 if (IS_DAX(inode))
360 ret = xfs_file_dax_read(iocb, to);
361 else if (iocb->ki_flags & IOCB_DIRECT)
362 ret = xfs_file_dio_aio_read(iocb, to);
363 else
364 ret = xfs_file_buffered_aio_read(iocb, to);
365
366 if (ret > 0)
367 XFS_STATS_ADD(mp, xs_read_bytes, ret);
368 return ret;
369 }
370
371 /*
372 * Zero any on disk space between the current EOF and the new, larger EOF.
373 *
374 * This handles the normal case of zeroing the remainder of the last block in
375 * the file and the unusual case of zeroing blocks out beyond the size of the
376 * file. This second case only happens with fixed size extents and when the
377 * system crashes before the inode size was updated but after blocks were
378 * allocated.
379 *
380 * Expects the iolock to be held exclusive, and will take the ilock internally.
381 */
382 int /* error (positive) */
xfs_zero_eof(struct xfs_inode * ip,xfs_off_t offset,xfs_fsize_t isize,bool * did_zeroing)383 xfs_zero_eof(
384 struct xfs_inode *ip,
385 xfs_off_t offset, /* starting I/O offset */
386 xfs_fsize_t isize, /* current inode size */
387 bool *did_zeroing)
388 {
389 ASSERT(xfs_isilocked(ip, XFS_IOLOCK_EXCL));
390 ASSERT(offset > isize);
391
392 trace_xfs_zero_eof(ip, isize, offset - isize);
393 return xfs_zero_range(ip, isize, offset - isize, did_zeroing);
394 }
395
396 /*
397 * Common pre-write limit and setup checks.
398 *
399 * Called with the iolocked held either shared and exclusive according to
400 * @iolock, and returns with it held. Might upgrade the iolock to exclusive
401 * if called for a direct write beyond i_size.
402 */
403 STATIC ssize_t
xfs_file_aio_write_checks(struct kiocb * iocb,struct iov_iter * from,int * iolock)404 xfs_file_aio_write_checks(
405 struct kiocb *iocb,
406 struct iov_iter *from,
407 int *iolock)
408 {
409 struct file *file = iocb->ki_filp;
410 struct inode *inode = file->f_mapping->host;
411 struct xfs_inode *ip = XFS_I(inode);
412 ssize_t error = 0;
413 size_t count = iov_iter_count(from);
414 bool drained_dio = false;
415
416 restart:
417 error = generic_write_checks(iocb, from);
418 if (error <= 0)
419 return error;
420
421 error = xfs_break_layouts(inode, iolock, true);
422 if (error)
423 return error;
424
425 /* For changing security info in file_remove_privs() we need i_mutex */
426 if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
427 xfs_rw_iunlock(ip, *iolock);
428 *iolock = XFS_IOLOCK_EXCL;
429 xfs_rw_ilock(ip, *iolock);
430 goto restart;
431 }
432 /*
433 * If the offset is beyond the size of the file, we need to zero any
434 * blocks that fall between the existing EOF and the start of this
435 * write. If zeroing is needed and we are currently holding the
436 * iolock shared, we need to update it to exclusive which implies
437 * having to redo all checks before.
438 *
439 * We need to serialise against EOF updates that occur in IO
440 * completions here. We want to make sure that nobody is changing the
441 * size while we do this check until we have placed an IO barrier (i.e.
442 * hold the XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.
443 * The spinlock effectively forms a memory barrier once we have the
444 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value
445 * and hence be able to correctly determine if we need to run zeroing.
446 */
447 spin_lock(&ip->i_flags_lock);
448 if (iocb->ki_pos > i_size_read(inode)) {
449 bool zero = false;
450
451 spin_unlock(&ip->i_flags_lock);
452 if (!drained_dio) {
453 if (*iolock == XFS_IOLOCK_SHARED) {
454 xfs_rw_iunlock(ip, *iolock);
455 *iolock = XFS_IOLOCK_EXCL;
456 xfs_rw_ilock(ip, *iolock);
457 iov_iter_reexpand(from, count);
458 }
459 /*
460 * We now have an IO submission barrier in place, but
461 * AIO can do EOF updates during IO completion and hence
462 * we now need to wait for all of them to drain. Non-AIO
463 * DIO will have drained before we are given the
464 * XFS_IOLOCK_EXCL, and so for most cases this wait is a
465 * no-op.
466 */
467 inode_dio_wait(inode);
468 drained_dio = true;
469 goto restart;
470 }
471 error = xfs_zero_eof(ip, iocb->ki_pos, i_size_read(inode), &zero);
472 if (error)
473 return error;
474 } else
475 spin_unlock(&ip->i_flags_lock);
476
477 /*
478 * Updating the timestamps will grab the ilock again from
479 * xfs_fs_dirty_inode, so we have to call it after dropping the
480 * lock above. Eventually we should look into a way to avoid
481 * the pointless lock roundtrip.
482 */
483 if (likely(!(file->f_mode & FMODE_NOCMTIME))) {
484 error = file_update_time(file);
485 if (error)
486 return error;
487 }
488
489 /*
490 * If we're writing the file then make sure to clear the setuid and
491 * setgid bits if the process is not being run by root. This keeps
492 * people from modifying setuid and setgid binaries.
493 */
494 if (!IS_NOSEC(inode))
495 return file_remove_privs(file);
496 return 0;
497 }
498
499 /*
500 * xfs_file_dio_aio_write - handle direct IO writes
501 *
502 * Lock the inode appropriately to prepare for and issue a direct IO write.
503 * By separating it from the buffered write path we remove all the tricky to
504 * follow locking changes and looping.
505 *
506 * If there are cached pages or we're extending the file, we need IOLOCK_EXCL
507 * until we're sure the bytes at the new EOF have been zeroed and/or the cached
508 * pages are flushed out.
509 *
510 * In most cases the direct IO writes will be done holding IOLOCK_SHARED
511 * allowing them to be done in parallel with reads and other direct IO writes.
512 * However, if the IO is not aligned to filesystem blocks, the direct IO layer
513 * needs to do sub-block zeroing and that requires serialisation against other
514 * direct IOs to the same block. In this case we need to serialise the
515 * submission of the unaligned IOs so that we don't get racing block zeroing in
516 * the dio layer. To avoid the problem with aio, we also need to wait for
517 * outstanding IOs to complete so that unwritten extent conversion is completed
518 * before we try to map the overlapping block. This is currently implemented by
519 * hitting it with a big hammer (i.e. inode_dio_wait()).
520 *
521 * Returns with locks held indicated by @iolock and errors indicated by
522 * negative return values.
523 */
524 STATIC ssize_t
xfs_file_dio_aio_write(struct kiocb * iocb,struct iov_iter * from)525 xfs_file_dio_aio_write(
526 struct kiocb *iocb,
527 struct iov_iter *from)
528 {
529 struct file *file = iocb->ki_filp;
530 struct address_space *mapping = file->f_mapping;
531 struct inode *inode = mapping->host;
532 struct xfs_inode *ip = XFS_I(inode);
533 struct xfs_mount *mp = ip->i_mount;
534 ssize_t ret = 0;
535 int unaligned_io = 0;
536 int iolock;
537 size_t count = iov_iter_count(from);
538 loff_t end;
539 struct iov_iter data;
540 struct xfs_buftarg *target = XFS_IS_REALTIME_INODE(ip) ?
541 mp->m_rtdev_targp : mp->m_ddev_targp;
542
543 /* DIO must be aligned to device logical sector size */
544 if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
545 return -EINVAL;
546
547 /*
548 * Don't take the exclusive iolock here unless the I/O is unaligned to
549 * the file system block size. We don't need to consider the EOF
550 * extension case here because xfs_file_aio_write_checks() will relock
551 * the inode as necessary for EOF zeroing cases and fill out the new
552 * inode size as appropriate.
553 */
554 if ((iocb->ki_pos & mp->m_blockmask) ||
555 ((iocb->ki_pos + count) & mp->m_blockmask)) {
556 unaligned_io = 1;
557
558 /*
559 * We can't properly handle unaligned direct I/O to reflink
560 * files yet, as we can't unshare a partial block.
561 */
562 if (xfs_is_reflink_inode(ip)) {
563 trace_xfs_reflink_bounce_dio_write(ip, iocb->ki_pos, count);
564 return -EREMCHG;
565 }
566 iolock = XFS_IOLOCK_EXCL;
567 } else {
568 iolock = XFS_IOLOCK_SHARED;
569 }
570
571 xfs_rw_ilock(ip, iolock);
572
573 ret = xfs_file_aio_write_checks(iocb, from, &iolock);
574 if (ret)
575 goto out;
576 count = iov_iter_count(from);
577 end = iocb->ki_pos + count - 1;
578
579 if (mapping->nrpages) {
580 ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end);
581 if (ret)
582 goto out;
583
584 /*
585 * Invalidate whole pages. This can return an error if we fail
586 * to invalidate a page, but this should never happen on XFS.
587 * Warn if it does fail.
588 */
589 ret = invalidate_inode_pages2_range(mapping,
590 iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
591 WARN_ON_ONCE(ret);
592 ret = 0;
593 }
594
595 /*
596 * If we are doing unaligned IO, wait for all other IO to drain,
597 * otherwise demote the lock if we had to take the exclusive lock
598 * for other reasons in xfs_file_aio_write_checks.
599 */
600 if (unaligned_io)
601 inode_dio_wait(inode);
602 else if (iolock == XFS_IOLOCK_EXCL) {
603 xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
604 iolock = XFS_IOLOCK_SHARED;
605 }
606
607 trace_xfs_file_direct_write(ip, count, iocb->ki_pos);
608
609 /* If this is a block-aligned directio CoW, remap immediately. */
610 if (xfs_is_reflink_inode(ip) && !unaligned_io) {
611 ret = xfs_reflink_allocate_cow_range(ip, iocb->ki_pos, count);
612 if (ret)
613 goto out;
614 }
615
616 data = *from;
617 ret = __blockdev_direct_IO(iocb, inode, target->bt_bdev, &data,
618 xfs_get_blocks_direct, xfs_end_io_direct_write,
619 NULL, DIO_ASYNC_EXTEND);
620
621 /* see generic_file_direct_write() for why this is necessary */
622 if (mapping->nrpages) {
623 invalidate_inode_pages2_range(mapping,
624 iocb->ki_pos >> PAGE_SHIFT,
625 end >> PAGE_SHIFT);
626 }
627
628 if (ret > 0) {
629 iocb->ki_pos += ret;
630 iov_iter_advance(from, ret);
631 }
632 out:
633 xfs_rw_iunlock(ip, iolock);
634
635 /*
636 * No fallback to buffered IO on errors for XFS, direct IO will either
637 * complete fully or fail.
638 */
639 ASSERT(ret < 0 || ret == count);
640 return ret;
641 }
642
643 static noinline ssize_t
xfs_file_dax_write(struct kiocb * iocb,struct iov_iter * from)644 xfs_file_dax_write(
645 struct kiocb *iocb,
646 struct iov_iter *from)
647 {
648 struct inode *inode = iocb->ki_filp->f_mapping->host;
649 struct xfs_inode *ip = XFS_I(inode);
650 int iolock = XFS_IOLOCK_EXCL;
651 ssize_t ret, error = 0;
652 size_t count;
653 loff_t pos;
654
655 xfs_rw_ilock(ip, iolock);
656 ret = xfs_file_aio_write_checks(iocb, from, &iolock);
657 if (ret)
658 goto out;
659
660 pos = iocb->ki_pos;
661 count = iov_iter_count(from);
662
663 trace_xfs_file_dax_write(ip, count, pos);
664
665 ret = iomap_dax_rw(iocb, from, &xfs_iomap_ops);
666 if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
667 i_size_write(inode, iocb->ki_pos);
668 error = xfs_setfilesize(ip, pos, ret);
669 }
670
671 out:
672 xfs_rw_iunlock(ip, iolock);
673 return error ? error : ret;
674 }
675
676 STATIC ssize_t
xfs_file_buffered_aio_write(struct kiocb * iocb,struct iov_iter * from)677 xfs_file_buffered_aio_write(
678 struct kiocb *iocb,
679 struct iov_iter *from)
680 {
681 struct file *file = iocb->ki_filp;
682 struct address_space *mapping = file->f_mapping;
683 struct inode *inode = mapping->host;
684 struct xfs_inode *ip = XFS_I(inode);
685 ssize_t ret;
686 int enospc = 0;
687 int iolock;
688
689 write_retry:
690 iolock = XFS_IOLOCK_EXCL;
691 xfs_rw_ilock(ip, iolock);
692
693 ret = xfs_file_aio_write_checks(iocb, from, &iolock);
694 if (ret)
695 goto out;
696
697 /* We can write back this queue in page reclaim */
698 current->backing_dev_info = inode_to_bdi(inode);
699
700 trace_xfs_file_buffered_write(ip, iov_iter_count(from), iocb->ki_pos);
701 ret = iomap_file_buffered_write(iocb, from, &xfs_iomap_ops);
702 if (likely(ret >= 0))
703 iocb->ki_pos += ret;
704
705 /*
706 * If we hit a space limit, try to free up some lingering preallocated
707 * space before returning an error. In the case of ENOSPC, first try to
708 * write back all dirty inodes to free up some of the excess reserved
709 * metadata space. This reduces the chances that the eofblocks scan
710 * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
711 * also behaves as a filter to prevent too many eofblocks scans from
712 * running at the same time.
713 */
714 if (ret == -EDQUOT && !enospc) {
715 xfs_rw_iunlock(ip, iolock);
716 enospc = xfs_inode_free_quota_eofblocks(ip);
717 if (enospc)
718 goto write_retry;
719 enospc = xfs_inode_free_quota_cowblocks(ip);
720 if (enospc)
721 goto write_retry;
722 iolock = 0;
723 } else if (ret == -ENOSPC && !enospc) {
724 struct xfs_eofblocks eofb = {0};
725
726 enospc = 1;
727 xfs_flush_inodes(ip->i_mount);
728
729 xfs_rw_iunlock(ip, iolock);
730 eofb.eof_flags = XFS_EOF_FLAGS_SYNC;
731 xfs_icache_free_eofblocks(ip->i_mount, &eofb);
732 xfs_icache_free_cowblocks(ip->i_mount, &eofb);
733 goto write_retry;
734 }
735
736 current->backing_dev_info = NULL;
737 out:
738 if (iolock)
739 xfs_rw_iunlock(ip, iolock);
740 return ret;
741 }
742
743 STATIC ssize_t
xfs_file_write_iter(struct kiocb * iocb,struct iov_iter * from)744 xfs_file_write_iter(
745 struct kiocb *iocb,
746 struct iov_iter *from)
747 {
748 struct file *file = iocb->ki_filp;
749 struct address_space *mapping = file->f_mapping;
750 struct inode *inode = mapping->host;
751 struct xfs_inode *ip = XFS_I(inode);
752 ssize_t ret;
753 size_t ocount = iov_iter_count(from);
754
755 XFS_STATS_INC(ip->i_mount, xs_write_calls);
756
757 if (ocount == 0)
758 return 0;
759
760 if (XFS_FORCED_SHUTDOWN(ip->i_mount))
761 return -EIO;
762
763 if (IS_DAX(inode))
764 ret = xfs_file_dax_write(iocb, from);
765 else if (iocb->ki_flags & IOCB_DIRECT) {
766 /*
767 * Allow a directio write to fall back to a buffered
768 * write *only* in the case that we're doing a reflink
769 * CoW. In all other directio scenarios we do not
770 * allow an operation to fall back to buffered mode.
771 */
772 ret = xfs_file_dio_aio_write(iocb, from);
773 if (ret == -EREMCHG)
774 goto buffered;
775 } else {
776 buffered:
777 ret = xfs_file_buffered_aio_write(iocb, from);
778 }
779
780 if (ret > 0) {
781 XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
782
783 /* Handle various SYNC-type writes */
784 ret = generic_write_sync(iocb, ret);
785 }
786 return ret;
787 }
788
789 #define XFS_FALLOC_FL_SUPPORTED \
790 (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
791 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | \
792 FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
793
794 STATIC long
xfs_file_fallocate(struct file * file,int mode,loff_t offset,loff_t len)795 xfs_file_fallocate(
796 struct file *file,
797 int mode,
798 loff_t offset,
799 loff_t len)
800 {
801 struct inode *inode = file_inode(file);
802 struct xfs_inode *ip = XFS_I(inode);
803 long error;
804 enum xfs_prealloc_flags flags = 0;
805 uint iolock = XFS_IOLOCK_EXCL;
806 loff_t new_size = 0;
807 bool do_file_insert = 0;
808
809 if (!S_ISREG(inode->i_mode))
810 return -EINVAL;
811 if (mode & ~XFS_FALLOC_FL_SUPPORTED)
812 return -EOPNOTSUPP;
813
814 xfs_ilock(ip, iolock);
815 error = xfs_break_layouts(inode, &iolock, false);
816 if (error)
817 goto out_unlock;
818
819 xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
820 iolock |= XFS_MMAPLOCK_EXCL;
821
822 if (mode & FALLOC_FL_PUNCH_HOLE) {
823 error = xfs_free_file_space(ip, offset, len);
824 if (error)
825 goto out_unlock;
826 } else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
827 unsigned int blksize_mask = i_blocksize(inode) - 1;
828
829 if (offset & blksize_mask || len & blksize_mask) {
830 error = -EINVAL;
831 goto out_unlock;
832 }
833
834 /*
835 * There is no need to overlap collapse range with EOF,
836 * in which case it is effectively a truncate operation
837 */
838 if (offset + len >= i_size_read(inode)) {
839 error = -EINVAL;
840 goto out_unlock;
841 }
842
843 new_size = i_size_read(inode) - len;
844
845 error = xfs_collapse_file_space(ip, offset, len);
846 if (error)
847 goto out_unlock;
848 } else if (mode & FALLOC_FL_INSERT_RANGE) {
849 unsigned int blksize_mask = i_blocksize(inode) - 1;
850
851 new_size = i_size_read(inode) + len;
852 if (offset & blksize_mask || len & blksize_mask) {
853 error = -EINVAL;
854 goto out_unlock;
855 }
856
857 /* check the new inode size does not wrap through zero */
858 if (new_size > inode->i_sb->s_maxbytes) {
859 error = -EFBIG;
860 goto out_unlock;
861 }
862
863 /* Offset should be less than i_size */
864 if (offset >= i_size_read(inode)) {
865 error = -EINVAL;
866 goto out_unlock;
867 }
868 do_file_insert = 1;
869 } else {
870 flags |= XFS_PREALLOC_SET;
871
872 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
873 offset + len > i_size_read(inode)) {
874 new_size = offset + len;
875 error = inode_newsize_ok(inode, new_size);
876 if (error)
877 goto out_unlock;
878 }
879
880 if (mode & FALLOC_FL_ZERO_RANGE)
881 error = xfs_zero_file_space(ip, offset, len);
882 else {
883 if (mode & FALLOC_FL_UNSHARE_RANGE) {
884 error = xfs_reflink_unshare(ip, offset, len);
885 if (error)
886 goto out_unlock;
887 }
888 error = xfs_alloc_file_space(ip, offset, len,
889 XFS_BMAPI_PREALLOC);
890 }
891 if (error)
892 goto out_unlock;
893 }
894
895 if (file->f_flags & O_DSYNC)
896 flags |= XFS_PREALLOC_SYNC;
897
898 error = xfs_update_prealloc_flags(ip, flags);
899 if (error)
900 goto out_unlock;
901
902 /* Change file size if needed */
903 if (new_size) {
904 struct iattr iattr;
905
906 iattr.ia_valid = ATTR_SIZE;
907 iattr.ia_size = new_size;
908 error = xfs_vn_setattr_size(file_dentry(file), &iattr);
909 if (error)
910 goto out_unlock;
911 }
912
913 /*
914 * Perform hole insertion now that the file size has been
915 * updated so that if we crash during the operation we don't
916 * leave shifted extents past EOF and hence losing access to
917 * the data that is contained within them.
918 */
919 if (do_file_insert)
920 error = xfs_insert_file_space(ip, offset, len);
921
922 out_unlock:
923 xfs_iunlock(ip, iolock);
924 return error;
925 }
926
927 STATIC ssize_t
xfs_file_copy_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t len,unsigned int flags)928 xfs_file_copy_range(
929 struct file *file_in,
930 loff_t pos_in,
931 struct file *file_out,
932 loff_t pos_out,
933 size_t len,
934 unsigned int flags)
935 {
936 int error;
937
938 error = xfs_reflink_remap_range(file_in, pos_in, file_out, pos_out,
939 len, false);
940 if (error)
941 return error;
942 return len;
943 }
944
945 STATIC int
xfs_file_clone_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,u64 len)946 xfs_file_clone_range(
947 struct file *file_in,
948 loff_t pos_in,
949 struct file *file_out,
950 loff_t pos_out,
951 u64 len)
952 {
953 return xfs_reflink_remap_range(file_in, pos_in, file_out, pos_out,
954 len, false);
955 }
956
957 STATIC ssize_t
xfs_file_dedupe_range(struct file * src_file,u64 loff,u64 len,struct file * dst_file,u64 dst_loff)958 xfs_file_dedupe_range(
959 struct file *src_file,
960 u64 loff,
961 u64 len,
962 struct file *dst_file,
963 u64 dst_loff)
964 {
965 int error;
966
967 error = xfs_reflink_remap_range(src_file, loff, dst_file, dst_loff,
968 len, true);
969 if (error)
970 return error;
971 return len;
972 }
973
974 STATIC int
xfs_file_open(struct inode * inode,struct file * file)975 xfs_file_open(
976 struct inode *inode,
977 struct file *file)
978 {
979 if (!(file->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS)
980 return -EFBIG;
981 if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
982 return -EIO;
983 return 0;
984 }
985
986 STATIC int
xfs_dir_open(struct inode * inode,struct file * file)987 xfs_dir_open(
988 struct inode *inode,
989 struct file *file)
990 {
991 struct xfs_inode *ip = XFS_I(inode);
992 int mode;
993 int error;
994
995 error = xfs_file_open(inode, file);
996 if (error)
997 return error;
998
999 /*
1000 * If there are any blocks, read-ahead block 0 as we're almost
1001 * certain to have the next operation be a read there.
1002 */
1003 mode = xfs_ilock_data_map_shared(ip);
1004 if (ip->i_d.di_nextents > 0)
1005 error = xfs_dir3_data_readahead(ip, 0, -1);
1006 xfs_iunlock(ip, mode);
1007 return error;
1008 }
1009
1010 STATIC int
xfs_file_release(struct inode * inode,struct file * filp)1011 xfs_file_release(
1012 struct inode *inode,
1013 struct file *filp)
1014 {
1015 return xfs_release(XFS_I(inode));
1016 }
1017
1018 STATIC int
xfs_file_readdir(struct file * file,struct dir_context * ctx)1019 xfs_file_readdir(
1020 struct file *file,
1021 struct dir_context *ctx)
1022 {
1023 struct inode *inode = file_inode(file);
1024 xfs_inode_t *ip = XFS_I(inode);
1025 size_t bufsize;
1026
1027 /*
1028 * The Linux API doesn't pass down the total size of the buffer
1029 * we read into down to the filesystem. With the filldir concept
1030 * it's not needed for correct information, but the XFS dir2 leaf
1031 * code wants an estimate of the buffer size to calculate it's
1032 * readahead window and size the buffers used for mapping to
1033 * physical blocks.
1034 *
1035 * Try to give it an estimate that's good enough, maybe at some
1036 * point we can change the ->readdir prototype to include the
1037 * buffer size. For now we use the current glibc buffer size.
1038 */
1039 bufsize = (size_t)min_t(loff_t, 32768, ip->i_d.di_size);
1040
1041 return xfs_readdir(ip, ctx, bufsize);
1042 }
1043
1044 /*
1045 * This type is designed to indicate the type of offset we would like
1046 * to search from page cache for xfs_seek_hole_data().
1047 */
1048 enum {
1049 HOLE_OFF = 0,
1050 DATA_OFF,
1051 };
1052
1053 /*
1054 * Lookup the desired type of offset from the given page.
1055 *
1056 * On success, return true and the offset argument will point to the
1057 * start of the region that was found. Otherwise this function will
1058 * return false and keep the offset argument unchanged.
1059 */
1060 STATIC bool
xfs_lookup_buffer_offset(struct page * page,loff_t * offset,unsigned int type)1061 xfs_lookup_buffer_offset(
1062 struct page *page,
1063 loff_t *offset,
1064 unsigned int type)
1065 {
1066 loff_t lastoff = page_offset(page);
1067 bool found = false;
1068 struct buffer_head *bh, *head;
1069
1070 bh = head = page_buffers(page);
1071 do {
1072 /*
1073 * Unwritten extents that have data in the page
1074 * cache covering them can be identified by the
1075 * BH_Unwritten state flag. Pages with multiple
1076 * buffers might have a mix of holes, data and
1077 * unwritten extents - any buffer with valid
1078 * data in it should have BH_Uptodate flag set
1079 * on it.
1080 */
1081 if (buffer_unwritten(bh) ||
1082 buffer_uptodate(bh)) {
1083 if (type == DATA_OFF)
1084 found = true;
1085 } else {
1086 if (type == HOLE_OFF)
1087 found = true;
1088 }
1089
1090 if (found) {
1091 *offset = lastoff;
1092 break;
1093 }
1094 lastoff += bh->b_size;
1095 } while ((bh = bh->b_this_page) != head);
1096
1097 return found;
1098 }
1099
1100 /*
1101 * This routine is called to find out and return a data or hole offset
1102 * from the page cache for unwritten extents according to the desired
1103 * type for xfs_seek_hole_data().
1104 *
1105 * The argument offset is used to tell where we start to search from the
1106 * page cache. Map is used to figure out the end points of the range to
1107 * lookup pages.
1108 *
1109 * Return true if the desired type of offset was found, and the argument
1110 * offset is filled with that address. Otherwise, return false and keep
1111 * offset unchanged.
1112 */
1113 STATIC bool
xfs_find_get_desired_pgoff(struct inode * inode,struct xfs_bmbt_irec * map,unsigned int type,loff_t * offset)1114 xfs_find_get_desired_pgoff(
1115 struct inode *inode,
1116 struct xfs_bmbt_irec *map,
1117 unsigned int type,
1118 loff_t *offset)
1119 {
1120 struct xfs_inode *ip = XFS_I(inode);
1121 struct xfs_mount *mp = ip->i_mount;
1122 struct pagevec pvec;
1123 pgoff_t index;
1124 pgoff_t end;
1125 loff_t endoff;
1126 loff_t startoff = *offset;
1127 loff_t lastoff = startoff;
1128 bool found = false;
1129
1130 pagevec_init(&pvec, 0);
1131
1132 index = startoff >> PAGE_SHIFT;
1133 endoff = XFS_FSB_TO_B(mp, map->br_startoff + map->br_blockcount);
1134 end = (endoff - 1) >> PAGE_SHIFT;
1135 do {
1136 int want;
1137 unsigned nr_pages;
1138 unsigned int i;
1139
1140 want = min_t(pgoff_t, end - index, PAGEVEC_SIZE - 1) + 1;
1141 nr_pages = pagevec_lookup(&pvec, inode->i_mapping, index,
1142 want);
1143 if (nr_pages == 0)
1144 break;
1145
1146 for (i = 0; i < nr_pages; i++) {
1147 struct page *page = pvec.pages[i];
1148 loff_t b_offset;
1149
1150 /*
1151 * At this point, the page may be truncated or
1152 * invalidated (changing page->mapping to NULL),
1153 * or even swizzled back from swapper_space to tmpfs
1154 * file mapping. However, page->index will not change
1155 * because we have a reference on the page.
1156 *
1157 * If current page offset is beyond where we've ended,
1158 * we've found a hole.
1159 */
1160 if (type == HOLE_OFF && lastoff < endoff &&
1161 lastoff < page_offset(pvec.pages[i])) {
1162 found = true;
1163 *offset = lastoff;
1164 goto out;
1165 }
1166 /* Searching done if the page index is out of range. */
1167 if (page->index > end)
1168 goto out;
1169
1170 lock_page(page);
1171 /*
1172 * Page truncated or invalidated(page->mapping == NULL).
1173 * We can freely skip it and proceed to check the next
1174 * page.
1175 */
1176 if (unlikely(page->mapping != inode->i_mapping)) {
1177 unlock_page(page);
1178 continue;
1179 }
1180
1181 if (!page_has_buffers(page)) {
1182 unlock_page(page);
1183 continue;
1184 }
1185
1186 found = xfs_lookup_buffer_offset(page, &b_offset, type);
1187 if (found) {
1188 /*
1189 * The found offset may be less than the start
1190 * point to search if this is the first time to
1191 * come here.
1192 */
1193 *offset = max_t(loff_t, startoff, b_offset);
1194 unlock_page(page);
1195 goto out;
1196 }
1197
1198 /*
1199 * We either searching data but nothing was found, or
1200 * searching hole but found a data buffer. In either
1201 * case, probably the next page contains the desired
1202 * things, update the last offset to it so.
1203 */
1204 lastoff = page_offset(page) + PAGE_SIZE;
1205 unlock_page(page);
1206 }
1207
1208 /*
1209 * The number of returned pages less than our desired, search
1210 * done.
1211 */
1212 if (nr_pages < want)
1213 break;
1214
1215 index = pvec.pages[i - 1]->index + 1;
1216 pagevec_release(&pvec);
1217 } while (index <= end);
1218
1219 /* No page at lastoff and we are not done - we found a hole. */
1220 if (type == HOLE_OFF && lastoff < endoff) {
1221 *offset = lastoff;
1222 found = true;
1223 }
1224 out:
1225 pagevec_release(&pvec);
1226 return found;
1227 }
1228
1229 /*
1230 * caller must lock inode with xfs_ilock_data_map_shared,
1231 * can we craft an appropriate ASSERT?
1232 *
1233 * end is because the VFS-level lseek interface is defined such that any
1234 * offset past i_size shall return -ENXIO, but we use this for quota code
1235 * which does not maintain i_size, and we want to SEEK_DATA past i_size.
1236 */
1237 loff_t
__xfs_seek_hole_data(struct inode * inode,loff_t start,loff_t end,int whence)1238 __xfs_seek_hole_data(
1239 struct inode *inode,
1240 loff_t start,
1241 loff_t end,
1242 int whence)
1243 {
1244 struct xfs_inode *ip = XFS_I(inode);
1245 struct xfs_mount *mp = ip->i_mount;
1246 loff_t uninitialized_var(offset);
1247 xfs_fileoff_t fsbno;
1248 xfs_filblks_t lastbno;
1249 int error;
1250
1251 if (start >= end) {
1252 error = -ENXIO;
1253 goto out_error;
1254 }
1255
1256 /*
1257 * Try to read extents from the first block indicated
1258 * by fsbno to the end block of the file.
1259 */
1260 fsbno = XFS_B_TO_FSBT(mp, start);
1261 lastbno = XFS_B_TO_FSB(mp, end);
1262
1263 for (;;) {
1264 struct xfs_bmbt_irec map[2];
1265 int nmap = 2;
1266 unsigned int i;
1267
1268 error = xfs_bmapi_read(ip, fsbno, lastbno - fsbno, map, &nmap,
1269 XFS_BMAPI_ENTIRE);
1270 if (error)
1271 goto out_error;
1272
1273 /* No extents at given offset, must be beyond EOF */
1274 if (nmap == 0) {
1275 error = -ENXIO;
1276 goto out_error;
1277 }
1278
1279 for (i = 0; i < nmap; i++) {
1280 offset = max_t(loff_t, start,
1281 XFS_FSB_TO_B(mp, map[i].br_startoff));
1282
1283 /* Landed in the hole we wanted? */
1284 if (whence == SEEK_HOLE &&
1285 map[i].br_startblock == HOLESTARTBLOCK)
1286 goto out;
1287
1288 /* Landed in the data extent we wanted? */
1289 if (whence == SEEK_DATA &&
1290 (map[i].br_startblock == DELAYSTARTBLOCK ||
1291 (map[i].br_state == XFS_EXT_NORM &&
1292 !isnullstartblock(map[i].br_startblock))))
1293 goto out;
1294
1295 /*
1296 * Landed in an unwritten extent, try to search
1297 * for hole or data from page cache.
1298 */
1299 if (map[i].br_state == XFS_EXT_UNWRITTEN) {
1300 if (xfs_find_get_desired_pgoff(inode, &map[i],
1301 whence == SEEK_HOLE ? HOLE_OFF : DATA_OFF,
1302 &offset))
1303 goto out;
1304 }
1305 }
1306
1307 /*
1308 * We only received one extent out of the two requested. This
1309 * means we've hit EOF and didn't find what we are looking for.
1310 */
1311 if (nmap == 1) {
1312 /*
1313 * If we were looking for a hole, set offset to
1314 * the end of the file (i.e., there is an implicit
1315 * hole at the end of any file).
1316 */
1317 if (whence == SEEK_HOLE) {
1318 offset = end;
1319 break;
1320 }
1321 /*
1322 * If we were looking for data, it's nowhere to be found
1323 */
1324 ASSERT(whence == SEEK_DATA);
1325 error = -ENXIO;
1326 goto out_error;
1327 }
1328
1329 ASSERT(i > 1);
1330
1331 /*
1332 * Nothing was found, proceed to the next round of search
1333 * if the next reading offset is not at or beyond EOF.
1334 */
1335 fsbno = map[i - 1].br_startoff + map[i - 1].br_blockcount;
1336 start = XFS_FSB_TO_B(mp, fsbno);
1337 if (start >= end) {
1338 if (whence == SEEK_HOLE) {
1339 offset = end;
1340 break;
1341 }
1342 ASSERT(whence == SEEK_DATA);
1343 error = -ENXIO;
1344 goto out_error;
1345 }
1346 }
1347
1348 out:
1349 /*
1350 * If at this point we have found the hole we wanted, the returned
1351 * offset may be bigger than the file size as it may be aligned to
1352 * page boundary for unwritten extents. We need to deal with this
1353 * situation in particular.
1354 */
1355 if (whence == SEEK_HOLE)
1356 offset = min_t(loff_t, offset, end);
1357
1358 return offset;
1359
1360 out_error:
1361 return error;
1362 }
1363
1364 STATIC loff_t
xfs_seek_hole_data(struct file * file,loff_t start,int whence)1365 xfs_seek_hole_data(
1366 struct file *file,
1367 loff_t start,
1368 int whence)
1369 {
1370 struct inode *inode = file->f_mapping->host;
1371 struct xfs_inode *ip = XFS_I(inode);
1372 struct xfs_mount *mp = ip->i_mount;
1373 uint lock;
1374 loff_t offset, end;
1375 int error = 0;
1376
1377 if (XFS_FORCED_SHUTDOWN(mp))
1378 return -EIO;
1379
1380 lock = xfs_ilock_data_map_shared(ip);
1381
1382 end = i_size_read(inode);
1383 offset = __xfs_seek_hole_data(inode, start, end, whence);
1384 if (offset < 0) {
1385 error = offset;
1386 goto out_unlock;
1387 }
1388
1389 offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
1390
1391 out_unlock:
1392 xfs_iunlock(ip, lock);
1393
1394 if (error)
1395 return error;
1396 return offset;
1397 }
1398
1399 STATIC loff_t
xfs_file_llseek(struct file * file,loff_t offset,int whence)1400 xfs_file_llseek(
1401 struct file *file,
1402 loff_t offset,
1403 int whence)
1404 {
1405 switch (whence) {
1406 case SEEK_END:
1407 case SEEK_CUR:
1408 case SEEK_SET:
1409 return generic_file_llseek(file, offset, whence);
1410 case SEEK_HOLE:
1411 case SEEK_DATA:
1412 return xfs_seek_hole_data(file, offset, whence);
1413 default:
1414 return -EINVAL;
1415 }
1416 }
1417
1418 /*
1419 * Locking for serialisation of IO during page faults. This results in a lock
1420 * ordering of:
1421 *
1422 * mmap_sem (MM)
1423 * sb_start_pagefault(vfs, freeze)
1424 * i_mmaplock (XFS - truncate serialisation)
1425 * page_lock (MM)
1426 * i_lock (XFS - extent map serialisation)
1427 */
1428
1429 /*
1430 * mmap()d file has taken write protection fault and is being made writable. We
1431 * can set the page state up correctly for a writable page, which means we can
1432 * do correct delalloc accounting (ENOSPC checking!) and unwritten extent
1433 * mapping.
1434 */
1435 STATIC int
xfs_filemap_page_mkwrite(struct vm_area_struct * vma,struct vm_fault * vmf)1436 xfs_filemap_page_mkwrite(
1437 struct vm_area_struct *vma,
1438 struct vm_fault *vmf)
1439 {
1440 struct inode *inode = file_inode(vma->vm_file);
1441 int ret;
1442
1443 trace_xfs_filemap_page_mkwrite(XFS_I(inode));
1444
1445 sb_start_pagefault(inode->i_sb);
1446 file_update_time(vma->vm_file);
1447 xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1448
1449 if (IS_DAX(inode)) {
1450 ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops);
1451 } else {
1452 ret = iomap_page_mkwrite(vma, vmf, &xfs_iomap_ops);
1453 ret = block_page_mkwrite_return(ret);
1454 }
1455
1456 xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1457 sb_end_pagefault(inode->i_sb);
1458
1459 return ret;
1460 }
1461
1462 STATIC int
xfs_filemap_fault(struct vm_area_struct * vma,struct vm_fault * vmf)1463 xfs_filemap_fault(
1464 struct vm_area_struct *vma,
1465 struct vm_fault *vmf)
1466 {
1467 struct inode *inode = file_inode(vma->vm_file);
1468 int ret;
1469
1470 trace_xfs_filemap_fault(XFS_I(inode));
1471
1472 /* DAX can shortcut the normal fault path on write faults! */
1473 if ((vmf->flags & FAULT_FLAG_WRITE) && IS_DAX(inode))
1474 return xfs_filemap_page_mkwrite(vma, vmf);
1475
1476 xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1477 if (IS_DAX(inode)) {
1478 /*
1479 * we do not want to trigger unwritten extent conversion on read
1480 * faults - that is unnecessary overhead and would also require
1481 * changes to xfs_get_blocks_direct() to map unwritten extent
1482 * ioend for conversion on read-only mappings.
1483 */
1484 ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops);
1485 } else
1486 ret = filemap_fault(vma, vmf);
1487 xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1488
1489 return ret;
1490 }
1491
1492 /*
1493 * Similar to xfs_filemap_fault(), the DAX fault path can call into here on
1494 * both read and write faults. Hence we need to handle both cases. There is no
1495 * ->pmd_mkwrite callout for huge pages, so we have a single function here to
1496 * handle both cases here. @flags carries the information on the type of fault
1497 * occuring.
1498 */
1499 STATIC int
xfs_filemap_pmd_fault(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmd,unsigned int flags)1500 xfs_filemap_pmd_fault(
1501 struct vm_area_struct *vma,
1502 unsigned long addr,
1503 pmd_t *pmd,
1504 unsigned int flags)
1505 {
1506 struct inode *inode = file_inode(vma->vm_file);
1507 struct xfs_inode *ip = XFS_I(inode);
1508 int ret;
1509
1510 if (!IS_DAX(inode))
1511 return VM_FAULT_FALLBACK;
1512
1513 trace_xfs_filemap_pmd_fault(ip);
1514
1515 if (flags & FAULT_FLAG_WRITE) {
1516 sb_start_pagefault(inode->i_sb);
1517 file_update_time(vma->vm_file);
1518 }
1519
1520 xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1521 ret = dax_pmd_fault(vma, addr, pmd, flags, xfs_get_blocks_dax_fault);
1522 xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1523
1524 if (flags & FAULT_FLAG_WRITE)
1525 sb_end_pagefault(inode->i_sb);
1526
1527 return ret;
1528 }
1529
1530 /*
1531 * pfn_mkwrite was originally inteneded to ensure we capture time stamp
1532 * updates on write faults. In reality, it's need to serialise against
1533 * truncate similar to page_mkwrite. Hence we cycle the XFS_MMAPLOCK_SHARED
1534 * to ensure we serialise the fault barrier in place.
1535 */
1536 static int
xfs_filemap_pfn_mkwrite(struct vm_area_struct * vma,struct vm_fault * vmf)1537 xfs_filemap_pfn_mkwrite(
1538 struct vm_area_struct *vma,
1539 struct vm_fault *vmf)
1540 {
1541
1542 struct inode *inode = file_inode(vma->vm_file);
1543 struct xfs_inode *ip = XFS_I(inode);
1544 int ret = VM_FAULT_NOPAGE;
1545 loff_t size;
1546
1547 trace_xfs_filemap_pfn_mkwrite(ip);
1548
1549 sb_start_pagefault(inode->i_sb);
1550 file_update_time(vma->vm_file);
1551
1552 /* check if the faulting page hasn't raced with truncate */
1553 xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
1554 size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1555 if (vmf->pgoff >= size)
1556 ret = VM_FAULT_SIGBUS;
1557 else if (IS_DAX(inode))
1558 ret = dax_pfn_mkwrite(vma, vmf);
1559 xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
1560 sb_end_pagefault(inode->i_sb);
1561 return ret;
1562
1563 }
1564
1565 static const struct vm_operations_struct xfs_file_vm_ops = {
1566 .fault = xfs_filemap_fault,
1567 .pmd_fault = xfs_filemap_pmd_fault,
1568 .map_pages = filemap_map_pages,
1569 .page_mkwrite = xfs_filemap_page_mkwrite,
1570 .pfn_mkwrite = xfs_filemap_pfn_mkwrite,
1571 };
1572
1573 STATIC int
xfs_file_mmap(struct file * filp,struct vm_area_struct * vma)1574 xfs_file_mmap(
1575 struct file *filp,
1576 struct vm_area_struct *vma)
1577 {
1578 file_accessed(filp);
1579 vma->vm_ops = &xfs_file_vm_ops;
1580 if (IS_DAX(file_inode(filp)))
1581 vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
1582 return 0;
1583 }
1584
1585 const struct file_operations xfs_file_operations = {
1586 .llseek = xfs_file_llseek,
1587 .read_iter = xfs_file_read_iter,
1588 .write_iter = xfs_file_write_iter,
1589 .splice_read = generic_file_splice_read,
1590 .splice_write = iter_file_splice_write,
1591 .unlocked_ioctl = xfs_file_ioctl,
1592 #ifdef CONFIG_COMPAT
1593 .compat_ioctl = xfs_file_compat_ioctl,
1594 #endif
1595 .mmap = xfs_file_mmap,
1596 .open = xfs_file_open,
1597 .release = xfs_file_release,
1598 .fsync = xfs_file_fsync,
1599 .get_unmapped_area = thp_get_unmapped_area,
1600 .fallocate = xfs_file_fallocate,
1601 .copy_file_range = xfs_file_copy_range,
1602 .clone_file_range = xfs_file_clone_range,
1603 .dedupe_file_range = xfs_file_dedupe_range,
1604 };
1605
1606 const struct file_operations xfs_dir_file_operations = {
1607 .open = xfs_dir_open,
1608 .read = generic_read_dir,
1609 .iterate_shared = xfs_file_readdir,
1610 .llseek = generic_file_llseek,
1611 .unlocked_ioctl = xfs_file_ioctl,
1612 #ifdef CONFIG_COMPAT
1613 .compat_ioctl = xfs_file_compat_ioctl,
1614 #endif
1615 .fsync = xfs_dir_fsync,
1616 };
1617