1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4 * Copyright (c) 2016-2018 Christoph Hellwig.
5 * All Rights Reserved.
6 */
7 #include "xfs.h"
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_trans.h"
15 #include "xfs_iomap.h"
16 #include "xfs_trace.h"
17 #include "xfs_bmap.h"
18 #include "xfs_bmap_util.h"
19 #include "xfs_reflink.h"
20
21 struct xfs_writepage_ctx {
22 struct iomap_writepage_ctx ctx;
23 unsigned int data_seq;
24 unsigned int cow_seq;
25 };
26
27 static inline struct xfs_writepage_ctx *
XFS_WPC(struct iomap_writepage_ctx * ctx)28 XFS_WPC(struct iomap_writepage_ctx *ctx)
29 {
30 return container_of(ctx, struct xfs_writepage_ctx, ctx);
31 }
32
33 /*
34 * Fast and loose check if this write could update the on-disk inode size.
35 */
xfs_ioend_is_append(struct iomap_ioend * ioend)36 static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
37 {
38 return ioend->io_offset + ioend->io_size >
39 XFS_I(ioend->io_inode)->i_disk_size;
40 }
41
42 /*
43 * Update on-disk file size now that data has been written to disk.
44 */
45 int
xfs_setfilesize(struct xfs_inode * ip,xfs_off_t offset,size_t size)46 xfs_setfilesize(
47 struct xfs_inode *ip,
48 xfs_off_t offset,
49 size_t size)
50 {
51 struct xfs_mount *mp = ip->i_mount;
52 struct xfs_trans *tp;
53 xfs_fsize_t isize;
54 int error;
55
56 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
57 if (error)
58 return error;
59
60 xfs_ilock(ip, XFS_ILOCK_EXCL);
61 isize = xfs_new_eof(ip, offset + size);
62 if (!isize) {
63 xfs_iunlock(ip, XFS_ILOCK_EXCL);
64 xfs_trans_cancel(tp);
65 return 0;
66 }
67
68 trace_xfs_setfilesize(ip, offset, size);
69
70 ip->i_disk_size = isize;
71 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
72 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
73
74 return xfs_trans_commit(tp);
75 }
76
77 /*
78 * IO write completion.
79 */
80 STATIC void
xfs_end_ioend(struct iomap_ioend * ioend)81 xfs_end_ioend(
82 struct iomap_ioend *ioend)
83 {
84 struct xfs_inode *ip = XFS_I(ioend->io_inode);
85 struct xfs_mount *mp = ip->i_mount;
86 xfs_off_t offset = ioend->io_offset;
87 size_t size = ioend->io_size;
88 unsigned int nofs_flag;
89 int error;
90
91 /*
92 * We can allocate memory here while doing writeback on behalf of
93 * memory reclaim. To avoid memory allocation deadlocks set the
94 * task-wide nofs context for the following operations.
95 */
96 nofs_flag = memalloc_nofs_save();
97
98 /*
99 * Just clean up the in-memory structures if the fs has been shut down.
100 */
101 if (xfs_is_shutdown(mp)) {
102 error = -EIO;
103 goto done;
104 }
105
106 /*
107 * Clean up all COW blocks and underlying data fork delalloc blocks on
108 * I/O error. The delalloc punch is required because this ioend was
109 * mapped to blocks in the COW fork and the associated pages are no
110 * longer dirty. If we don't remove delalloc blocks here, they become
111 * stale and can corrupt free space accounting on unmount.
112 */
113 error = blk_status_to_errno(ioend->io_bio->bi_status);
114 if (unlikely(error)) {
115 if (ioend->io_flags & IOMAP_F_SHARED) {
116 xfs_reflink_cancel_cow_range(ip, offset, size, true);
117 xfs_bmap_punch_delalloc_range(ip,
118 XFS_B_TO_FSBT(mp, offset),
119 XFS_B_TO_FSB(mp, size));
120 }
121 goto done;
122 }
123
124 /*
125 * Success: commit the COW or unwritten blocks if needed.
126 */
127 if (ioend->io_flags & IOMAP_F_SHARED)
128 error = xfs_reflink_end_cow(ip, offset, size);
129 else if (ioend->io_type == IOMAP_UNWRITTEN)
130 error = xfs_iomap_write_unwritten(ip, offset, size, false);
131
132 if (!error && xfs_ioend_is_append(ioend))
133 error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
134 done:
135 iomap_finish_ioends(ioend, error);
136 memalloc_nofs_restore(nofs_flag);
137 }
138
139 /* Finish all pending io completions. */
140 void
xfs_end_io(struct work_struct * work)141 xfs_end_io(
142 struct work_struct *work)
143 {
144 struct xfs_inode *ip =
145 container_of(work, struct xfs_inode, i_ioend_work);
146 struct iomap_ioend *ioend;
147 struct list_head tmp;
148 unsigned long flags;
149
150 spin_lock_irqsave(&ip->i_ioend_lock, flags);
151 list_replace_init(&ip->i_ioend_list, &tmp);
152 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
153
154 iomap_sort_ioends(&tmp);
155 while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
156 io_list))) {
157 list_del_init(&ioend->io_list);
158 iomap_ioend_try_merge(ioend, &tmp);
159 xfs_end_ioend(ioend);
160 }
161 }
162
163 STATIC void
xfs_end_bio(struct bio * bio)164 xfs_end_bio(
165 struct bio *bio)
166 {
167 struct iomap_ioend *ioend = bio->bi_private;
168 struct xfs_inode *ip = XFS_I(ioend->io_inode);
169 unsigned long flags;
170
171 spin_lock_irqsave(&ip->i_ioend_lock, flags);
172 if (list_empty(&ip->i_ioend_list))
173 WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
174 &ip->i_ioend_work));
175 list_add_tail(&ioend->io_list, &ip->i_ioend_list);
176 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
177 }
178
179 /*
180 * Fast revalidation of the cached writeback mapping. Return true if the current
181 * mapping is valid, false otherwise.
182 */
183 static bool
xfs_imap_valid(struct iomap_writepage_ctx * wpc,struct xfs_inode * ip,loff_t offset)184 xfs_imap_valid(
185 struct iomap_writepage_ctx *wpc,
186 struct xfs_inode *ip,
187 loff_t offset)
188 {
189 if (offset < wpc->iomap.offset ||
190 offset >= wpc->iomap.offset + wpc->iomap.length)
191 return false;
192 /*
193 * If this is a COW mapping, it is sufficient to check that the mapping
194 * covers the offset. Be careful to check this first because the caller
195 * can revalidate a COW mapping without updating the data seqno.
196 */
197 if (wpc->iomap.flags & IOMAP_F_SHARED)
198 return true;
199
200 /*
201 * This is not a COW mapping. Check the sequence number of the data fork
202 * because concurrent changes could have invalidated the extent. Check
203 * the COW fork because concurrent changes since the last time we
204 * checked (and found nothing at this offset) could have added
205 * overlapping blocks.
206 */
207 if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
208 return false;
209 if (xfs_inode_has_cow_data(ip) &&
210 XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
211 return false;
212 return true;
213 }
214
215 /*
216 * Pass in a dellalloc extent and convert it to real extents, return the real
217 * extent that maps offset_fsb in wpc->iomap.
218 *
219 * The current page is held locked so nothing could have removed the block
220 * backing offset_fsb, although it could have moved from the COW to the data
221 * fork by another thread.
222 */
223 static int
xfs_convert_blocks(struct iomap_writepage_ctx * wpc,struct xfs_inode * ip,int whichfork,loff_t offset)224 xfs_convert_blocks(
225 struct iomap_writepage_ctx *wpc,
226 struct xfs_inode *ip,
227 int whichfork,
228 loff_t offset)
229 {
230 int error;
231 unsigned *seq;
232
233 if (whichfork == XFS_COW_FORK)
234 seq = &XFS_WPC(wpc)->cow_seq;
235 else
236 seq = &XFS_WPC(wpc)->data_seq;
237
238 /*
239 * Attempt to allocate whatever delalloc extent currently backs offset
240 * and put the result into wpc->iomap. Allocate in a loop because it
241 * may take several attempts to allocate real blocks for a contiguous
242 * delalloc extent if free space is sufficiently fragmented.
243 */
244 do {
245 error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
246 &wpc->iomap, seq);
247 if (error)
248 return error;
249 } while (wpc->iomap.offset + wpc->iomap.length <= offset);
250
251 return 0;
252 }
253
254 static int
xfs_map_blocks(struct iomap_writepage_ctx * wpc,struct inode * inode,loff_t offset)255 xfs_map_blocks(
256 struct iomap_writepage_ctx *wpc,
257 struct inode *inode,
258 loff_t offset)
259 {
260 struct xfs_inode *ip = XFS_I(inode);
261 struct xfs_mount *mp = ip->i_mount;
262 ssize_t count = i_blocksize(inode);
263 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
264 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
265 xfs_fileoff_t cow_fsb;
266 int whichfork;
267 struct xfs_bmbt_irec imap;
268 struct xfs_iext_cursor icur;
269 int retries = 0;
270 int error = 0;
271
272 if (xfs_is_shutdown(mp))
273 return -EIO;
274
275 /*
276 * COW fork blocks can overlap data fork blocks even if the blocks
277 * aren't shared. COW I/O always takes precedent, so we must always
278 * check for overlap on reflink inodes unless the mapping is already a
279 * COW one, or the COW fork hasn't changed from the last time we looked
280 * at it.
281 *
282 * It's safe to check the COW fork if_seq here without the ILOCK because
283 * we've indirectly protected against concurrent updates: writeback has
284 * the page locked, which prevents concurrent invalidations by reflink
285 * and directio and prevents concurrent buffered writes to the same
286 * page. Changes to if_seq always happen under i_lock, which protects
287 * against concurrent updates and provides a memory barrier on the way
288 * out that ensures that we always see the current value.
289 */
290 if (xfs_imap_valid(wpc, ip, offset))
291 return 0;
292
293 /*
294 * If we don't have a valid map, now it's time to get a new one for this
295 * offset. This will convert delayed allocations (including COW ones)
296 * into real extents. If we return without a valid map, it means we
297 * landed in a hole and we skip the block.
298 */
299 retry:
300 cow_fsb = NULLFILEOFF;
301 whichfork = XFS_DATA_FORK;
302 xfs_ilock(ip, XFS_ILOCK_SHARED);
303 ASSERT(!xfs_need_iread_extents(&ip->i_df));
304
305 /*
306 * Check if this is offset is covered by a COW extents, and if yes use
307 * it directly instead of looking up anything in the data fork.
308 */
309 if (xfs_inode_has_cow_data(ip) &&
310 xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
311 cow_fsb = imap.br_startoff;
312 if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
313 XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
314 xfs_iunlock(ip, XFS_ILOCK_SHARED);
315
316 whichfork = XFS_COW_FORK;
317 goto allocate_blocks;
318 }
319
320 /*
321 * No COW extent overlap. Revalidate now that we may have updated
322 * ->cow_seq. If the data mapping is still valid, we're done.
323 */
324 if (xfs_imap_valid(wpc, ip, offset)) {
325 xfs_iunlock(ip, XFS_ILOCK_SHARED);
326 return 0;
327 }
328
329 /*
330 * If we don't have a valid map, now it's time to get a new one for this
331 * offset. This will convert delayed allocations (including COW ones)
332 * into real extents.
333 */
334 if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
335 imap.br_startoff = end_fsb; /* fake a hole past EOF */
336 XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
337 xfs_iunlock(ip, XFS_ILOCK_SHARED);
338
339 /* landed in a hole or beyond EOF? */
340 if (imap.br_startoff > offset_fsb) {
341 imap.br_blockcount = imap.br_startoff - offset_fsb;
342 imap.br_startoff = offset_fsb;
343 imap.br_startblock = HOLESTARTBLOCK;
344 imap.br_state = XFS_EXT_NORM;
345 }
346
347 /*
348 * Truncate to the next COW extent if there is one. This is the only
349 * opportunity to do this because we can skip COW fork lookups for the
350 * subsequent blocks in the mapping; however, the requirement to treat
351 * the COW range separately remains.
352 */
353 if (cow_fsb != NULLFILEOFF &&
354 cow_fsb < imap.br_startoff + imap.br_blockcount)
355 imap.br_blockcount = cow_fsb - imap.br_startoff;
356
357 /* got a delalloc extent? */
358 if (imap.br_startblock != HOLESTARTBLOCK &&
359 isnullstartblock(imap.br_startblock))
360 goto allocate_blocks;
361
362 xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0);
363 trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
364 return 0;
365 allocate_blocks:
366 error = xfs_convert_blocks(wpc, ip, whichfork, offset);
367 if (error) {
368 /*
369 * If we failed to find the extent in the COW fork we might have
370 * raced with a COW to data fork conversion or truncate.
371 * Restart the lookup to catch the extent in the data fork for
372 * the former case, but prevent additional retries to avoid
373 * looping forever for the latter case.
374 */
375 if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
376 goto retry;
377 ASSERT(error != -EAGAIN);
378 return error;
379 }
380
381 /*
382 * Due to merging the return real extent might be larger than the
383 * original delalloc one. Trim the return extent to the next COW
384 * boundary again to force a re-lookup.
385 */
386 if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
387 loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
388
389 if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
390 wpc->iomap.length = cow_offset - wpc->iomap.offset;
391 }
392
393 ASSERT(wpc->iomap.offset <= offset);
394 ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
395 trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
396 return 0;
397 }
398
399 static int
xfs_prepare_ioend(struct iomap_ioend * ioend,int status)400 xfs_prepare_ioend(
401 struct iomap_ioend *ioend,
402 int status)
403 {
404 unsigned int nofs_flag;
405
406 /*
407 * We can allocate memory here while doing writeback on behalf of
408 * memory reclaim. To avoid memory allocation deadlocks set the
409 * task-wide nofs context for the following operations.
410 */
411 nofs_flag = memalloc_nofs_save();
412
413 /* Convert CoW extents to regular */
414 if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
415 status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
416 ioend->io_offset, ioend->io_size);
417 }
418
419 memalloc_nofs_restore(nofs_flag);
420
421 /* send ioends that might require a transaction to the completion wq */
422 if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN ||
423 (ioend->io_flags & IOMAP_F_SHARED))
424 ioend->io_bio->bi_end_io = xfs_end_bio;
425 return status;
426 }
427
428 /*
429 * If the page has delalloc blocks on it, we need to punch them out before we
430 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
431 * inode that can trip up a later direct I/O read operation on the same region.
432 *
433 * We prevent this by truncating away the delalloc regions on the page. Because
434 * they are delalloc, we can do this without needing a transaction. Indeed - if
435 * we get ENOSPC errors, we have to be able to do this truncation without a
436 * transaction as there is no space left for block reservation (typically why we
437 * see a ENOSPC in writeback).
438 */
439 static void
xfs_discard_page(struct page * page,loff_t fileoff)440 xfs_discard_page(
441 struct page *page,
442 loff_t fileoff)
443 {
444 struct inode *inode = page->mapping->host;
445 struct xfs_inode *ip = XFS_I(inode);
446 struct xfs_mount *mp = ip->i_mount;
447 unsigned int pageoff = offset_in_page(fileoff);
448 xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, fileoff);
449 xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, pageoff);
450 int error;
451
452 if (xfs_is_shutdown(mp))
453 goto out_invalidate;
454
455 xfs_alert_ratelimited(mp,
456 "page discard on page "PTR_FMT", inode 0x%llx, offset %llu.",
457 page, ip->i_ino, fileoff);
458
459 error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
460 i_blocks_per_page(inode, page) - pageoff_fsb);
461 if (error && !xfs_is_shutdown(mp))
462 xfs_alert(mp, "page discard unable to remove delalloc mapping.");
463 out_invalidate:
464 iomap_invalidatepage(page, pageoff, PAGE_SIZE - pageoff);
465 }
466
467 static const struct iomap_writeback_ops xfs_writeback_ops = {
468 .map_blocks = xfs_map_blocks,
469 .prepare_ioend = xfs_prepare_ioend,
470 .discard_page = xfs_discard_page,
471 };
472
473 STATIC int
xfs_vm_writepages(struct address_space * mapping,struct writeback_control * wbc)474 xfs_vm_writepages(
475 struct address_space *mapping,
476 struct writeback_control *wbc)
477 {
478 struct xfs_writepage_ctx wpc = { };
479
480 /*
481 * Writing back data in a transaction context can result in recursive
482 * transactions. This is bad, so issue a warning and get out of here.
483 */
484 if (WARN_ON_ONCE(current->journal_info))
485 return 0;
486
487 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
488 return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
489 }
490
491 STATIC int
xfs_dax_writepages(struct address_space * mapping,struct writeback_control * wbc)492 xfs_dax_writepages(
493 struct address_space *mapping,
494 struct writeback_control *wbc)
495 {
496 struct xfs_inode *ip = XFS_I(mapping->host);
497
498 xfs_iflags_clear(ip, XFS_ITRUNCATED);
499 return dax_writeback_mapping_range(mapping,
500 xfs_inode_buftarg(ip)->bt_daxdev, wbc);
501 }
502
503 STATIC sector_t
xfs_vm_bmap(struct address_space * mapping,sector_t block)504 xfs_vm_bmap(
505 struct address_space *mapping,
506 sector_t block)
507 {
508 struct xfs_inode *ip = XFS_I(mapping->host);
509
510 trace_xfs_vm_bmap(ip);
511
512 /*
513 * The swap code (ab-)uses ->bmap to get a block mapping and then
514 * bypasses the file system for actual I/O. We really can't allow
515 * that on reflinks inodes, so we have to skip out here. And yes,
516 * 0 is the magic code for a bmap error.
517 *
518 * Since we don't pass back blockdev info, we can't return bmap
519 * information for rt files either.
520 */
521 if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
522 return 0;
523 return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
524 }
525
526 STATIC int
xfs_vm_readpage(struct file * unused,struct page * page)527 xfs_vm_readpage(
528 struct file *unused,
529 struct page *page)
530 {
531 return iomap_readpage(page, &xfs_read_iomap_ops);
532 }
533
534 STATIC void
xfs_vm_readahead(struct readahead_control * rac)535 xfs_vm_readahead(
536 struct readahead_control *rac)
537 {
538 iomap_readahead(rac, &xfs_read_iomap_ops);
539 }
540
541 static int
xfs_iomap_swapfile_activate(struct swap_info_struct * sis,struct file * swap_file,sector_t * span)542 xfs_iomap_swapfile_activate(
543 struct swap_info_struct *sis,
544 struct file *swap_file,
545 sector_t *span)
546 {
547 sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
548 return iomap_swapfile_activate(sis, swap_file, span,
549 &xfs_read_iomap_ops);
550 }
551
552 const struct address_space_operations xfs_address_space_operations = {
553 .readpage = xfs_vm_readpage,
554 .readahead = xfs_vm_readahead,
555 .writepages = xfs_vm_writepages,
556 .set_page_dirty = __set_page_dirty_nobuffers,
557 .releasepage = iomap_releasepage,
558 .invalidatepage = iomap_invalidatepage,
559 .bmap = xfs_vm_bmap,
560 .direct_IO = noop_direct_IO,
561 .migratepage = iomap_migrate_page,
562 .is_partially_uptodate = iomap_is_partially_uptodate,
563 .error_remove_page = generic_error_remove_page,
564 .swap_activate = xfs_iomap_swapfile_activate,
565 };
566
567 const struct address_space_operations xfs_dax_aops = {
568 .writepages = xfs_dax_writepages,
569 .direct_IO = noop_direct_IO,
570 .set_page_dirty = __set_page_dirty_no_writeback,
571 .invalidatepage = noop_invalidatepage,
572 .swap_activate = xfs_iomap_swapfile_activate,
573 };
574