Lines Matching refs:folio
37 static inline struct iomap_page *to_iomap_page(struct folio *folio) in to_iomap_page() argument
39 if (folio_test_private(folio)) in to_iomap_page()
40 return folio_get_private(folio); in to_iomap_page()
47 iomap_page_create(struct inode *inode, struct folio *folio, unsigned int flags) in iomap_page_create() argument
49 struct iomap_page *iop = to_iomap_page(folio); in iomap_page_create()
50 unsigned int nr_blocks = i_blocks_per_folio(inode, folio); in iomap_page_create()
65 if (folio_test_uptodate(folio)) in iomap_page_create()
67 folio_attach_private(folio, iop); in iomap_page_create()
72 static void iomap_page_release(struct folio *folio) in iomap_page_release() argument
74 struct iomap_page *iop = folio_detach_private(folio); in iomap_page_release()
75 struct inode *inode = folio->mapping->host; in iomap_page_release()
76 unsigned int nr_blocks = i_blocks_per_folio(inode, folio); in iomap_page_release()
83 folio_test_uptodate(folio)); in iomap_page_release()
90 static void iomap_adjust_read_range(struct inode *inode, struct folio *folio, in iomap_adjust_read_range() argument
93 struct iomap_page *iop = to_iomap_page(folio); in iomap_adjust_read_range()
98 size_t poff = offset_in_folio(folio, *pos); in iomap_adjust_read_range()
99 size_t plen = min_t(loff_t, folio_size(folio) - poff, length); in iomap_adjust_read_range()
137 unsigned end = offset_in_folio(folio, isize - 1) >> block_bits; in iomap_adjust_read_range()
147 static void iomap_iop_set_range_uptodate(struct folio *folio, in iomap_iop_set_range_uptodate() argument
150 struct inode *inode = folio->mapping->host; in iomap_iop_set_range_uptodate()
157 if (bitmap_full(iop->uptodate, i_blocks_per_folio(inode, folio))) in iomap_iop_set_range_uptodate()
158 folio_mark_uptodate(folio); in iomap_iop_set_range_uptodate()
162 static void iomap_set_range_uptodate(struct folio *folio, in iomap_set_range_uptodate() argument
166 iomap_iop_set_range_uptodate(folio, iop, off, len); in iomap_set_range_uptodate()
168 folio_mark_uptodate(folio); in iomap_set_range_uptodate()
171 static void iomap_finish_folio_read(struct folio *folio, size_t offset, in iomap_finish_folio_read() argument
174 struct iomap_page *iop = to_iomap_page(folio); in iomap_finish_folio_read()
177 folio_clear_uptodate(folio); in iomap_finish_folio_read()
178 folio_set_error(folio); in iomap_finish_folio_read()
180 iomap_set_range_uptodate(folio, iop, offset, len); in iomap_finish_folio_read()
184 folio_unlock(folio); in iomap_finish_folio_read()
193 iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error); in iomap_read_end_io()
198 struct folio *cur_folio;
214 struct folio *folio) in iomap_read_inline_data() argument
220 size_t offset = offset_in_folio(folio, iomap->offset); in iomap_read_inline_data()
223 if (folio_test_uptodate(folio)) in iomap_read_inline_data()
234 iop = iomap_page_create(iter->inode, folio, iter->flags); in iomap_read_inline_data()
236 iop = to_iomap_page(folio); in iomap_read_inline_data()
238 addr = kmap_local_folio(folio, offset); in iomap_read_inline_data()
242 iomap_set_range_uptodate(folio, iop, offset, PAGE_SIZE - poff); in iomap_read_inline_data()
262 struct folio *folio = ctx->cur_folio; in iomap_readpage_iter() local
269 return iomap_read_inline_data(iter, folio); in iomap_readpage_iter()
272 iop = iomap_page_create(iter->inode, folio, iter->flags); in iomap_readpage_iter()
273 iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen); in iomap_readpage_iter()
278 folio_zero_range(folio, poff, plen); in iomap_readpage_iter()
279 iomap_set_range_uptodate(folio, iop, poff, plen); in iomap_readpage_iter()
290 !bio_add_folio(ctx->bio, folio, plen, poff)) { in iomap_readpage_iter()
291 gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL); in iomap_readpage_iter()
315 bio_add_folio(ctx->bio, folio, plen, poff); in iomap_readpage_iter()
328 int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops) in iomap_read_folio() argument
331 .inode = folio->mapping->host, in iomap_read_folio()
332 .pos = folio_pos(folio), in iomap_read_folio()
333 .len = folio_size(folio), in iomap_read_folio()
336 .cur_folio = folio, in iomap_read_folio()
346 folio_set_error(folio); in iomap_read_folio()
353 folio_unlock(folio); in iomap_read_folio()
437 bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count) in iomap_is_partially_uptodate() argument
439 struct iomap_page *iop = to_iomap_page(folio); in iomap_is_partially_uptodate()
440 struct inode *inode = folio->mapping->host; in iomap_is_partially_uptodate()
447 count = min(folio_size(folio) - from, count); in iomap_is_partially_uptodate()
460 bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags) in iomap_release_folio() argument
462 trace_iomap_release_folio(folio->mapping->host, folio_pos(folio), in iomap_release_folio()
463 folio_size(folio)); in iomap_release_folio()
471 if (folio_test_dirty(folio) || folio_test_writeback(folio)) in iomap_release_folio()
473 iomap_page_release(folio); in iomap_release_folio()
478 void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len) in iomap_invalidate_folio() argument
480 trace_iomap_invalidate_folio(folio->mapping->host, in iomap_invalidate_folio()
481 folio_pos(folio) + offset, len); in iomap_invalidate_folio()
487 if (offset == 0 && len == folio_size(folio)) { in iomap_invalidate_folio()
488 WARN_ON_ONCE(folio_test_writeback(folio)); in iomap_invalidate_folio()
489 folio_cancel_dirty(folio); in iomap_invalidate_folio()
490 iomap_page_release(folio); in iomap_invalidate_folio()
509 static int iomap_read_folio_sync(loff_t block_start, struct folio *folio, in iomap_read_folio_sync() argument
517 bio_add_folio(&bio, folio, plen, poff); in iomap_read_folio_sync()
522 size_t len, struct folio *folio) in __iomap_write_begin() argument
529 unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio); in __iomap_write_begin()
530 size_t from = offset_in_folio(folio, pos), to = from + len; in __iomap_write_begin()
533 if (folio_test_uptodate(folio)) in __iomap_write_begin()
535 folio_clear_error(folio); in __iomap_write_begin()
537 iop = iomap_page_create(iter->inode, folio, iter->flags); in __iomap_write_begin()
542 iomap_adjust_read_range(iter->inode, folio, &block_start, in __iomap_write_begin()
555 folio_zero_segments(folio, poff, from, to, poff + plen); in __iomap_write_begin()
562 status = iomap_read_folio_sync(block_start, folio, in __iomap_write_begin()
567 iomap_set_range_uptodate(folio, iop, poff, plen); in __iomap_write_begin()
574 struct folio *folio) in iomap_write_begin_inline() argument
579 return iomap_read_inline_data(iter, folio); in iomap_write_begin_inline()
583 size_t len, struct folio **foliop) in iomap_write_begin()
587 struct folio *folio; in iomap_write_begin() local
610 folio = __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT, in iomap_write_begin()
612 if (!folio) { in iomap_write_begin()
616 if (pos + len > folio_pos(folio) + folio_size(folio)) in iomap_write_begin()
617 len = folio_pos(folio) + folio_size(folio) - pos; in iomap_write_begin()
620 status = iomap_write_begin_inline(iter, folio); in iomap_write_begin()
622 status = __block_write_begin_int(folio, pos, len, NULL, srcmap); in iomap_write_begin()
624 status = __iomap_write_begin(iter, pos, len, folio); in iomap_write_begin()
629 *foliop = folio; in iomap_write_begin()
633 folio_unlock(folio); in iomap_write_begin()
634 folio_put(folio); in iomap_write_begin()
644 size_t copied, struct folio *folio) in __iomap_write_end() argument
646 struct iomap_page *iop = to_iomap_page(folio); in __iomap_write_end()
647 flush_dcache_folio(folio); in __iomap_write_end()
660 if (unlikely(copied < len && !folio_test_uptodate(folio))) in __iomap_write_end()
662 iomap_set_range_uptodate(folio, iop, offset_in_folio(folio, pos), len); in __iomap_write_end()
663 filemap_dirty_folio(inode->i_mapping, folio); in __iomap_write_end()
668 struct folio *folio, loff_t pos, size_t copied) in iomap_write_end_inline() argument
673 WARN_ON_ONCE(!folio_test_uptodate(folio)); in iomap_write_end_inline()
676 flush_dcache_folio(folio); in iomap_write_end_inline()
677 addr = kmap_local_folio(folio, pos); in iomap_write_end_inline()
687 size_t copied, struct folio *folio) in iomap_write_end() argument
695 ret = iomap_write_end_inline(iter, folio, pos, copied); in iomap_write_end()
698 copied, &folio->page, NULL); in iomap_write_end()
700 ret = __iomap_write_end(iter->inode, pos, len, copied, folio); in iomap_write_end()
712 folio_unlock(folio); in iomap_write_end()
717 page_ops->page_done(iter->inode, pos, ret, &folio->page); in iomap_write_end()
718 folio_put(folio); in iomap_write_end()
735 struct folio *folio; in iomap_write_iter() local
768 status = iomap_write_begin(iter, pos, bytes, &folio); in iomap_write_iter()
772 page = folio_file_page(folio, pos >> PAGE_SHIFT); in iomap_write_iter()
778 status = iomap_write_end(iter, pos, bytes, copied, folio); in iomap_write_iter()
849 struct folio *folio; in iomap_unshare_iter() local
851 status = iomap_write_begin(iter, pos, bytes, &folio); in iomap_unshare_iter()
855 status = iomap_write_end(iter, pos, bytes, bytes, folio); in iomap_unshare_iter()
901 struct folio *folio; in iomap_zero_iter() local
906 status = iomap_write_begin(iter, pos, bytes, &folio); in iomap_zero_iter()
910 offset = offset_in_folio(folio, pos); in iomap_zero_iter()
911 if (bytes > folio_size(folio) - offset) in iomap_zero_iter()
912 bytes = folio_size(folio) - offset; in iomap_zero_iter()
914 folio_zero_range(folio, offset, bytes); in iomap_zero_iter()
915 folio_mark_accessed(folio); in iomap_zero_iter()
917 bytes = iomap_write_end(iter, pos, bytes, bytes, folio); in iomap_zero_iter()
964 struct folio *folio) in iomap_folio_mkwrite_iter() argument
970 ret = __block_write_begin_int(folio, iter->pos, length, NULL, in iomap_folio_mkwrite_iter()
974 block_commit_write(&folio->page, 0, length); in iomap_folio_mkwrite_iter()
976 WARN_ON_ONCE(!folio_test_uptodate(folio)); in iomap_folio_mkwrite_iter()
977 folio_mark_dirty(folio); in iomap_folio_mkwrite_iter()
989 struct folio *folio = page_folio(vmf->page); in iomap_page_mkwrite() local
992 folio_lock(folio); in iomap_page_mkwrite()
993 ret = folio_mkwrite_check_truncate(folio, iter.inode); in iomap_page_mkwrite()
996 iter.pos = folio_pos(folio); in iomap_page_mkwrite()
999 iter.processed = iomap_folio_mkwrite_iter(&iter, folio); in iomap_page_mkwrite()
1003 folio_wait_stable(folio); in iomap_page_mkwrite()
1006 folio_unlock(folio); in iomap_page_mkwrite()
1011 static void iomap_finish_folio_write(struct inode *inode, struct folio *folio, in iomap_finish_folio_write() argument
1014 struct iomap_page *iop = to_iomap_page(folio); in iomap_finish_folio_write()
1017 folio_set_error(folio); in iomap_finish_folio_write()
1021 WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !iop); in iomap_finish_folio_write()
1025 folio_end_writeback(folio); in iomap_finish_folio_write()
1058 iomap_finish_folio_write(inode, fi.folio, fi.length, in iomap_finish_ioend()
1290 iomap_add_to_ioend(struct inode *inode, loff_t pos, struct folio *folio, in iomap_add_to_ioend() argument
1296 size_t poff = offset_in_folio(folio, pos); in iomap_add_to_ioend()
1304 if (!bio_add_folio(wpc->ioend->io_bio, folio, len, poff)) { in iomap_add_to_ioend()
1306 bio_add_folio(wpc->ioend->io_bio, folio, len, poff); in iomap_add_to_ioend()
1312 wbc_account_cgroup_owner(wbc, &folio->page, len); in iomap_add_to_ioend()
1334 struct folio *folio, u64 end_pos) in iomap_writepage_map() argument
1336 struct iomap_page *iop = iomap_page_create(inode, folio, 0); in iomap_writepage_map()
1339 unsigned nblocks = i_blocks_per_folio(inode, folio); in iomap_writepage_map()
1340 u64 pos = folio_pos(folio); in iomap_writepage_map()
1363 iomap_add_to_ioend(inode, pos, folio, iop, wpc, wbc, in iomap_writepage_map()
1371 WARN_ON_ONCE(!folio_test_locked(folio)); in iomap_writepage_map()
1372 WARN_ON_ONCE(folio_test_writeback(folio)); in iomap_writepage_map()
1373 WARN_ON_ONCE(folio_test_dirty(folio)); in iomap_writepage_map()
1389 wpc->ops->discard_folio(folio, pos); in iomap_writepage_map()
1391 folio_unlock(folio); in iomap_writepage_map()
1396 folio_start_writeback(folio); in iomap_writepage_map()
1397 folio_unlock(folio); in iomap_writepage_map()
1418 folio_end_writeback(folio); in iomap_writepage_map()
1434 struct folio *folio = page_folio(page); in iomap_do_writepage() local
1436 struct inode *inode = folio->mapping->host; in iomap_do_writepage()
1439 trace_iomap_writepage(inode, folio_pos(folio), folio_size(folio)); in iomap_do_writepage()
1469 end_pos = folio_pos(folio) + folio_size(folio); in iomap_do_writepage()
1482 size_t poff = offset_in_folio(folio, isize); in iomap_do_writepage()
1502 if (folio->index > end_index || in iomap_do_writepage()
1503 (folio->index == end_index && poff == 0)) in iomap_do_writepage()
1514 folio_zero_segment(folio, poff, folio_size(folio)); in iomap_do_writepage()
1518 return iomap_writepage_map(wpc, wbc, inode, folio, end_pos); in iomap_do_writepage()
1521 folio_redirty_for_writepage(wbc, folio); in iomap_do_writepage()
1523 folio_unlock(folio); in iomap_do_writepage()