Lines Matching +full:page +full:- +full:offset
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) International Business Machines Corp., 2000-2005
4 * Portions Copyright (C) Christoph Hellwig, 2001-2002
25 uint pagealloc; /* # of page allocations */
26 uint pagefree; /* # of page frees */
31 #define metapage_locked(mp) test_bit(META_locked, &(mp)->flag)
32 #define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag)
36 clear_bit_unlock(META_locked, &mp->flag); in unlock_metapage()
37 wake_up(&mp->wait); in unlock_metapage()
44 add_wait_queue_exclusive(&mp->wait, &wait); in __lock_metapage()
48 unlock_page(mp->page); in __lock_metapage()
50 lock_page(mp->page); in __lock_metapage()
54 remove_wait_queue(&mp->wait, &wait); in __lock_metapage()
58 * Must have mp->page locked
79 #define mp_anchor(page) ((struct meta_anchor *)page_private(page)) argument
81 static inline struct metapage *page_to_mp(struct page *page, int offset) in page_to_mp() argument
83 if (!PagePrivate(page)) in page_to_mp()
85 return mp_anchor(page)->mp[offset >> L2PSIZE]; in page_to_mp()
88 static inline int insert_metapage(struct page *page, struct metapage *mp) in insert_metapage() argument
94 if (PagePrivate(page)) in insert_metapage()
95 a = mp_anchor(page); in insert_metapage()
99 return -ENOMEM; in insert_metapage()
100 set_page_private(page, (unsigned long)a); in insert_metapage()
101 SetPagePrivate(page); in insert_metapage()
102 kmap(page); in insert_metapage()
106 l2mp_blocks = L2PSIZE - page->mapping->host->i_blkbits; in insert_metapage()
107 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1); in insert_metapage()
108 a->mp_count++; in insert_metapage()
109 a->mp[index] = mp; in insert_metapage()
115 static inline void remove_metapage(struct page *page, struct metapage *mp) in remove_metapage() argument
117 struct meta_anchor *a = mp_anchor(page); in remove_metapage()
118 int l2mp_blocks = L2PSIZE - page->mapping->host->i_blkbits; in remove_metapage()
121 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1); in remove_metapage()
123 BUG_ON(a->mp[index] != mp); in remove_metapage()
125 a->mp[index] = NULL; in remove_metapage()
126 if (--a->mp_count == 0) { in remove_metapage()
128 set_page_private(page, 0); in remove_metapage()
129 ClearPagePrivate(page); in remove_metapage()
130 kunmap(page); in remove_metapage()
134 static inline void inc_io(struct page *page) in inc_io() argument
136 atomic_inc(&mp_anchor(page)->io_count); in inc_io()
139 static inline void dec_io(struct page *page, void (*handler) (struct page *)) in dec_io() argument
141 if (atomic_dec_and_test(&mp_anchor(page)->io_count)) in dec_io()
142 handler(page); in dec_io()
146 static inline struct metapage *page_to_mp(struct page *page, int offset) in page_to_mp() argument
148 return PagePrivate(page) ? (struct metapage *)page_private(page) : NULL; in page_to_mp()
151 static inline int insert_metapage(struct page *page, struct metapage *mp) in insert_metapage() argument
154 set_page_private(page, (unsigned long)mp); in insert_metapage()
155 SetPagePrivate(page); in insert_metapage()
156 kmap(page); in insert_metapage()
161 static inline void remove_metapage(struct page *page, struct metapage *mp) in remove_metapage() argument
163 set_page_private(page, 0); in remove_metapage()
164 ClearPagePrivate(page); in remove_metapage()
165 kunmap(page); in remove_metapage()
168 #define inc_io(page) do {} while(0) argument
169 #define dec_io(page, handler) handler(page) argument
178 mp->lid = 0; in alloc_metapage()
179 mp->lsn = 0; in alloc_metapage()
180 mp->data = NULL; in alloc_metapage()
181 mp->clsn = 0; in alloc_metapage()
182 mp->log = NULL; in alloc_metapage()
183 init_waitqueue_head(&mp->wait); in alloc_metapage()
201 return -ENOMEM; in metapage_init()
208 return -ENOMEM; in metapage_init()
220 static inline void drop_metapage(struct page *page, struct metapage *mp) in drop_metapage() argument
222 if (mp->count || mp->nohomeok || test_bit(META_dirty, &mp->flag) || in drop_metapage()
223 test_bit(META_io, &mp->flag)) in drop_metapage()
225 remove_metapage(page, mp); in drop_metapage()
240 sector_t file_blocks = (inode->i_size + inode->i_sb->s_blocksize - 1) >> in metapage_get_blocks()
241 inode->i_blkbits; in metapage_get_blocks()
246 *len = file_blocks - lblock; in metapage_get_blocks()
248 if (inode->i_ino) { in metapage_get_blocks()
259 static void last_read_complete(struct page *page) in last_read_complete() argument
261 if (!PageError(page)) in last_read_complete()
262 SetPageUptodate(page); in last_read_complete()
263 unlock_page(page); in last_read_complete()
268 struct page *page = bio->bi_private; in metapage_read_end_io() local
270 if (bio->bi_status) { in metapage_read_end_io()
272 SetPageError(page); in metapage_read_end_io()
275 dec_io(page, last_read_complete); in metapage_read_end_io()
281 struct jfs_log *log = mp->log; in remove_from_logsync()
291 if (mp->lsn) { in remove_from_logsync()
292 mp->log = NULL; in remove_from_logsync()
293 mp->lsn = 0; in remove_from_logsync()
294 mp->clsn = 0; in remove_from_logsync()
295 log->count--; in remove_from_logsync()
296 list_del(&mp->synclist); in remove_from_logsync()
301 static void last_write_complete(struct page *page) in last_write_complete() argument
304 unsigned int offset; in last_write_complete() local
306 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in last_write_complete()
307 mp = page_to_mp(page, offset); in last_write_complete()
308 if (mp && test_bit(META_io, &mp->flag)) { in last_write_complete()
309 if (mp->lsn) in last_write_complete()
311 clear_bit(META_io, &mp->flag); in last_write_complete()
315 * safe unless I have the page locked in last_write_complete()
318 end_page_writeback(page); in last_write_complete()
323 struct page *page = bio->bi_private; in metapage_write_end_io() local
325 BUG_ON(!PagePrivate(page)); in metapage_write_end_io()
327 if (bio->bi_status) { in metapage_write_end_io()
329 SetPageError(page); in metapage_write_end_io()
331 dec_io(page, last_write_complete); in metapage_write_end_io()
335 static int metapage_writepage(struct page *page, struct writeback_control *wbc) in metapage_writepage() argument
338 int block_offset; /* block offset of mp within page */ in metapage_writepage()
339 struct inode *inode = page->mapping->host; in metapage_writepage()
340 int blocks_per_mp = JFS_SBI(inode->i_sb)->nbperpage; in metapage_writepage()
352 int offset; in metapage_writepage() local
355 page_start = (sector_t)page->index << in metapage_writepage()
356 (PAGE_SHIFT - inode->i_blkbits); in metapage_writepage()
357 BUG_ON(!PageLocked(page)); in metapage_writepage()
358 BUG_ON(PageWriteback(page)); in metapage_writepage()
359 set_page_writeback(page); in metapage_writepage()
361 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in metapage_writepage()
362 mp = page_to_mp(page, offset); in metapage_writepage()
364 if (!mp || !test_bit(META_dirty, &mp->flag)) in metapage_writepage()
367 if (mp->nohomeok && !test_bit(META_forcewrite, &mp->flag)) { in metapage_writepage()
370 * Make sure this page isn't blocked indefinitely. in metapage_writepage()
373 if (mp->log && !(mp->log->cflag & logGC_PAGEOUT)) in metapage_writepage()
374 jfs_flush_journal(mp->log, 0); in metapage_writepage()
378 clear_bit(META_dirty, &mp->flag); in metapage_writepage()
379 set_bit(META_io, &mp->flag); in metapage_writepage()
380 block_offset = offset >> inode->i_blkbits; in metapage_writepage()
386 xlen -= len; in metapage_writepage()
387 bio_bytes += len << inode->i_blkbits; in metapage_writepage()
391 if (bio_add_page(bio, page, bio_bytes, bio_offset) < in metapage_writepage()
398 inc_io(page); in metapage_writepage()
399 if (!bio->bi_iter.bi_size) in metapage_writepage()
405 inc_io(page); in metapage_writepage()
406 xlen = (PAGE_SIZE - offset) >> inode->i_blkbits; in metapage_writepage()
412 * with dec_io() until we're done with the page in metapage_writepage()
417 len = min(xlen, (int)JFS_SBI(inode->i_sb)->nbperpage); in metapage_writepage()
420 bio_set_dev(bio, inode->i_sb->s_bdev); in metapage_writepage()
421 bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9); in metapage_writepage()
422 bio->bi_end_io = metapage_write_end_io; in metapage_writepage()
423 bio->bi_private = page; in metapage_writepage()
427 bio_offset = offset; in metapage_writepage()
428 bio_bytes = len << inode->i_blkbits; in metapage_writepage()
430 xlen -= len; in metapage_writepage()
434 if (bio_add_page(bio, page, bio_bytes, bio_offset) < bio_bytes) in metapage_writepage()
436 if (!bio->bi_iter.bi_size) in metapage_writepage()
443 redirty_page_for_writepage(wbc, page); in metapage_writepage()
445 unlock_page(page); in metapage_writepage()
451 end_page_writeback(page); in metapage_writepage()
463 unlock_page(page); in metapage_writepage()
464 dec_io(page, last_write_complete); in metapage_writepage()
466 while (bad_blocks--) in metapage_writepage()
467 dec_io(page, last_write_complete); in metapage_writepage()
468 return -EIO; in metapage_writepage()
471 static int metapage_readpage(struct file *fp, struct page *page) in metapage_readpage() argument
473 struct inode *inode = page->mapping->host; in metapage_readpage()
476 int blocks_per_page = i_blocks_per_page(inode, page); in metapage_readpage()
477 sector_t page_start; /* address of page in fs blocks */ in metapage_readpage()
481 int offset; in metapage_readpage() local
483 BUG_ON(!PageLocked(page)); in metapage_readpage()
484 page_start = (sector_t)page->index << in metapage_readpage()
485 (PAGE_SHIFT - inode->i_blkbits); in metapage_readpage()
489 xlen = blocks_per_page - block_offset; in metapage_readpage()
493 if (!PagePrivate(page)) in metapage_readpage()
494 insert_metapage(page, NULL); in metapage_readpage()
495 inc_io(page); in metapage_readpage()
500 bio_set_dev(bio, inode->i_sb->s_bdev); in metapage_readpage()
501 bio->bi_iter.bi_sector = in metapage_readpage()
502 pblock << (inode->i_blkbits - 9); in metapage_readpage()
503 bio->bi_end_io = metapage_read_end_io; in metapage_readpage()
504 bio->bi_private = page; in metapage_readpage()
506 len = xlen << inode->i_blkbits; in metapage_readpage()
507 offset = block_offset << inode->i_blkbits; in metapage_readpage()
508 if (bio_add_page(bio, page, len, offset) < len) in metapage_readpage()
517 unlock_page(page); in metapage_readpage()
524 dec_io(page, last_read_complete); in metapage_readpage()
525 return -EIO; in metapage_readpage()
528 static int metapage_releasepage(struct page *page, gfp_t gfp_mask) in metapage_releasepage() argument
532 int offset; in metapage_releasepage() local
534 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in metapage_releasepage()
535 mp = page_to_mp(page, offset); in metapage_releasepage()
541 if (mp->count || mp->nohomeok || in metapage_releasepage()
542 test_bit(META_dirty, &mp->flag)) { in metapage_releasepage()
543 jfs_info("count = %ld, nohomeok = %d", mp->count, in metapage_releasepage()
544 mp->nohomeok); in metapage_releasepage()
548 if (mp->lsn) in metapage_releasepage()
550 remove_metapage(page, mp); in metapage_releasepage()
557 static void metapage_invalidatepage(struct page *page, unsigned int offset, in metapage_invalidatepage() argument
560 BUG_ON(offset || length < PAGE_SIZE); in metapage_invalidatepage()
562 BUG_ON(PageWriteback(page)); in metapage_invalidatepage()
564 metapage_releasepage(page, 0); in metapage_invalidatepage()
583 struct page *page; in __get_metapage() local
588 inode->i_ino, lblock, absolute); in __get_metapage()
590 l2bsize = inode->i_blkbits; in __get_metapage()
591 l2BlocksPerPage = PAGE_SHIFT - l2bsize; in __get_metapage()
593 page_offset = (lblock - (page_index << l2BlocksPerPage)) << l2bsize; in __get_metapage()
595 jfs_err("MetaData crosses page boundary!!"); in __get_metapage()
601 mapping = JFS_SBI(inode->i_sb)->direct_inode->i_mapping; in __get_metapage()
608 if ((lblock << inode->i_blkbits) >= inode->i_size) in __get_metapage()
610 mapping = inode->i_mapping; in __get_metapage()
614 page = grab_cache_page(mapping, page_index); in __get_metapage()
615 if (!page) { in __get_metapage()
619 SetPageUptodate(page); in __get_metapage()
621 page = read_mapping_page(mapping, page_index, NULL); in __get_metapage()
622 if (IS_ERR(page) || !PageUptodate(page)) { in __get_metapage()
626 lock_page(page); in __get_metapage()
629 mp = page_to_mp(page, page_offset); in __get_metapage()
631 if (mp->logical_size != size) { in __get_metapage()
632 jfs_error(inode->i_sb, in __get_metapage()
633 "get_mp->logical_size != size\n"); in __get_metapage()
635 mp->logical_size, size); in __get_metapage()
639 mp->count++; in __get_metapage()
641 if (test_bit(META_discard, &mp->flag)) { in __get_metapage()
643 jfs_error(inode->i_sb, in __get_metapage()
648 clear_bit(META_discard, &mp->flag); in __get_metapage()
655 mp->page = page; in __get_metapage()
656 mp->sb = inode->i_sb; in __get_metapage()
657 mp->flag = 0; in __get_metapage()
658 mp->xflag = COMMIT_PAGE; in __get_metapage()
659 mp->count = 1; in __get_metapage()
660 mp->nohomeok = 0; in __get_metapage()
661 mp->logical_size = size; in __get_metapage()
662 mp->data = page_address(page) + page_offset; in __get_metapage()
663 mp->index = lblock; in __get_metapage()
664 if (unlikely(insert_metapage(page, mp))) { in __get_metapage()
673 memset(mp->data, 0, PSIZE); in __get_metapage()
676 unlock_page(page); in __get_metapage()
677 jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp, mp->data); in __get_metapage()
681 unlock_page(page); in __get_metapage()
688 get_page(mp->page); in grab_metapage()
689 lock_page(mp->page); in grab_metapage()
690 mp->count++; in grab_metapage()
692 unlock_page(mp->page); in grab_metapage()
697 struct page *page = mp->page; in force_metapage() local
699 set_bit(META_forcewrite, &mp->flag); in force_metapage()
700 clear_bit(META_sync, &mp->flag); in force_metapage()
701 get_page(page); in force_metapage()
702 lock_page(page); in force_metapage()
703 set_page_dirty(page); in force_metapage()
704 if (write_one_page(page)) in force_metapage()
705 jfs_error(mp->sb, "write_one_page() failed\n"); in force_metapage()
706 clear_bit(META_forcewrite, &mp->flag); in force_metapage()
707 put_page(page); in force_metapage()
712 lock_page(mp->page); in hold_metapage()
717 if (mp->count || mp->nohomeok) { in put_metapage()
719 unlock_page(mp->page); in put_metapage()
722 get_page(mp->page); in put_metapage()
723 mp->count++; in put_metapage()
725 unlock_page(mp->page); in put_metapage()
731 struct page *page = mp->page; in release_metapage() local
732 jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag); in release_metapage()
734 BUG_ON(!page); in release_metapage()
736 lock_page(page); in release_metapage()
739 assert(mp->count); in release_metapage()
740 if (--mp->count || mp->nohomeok) { in release_metapage()
741 unlock_page(page); in release_metapage()
742 put_page(page); in release_metapage()
746 if (test_bit(META_dirty, &mp->flag)) { in release_metapage()
747 set_page_dirty(page); in release_metapage()
748 if (test_bit(META_sync, &mp->flag)) { in release_metapage()
749 clear_bit(META_sync, &mp->flag); in release_metapage()
750 if (write_one_page(page)) in release_metapage()
751 jfs_error(mp->sb, "write_one_page() failed\n"); in release_metapage()
752 lock_page(page); /* write_one_page unlocks the page */ in release_metapage()
754 } else if (mp->lsn) /* discard_metapage doesn't remove it */ in release_metapage()
758 drop_metapage(page, mp); in release_metapage()
760 unlock_page(page); in release_metapage()
761 put_page(page); in release_metapage()
767 int l2BlocksPerPage = PAGE_SHIFT - ip->i_blkbits; in __invalidate_metapages()
771 JFS_SBI(ip->i_sb)->direct_inode->i_mapping; in __invalidate_metapages()
773 struct page *page; in __invalidate_metapages() local
774 unsigned int offset; in __invalidate_metapages() local
780 for (lblock = addr & ~(BlocksPerPage - 1); lblock < addr + len; in __invalidate_metapages()
782 page = find_lock_page(mapping, lblock >> l2BlocksPerPage); in __invalidate_metapages()
783 if (!page) in __invalidate_metapages()
785 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in __invalidate_metapages()
786 mp = page_to_mp(page, offset); in __invalidate_metapages()
789 if (mp->index < addr) in __invalidate_metapages()
791 if (mp->index >= addr + len) in __invalidate_metapages()
794 clear_bit(META_dirty, &mp->flag); in __invalidate_metapages()
795 set_bit(META_discard, &mp->flag); in __invalidate_metapages()
796 if (mp->lsn) in __invalidate_metapages()
799 unlock_page(page); in __invalidate_metapages()
800 put_page(page); in __invalidate_metapages()
810 "page allocations = %d\n" in jfs_mpstat_proc_show()
811 "page frees = %d\n" in jfs_mpstat_proc_show()