1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/truncate.c - code for taking down pages from address_spaces
12 #include <linux/backing-dev.h>
37 	XA_STATE(xas, &mapping->i_pages, index);  in __clear_shadow_entry()
43 	mapping->nrexceptional--;  in __clear_shadow_entry()
49 	xa_lock_irq(&mapping->i_pages);  in clear_shadow_entry()
51 	xa_unlock_irq(&mapping->i_pages);  in clear_shadow_entry()
71 		if (xa_is_value(pvec->pages[j]))  in truncate_exceptional_pvec_entries()
80 		xa_lock_irq(&mapping->i_pages);  in truncate_exceptional_pvec_entries()
83 		struct page *page = pvec->pages[i];  in truncate_exceptional_pvec_entries()
87 			pvec->pages[j++] = page;  in truncate_exceptional_pvec_entries()
103 		xa_unlock_irq(&mapping->i_pages);  in truncate_exceptional_pvec_entries()
104 	pvec->nr = j;  in truncate_exceptional_pvec_entries()
138  * do_invalidatepage - invalidate part or all of a page
150  * blocks on-disk.
157 	invalidatepage = page->mapping->a_ops->invalidatepage;  in do_invalidatepage()
167  * If truncate cannot remove the fs-private metadata from the page, the page
171  * We need to bail out if page->mapping is no longer equal to the original
185 	 * Some filesystems seem to re-dirty the page even after  in truncate_cleanup_page()
199  * Returns non-zero if the page was successfully invalidated.
206 	if (page->mapping != mapping)  in invalidate_complete_page()
221 	if (page->mapping != mapping)  in truncate_inode_page()
222 		return -EIO;  in truncate_inode_page()
235 		return -EINVAL;  in generic_error_remove_page()
240 	if (!S_ISREG(mapping->host->i_mode))  in generic_error_remove_page()
241 		return -EIO;  in generic_error_remove_page()
265  * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
268  * @lend: offset to which to truncate (inclusive)
274  * Truncate takes two passes - the first pass is nonblocking.  It will not
280  * We pass down the cache-hot hint to the page freeing code.  Even if the
284  * Note that since ->invalidatepage() accepts range to invalidate
291 	pgoff_t		start;		/* inclusive */  in truncate_inode_pages_range()
293 	unsigned int	partial_start;	/* inclusive */  in truncate_inode_pages_range()
300 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in truncate_inode_pages_range()
304 	partial_start = lstart & (PAGE_SIZE - 1);  in truncate_inode_pages_range()
305 	partial_end = (lend + 1) & (PAGE_SIZE - 1);  in truncate_inode_pages_range()
311 	 * Note that 'end' is exclusive while 'lend' is inclusive.  in truncate_inode_pages_range()
313 	start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;  in truncate_inode_pages_range()
314 	if (lend == -1)  in truncate_inode_pages_range()
316 		 * lend == -1 indicates end-of-file so we have to set 'end'  in truncate_inode_pages_range()
318 		 * unsigned we're using -1.  in truncate_inode_pages_range()
320 		end = -1;  in truncate_inode_pages_range()
327 			min(end - index, (pgoff_t)PAGEVEC_SIZE),  in truncate_inode_pages_range()
340 			/* We rely upon deletion not changing page->index */  in truncate_inode_pages_range()
355 			if (page->mapping != mapping) {  in truncate_inode_pages_range()
372 		struct page *page = find_lock_page(mapping, start - 1);  in truncate_inode_pages_range()
385 						  top - partial_start);  in truncate_inode_pages_range()
414 			min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {  in truncate_inode_pages_range()
432 			/* We rely upon deletion not changing page->index */  in truncate_inode_pages_range()
436 				index = start - 1;  in truncate_inode_pages_range()
460  * truncate_inode_pages - truncate *all* the pages from an offset
464  * Called under (and serialised by) inode->i_mutex.
468  * mapping->nrpages can be non-zero when this function returns even after
473 	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);  in truncate_inode_pages()
478  * truncate_inode_pages_final - truncate *all* pages before inode dies
481  * Called under (and serialized by) inode->i_mutex.
505 	nrpages = mapping->nrpages;  in truncate_inode_pages_final()
507 	nrexceptional = mapping->nrexceptional;  in truncate_inode_pages_final()
516 		xa_lock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
517 		xa_unlock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
540 			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,  in __invalidate_mapping_pages()
545 			/* We rely upon deletion not changing page->index */  in __invalidate_mapping_pages()
566 				index += HPAGE_PMD_NR - 1;  in __invalidate_mapping_pages()
567 				i += HPAGE_PMD_NR - 1;  in __invalidate_mapping_pages()
615  * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
618  * @end: the offset 'to' which to invalidate (inclusive)
659 	if (page->mapping != mapping)  in invalidate_complete_page2()
665 	xa_lock_irqsave(&mapping->i_pages, flags);  in invalidate_complete_page2()
671 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in invalidate_complete_page2()
673 	if (mapping->a_ops->freepage)  in invalidate_complete_page2()
674 		mapping->a_ops->freepage(page);  in invalidate_complete_page2()
679 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in invalidate_complete_page2()
687 	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)  in do_launder_page()
689 	return mapping->a_ops->launder_page(page);  in do_launder_page()
693  * invalidate_inode_pages2_range - remove range of pages from an address_space
696  * @end: the page offset 'to' which to invalidate (inclusive)
701  * Return: -EBUSY if any pages could not be invalidated.
714 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in invalidate_inode_pages2_range()
720 			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,  in invalidate_inode_pages2_range()
725 			/* We rely upon deletion not changing page->index */  in invalidate_inode_pages2_range()
733 					ret = -EBUSY;  in invalidate_inode_pages2_range()
743 						(1 + end - index), false);  in invalidate_inode_pages2_range()
749 			if (page->mapping != mapping) {  in invalidate_inode_pages2_range()
762 					ret2 = -EBUSY;  in invalidate_inode_pages2_range()
781 		unmap_mapping_pages(mapping, start, end - start + 1, false);  in invalidate_inode_pages2_range()
790  * invalidate_inode_pages2 - remove all pages from an address_space
796  * Return: -EBUSY if any pages could not be invalidated.
800 	return invalidate_inode_pages2_range(mapping, 0, -1);  in invalidate_inode_pages2()
805  * truncate_pagecache - unmap and remove pagecache that has been truncated
815  * with on-disk format, and the filesystem would not have to deal with
821 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache()
827 	 * single-page unmaps.  However after this first call, and  in truncate_pagecache()
840  * truncate_setsize - update inode and pagecache for a new file size
854 	loff_t oldsize = inode->i_size;  in truncate_setsize()
864  * pagecache_isize_extended - update pagecache after extension of i_size
878  * The function must be called while we still hold i_mutex - this not only
889 	WARN_ON(to > inode->i_size);  in pagecache_isize_extended()
895 	if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))  in pagecache_isize_extended()
899 	page = find_lock_page(inode->i_mapping, index);  in pagecache_isize_extended()
915  * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
923  * with on-disk format, and the filesystem would not have to deal with
929 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache_range()
931 	loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;  in truncate_pagecache_range()
937 	 * allows holelen 0 for all, and we allow lend -1 for end of file.  in truncate_pagecache_range()
943 	 * hole-punching should not remove private COWed pages from the hole.  in truncate_pagecache_range()
947 				    1 + unmap_end - unmap_start, 0);  in truncate_pagecache_range()