33 static inline void __clear_shadow_entry(struct address_space *mapping,  in __clear_shadow_entry()  argument
39 	if (!__radix_tree_lookup(&mapping->i_pages, index, &node, &slot))  in __clear_shadow_entry()
43 	__radix_tree_replace(&mapping->i_pages, node, slot, NULL,  in __clear_shadow_entry()
45 	mapping->nrexceptional--;  in __clear_shadow_entry()
48 static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,  in clear_shadow_entry()  argument
51 	xa_lock_irq(&mapping->i_pages);  in clear_shadow_entry()
52 	__clear_shadow_entry(mapping, index, entry);  in clear_shadow_entry()
53 	xa_unlock_irq(&mapping->i_pages);  in clear_shadow_entry()
61 static void truncate_exceptional_pvec_entries(struct address_space *mapping,  in truncate_exceptional_pvec_entries()  argument
69 	if (shmem_mapping(mapping))  in truncate_exceptional_pvec_entries()
79 	dax = dax_mapping(mapping);  in truncate_exceptional_pvec_entries()
82 		xa_lock_irq(&mapping->i_pages);  in truncate_exceptional_pvec_entries()
97 			dax_delete_mapping_entry(mapping, index);  in truncate_exceptional_pvec_entries()
101 		__clear_shadow_entry(mapping, index, page);  in truncate_exceptional_pvec_entries()
105 		xa_unlock_irq(&mapping->i_pages);  in truncate_exceptional_pvec_entries()
113 static int invalidate_exceptional_entry(struct address_space *mapping,  in invalidate_exceptional_entry()  argument
117 	if (shmem_mapping(mapping) || dax_mapping(mapping))  in invalidate_exceptional_entry()
119 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry()
127 static int invalidate_exceptional_entry2(struct address_space *mapping,  in invalidate_exceptional_entry2()  argument
131 	if (shmem_mapping(mapping))  in invalidate_exceptional_entry2()
133 	if (dax_mapping(mapping))  in invalidate_exceptional_entry2()
134 		return dax_invalidate_mapping_entry_sync(mapping, index);  in invalidate_exceptional_entry2()
135 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry2()
159 	invalidatepage = page->mapping->a_ops->invalidatepage;  in do_invalidatepage()
173  * We need to bale out if page->mapping is no longer equal to the original
174  * mapping.  This happens a) when the VM reclaimed the page while we waited on
179 truncate_cleanup_page(struct address_space *mapping, struct page *page)  in truncate_cleanup_page()  argument
183 		unmap_mapping_pages(mapping, page->index, nr, false);  in truncate_cleanup_page()
207 invalidate_complete_page(struct address_space *mapping, struct page *page)  in invalidate_complete_page()  argument
211 	if (page->mapping != mapping)  in invalidate_complete_page()
217 	ret = remove_mapping(mapping, page);  in invalidate_complete_page()
222 int truncate_inode_page(struct address_space *mapping, struct page *page)  in truncate_inode_page()  argument
226 	if (page->mapping != mapping)  in truncate_inode_page()
229 	truncate_cleanup_page(mapping, page);  in truncate_inode_page()
237 int generic_error_remove_page(struct address_space *mapping, struct page *page)  in generic_error_remove_page()  argument
239 	if (!mapping)  in generic_error_remove_page()
245 	if (!S_ISREG(mapping->host->i_mode))  in generic_error_remove_page()
247 	return truncate_inode_page(mapping, page);  in generic_error_remove_page()
252  * Safely invalidate one page from its pagecache mapping.
259 	struct address_space *mapping = page_mapping(page);  in invalidate_inode_page()  local
260 	if (!mapping)  in invalidate_inode_page()
266 	return invalidate_complete_page(mapping, page);  in invalidate_inode_page()
271  * @mapping: mapping to truncate
286  * mapping is large, it is probably the case that the final pages are the most
293 void truncate_inode_pages_range(struct address_space *mapping,  in truncate_inode_pages_range()  argument
305 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in truncate_inode_pages_range()
331 	while (index < end && pagevec_lookup_entries(&pvec, mapping, index,  in truncate_inode_pages_range()
360 			if (page->mapping != mapping) {  in truncate_inode_pages_range()
367 			truncate_cleanup_page(mapping, locked_pvec.pages[i]);  in truncate_inode_pages_range()
368 		delete_from_page_cache_batch(mapping, &locked_pvec);  in truncate_inode_pages_range()
371 		truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);  in truncate_inode_pages_range()
377 		struct page *page = find_lock_page(mapping, start - 1);  in truncate_inode_pages_range()
387 			cleancache_invalidate_page(mapping, page);  in truncate_inode_pages_range()
396 		struct page *page = find_lock_page(mapping, end);  in truncate_inode_pages_range()
400 			cleancache_invalidate_page(mapping, page);  in truncate_inode_pages_range()
418 		if (!pagevec_lookup_entries(&pvec, mapping, index,  in truncate_inode_pages_range()
451 			truncate_inode_page(mapping, page);  in truncate_inode_pages_range()
454 		truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);  in truncate_inode_pages_range()
460 	cleancache_invalidate_inode(mapping);  in truncate_inode_pages_range()
466  * @mapping: mapping to truncate
473  * mapping->nrpages can be non-zero when this function returns even after
474  * truncation of the whole mapping.
476 void truncate_inode_pages(struct address_space *mapping, loff_t lstart)  in truncate_inode_pages()  argument
478 	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);  in truncate_inode_pages()
484  * @mapping: mapping to truncate
491 void truncate_inode_pages_final(struct address_space *mapping)  in truncate_inode_pages_final()  argument
503 	mapping_set_exiting(mapping);  in truncate_inode_pages_final()
510 	nrpages = mapping->nrpages;  in truncate_inode_pages_final()
512 	nrexceptional = mapping->nrexceptional;  in truncate_inode_pages_final()
521 		xa_lock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
522 		xa_unlock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
529 	truncate_inode_pages(mapping, 0);  in truncate_inode_pages_final()
535  * @mapping: the address_space which holds the pages to invalidate
546 unsigned long invalidate_mapping_pages(struct address_space *mapping,  in invalidate_mapping_pages()  argument
557 	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,  in invalidate_mapping_pages()
569 				invalidate_exceptional_entry(mapping, index,  in invalidate_mapping_pages()
624 invalidate_complete_page2(struct address_space *mapping, struct page *page)  in invalidate_complete_page2()  argument
628 	if (page->mapping != mapping)  in invalidate_complete_page2()
634 	xa_lock_irqsave(&mapping->i_pages, flags);  in invalidate_complete_page2()
640 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in invalidate_complete_page2()
642 	if (mapping->a_ops->freepage)  in invalidate_complete_page2()
643 		mapping->a_ops->freepage(page);  in invalidate_complete_page2()
648 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in invalidate_complete_page2()
652 static int do_launder_page(struct address_space *mapping, struct page *page)  in do_launder_page()  argument
656 	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)  in do_launder_page()
658 	return mapping->a_ops->launder_page(page);  in do_launder_page()
663  * @mapping: the address_space
672 int invalidate_inode_pages2_range(struct address_space *mapping,  in invalidate_inode_pages2_range()  argument
683 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in invalidate_inode_pages2_range()
688 	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,  in invalidate_inode_pages2_range()
700 				if (!invalidate_exceptional_entry2(mapping,  in invalidate_inode_pages2_range()
708 			if (page->mapping != mapping) {  in invalidate_inode_pages2_range()
718 					unmap_mapping_pages(mapping, index,  in invalidate_inode_pages2_range()
725 					unmap_mapping_pages(mapping, index,  in invalidate_inode_pages2_range()
730 			ret2 = do_launder_page(mapping, page);  in invalidate_inode_pages2_range()
732 				if (!invalidate_complete_page2(mapping, page))  in invalidate_inode_pages2_range()
751 	if (dax_mapping(mapping)) {  in invalidate_inode_pages2_range()
752 		unmap_mapping_pages(mapping, start, end - start + 1, false);  in invalidate_inode_pages2_range()
755 	cleancache_invalidate_inode(mapping);  in invalidate_inode_pages2_range()
762  * @mapping: the address_space
769 int invalidate_inode_pages2(struct address_space *mapping)  in invalidate_inode_pages2()  argument
771 	return invalidate_inode_pages2_range(mapping, 0, -1);  in invalidate_inode_pages2()
792 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache()  local
804 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
805 	truncate_inode_pages(mapping, newsize);  in truncate_pagecache()
806 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
900 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache_range()  local
917 		unmap_mapping_range(mapping, unmap_start,  in truncate_pagecache_range()
919 	truncate_inode_pages_range(mapping, lstart, lend);  in truncate_pagecache_range()