34 static inline void __clear_shadow_entry(struct address_space *mapping,  in __clear_shadow_entry()  argument
37 	XA_STATE(xas, &mapping->i_pages, index);  in __clear_shadow_entry()
43 	mapping->nrexceptional--;  in __clear_shadow_entry()
46 static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,  in clear_shadow_entry()  argument
49 	xa_lock_irq(&mapping->i_pages);  in clear_shadow_entry()
50 	__clear_shadow_entry(mapping, index, entry);  in clear_shadow_entry()
51 	xa_unlock_irq(&mapping->i_pages);  in clear_shadow_entry()
59 static void truncate_exceptional_pvec_entries(struct address_space *mapping,  in truncate_exceptional_pvec_entries()  argument
67 	if (shmem_mapping(mapping))  in truncate_exceptional_pvec_entries()
77 	dax = dax_mapping(mapping);  in truncate_exceptional_pvec_entries()
80 		xa_lock_irq(&mapping->i_pages);  in truncate_exceptional_pvec_entries()
95 			dax_delete_mapping_entry(mapping, index);  in truncate_exceptional_pvec_entries()
99 		__clear_shadow_entry(mapping, index, page);  in truncate_exceptional_pvec_entries()
103 		xa_unlock_irq(&mapping->i_pages);  in truncate_exceptional_pvec_entries()
111 static int invalidate_exceptional_entry(struct address_space *mapping,  in invalidate_exceptional_entry()  argument
115 	if (shmem_mapping(mapping) || dax_mapping(mapping))  in invalidate_exceptional_entry()
117 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry()
125 static int invalidate_exceptional_entry2(struct address_space *mapping,  in invalidate_exceptional_entry2()  argument
129 	if (shmem_mapping(mapping))  in invalidate_exceptional_entry2()
131 	if (dax_mapping(mapping))  in invalidate_exceptional_entry2()
132 		return dax_invalidate_mapping_entry_sync(mapping, index);  in invalidate_exceptional_entry2()
133 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry2()
157 	invalidatepage = page->mapping->a_ops->invalidatepage;  in do_invalidatepage()
202 invalidate_complete_page(struct address_space *mapping, struct page *page)  in invalidate_complete_page()  argument
206 	if (page->mapping != mapping)  in invalidate_complete_page()
212 	ret = remove_mapping(mapping, page);  in invalidate_complete_page()
217 int truncate_inode_page(struct address_space *mapping, struct page *page)  in truncate_inode_page()  argument
221 	if (page->mapping != mapping)  in truncate_inode_page()
232 int generic_error_remove_page(struct address_space *mapping, struct page *page)  in generic_error_remove_page()  argument
234 	if (!mapping)  in generic_error_remove_page()
240 	if (!S_ISREG(mapping->host->i_mode))  in generic_error_remove_page()
242 	return truncate_inode_page(mapping, page);  in generic_error_remove_page()
254 	struct address_space *mapping = page_mapping(page);  in invalidate_inode_page()  local
255 	if (!mapping)  in invalidate_inode_page()
261 	return invalidate_complete_page(mapping, page);  in invalidate_inode_page()
288 void truncate_inode_pages_range(struct address_space *mapping,  in truncate_inode_pages_range()  argument
300 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in truncate_inode_pages_range()
326 	while (index < end && pagevec_lookup_entries(&pvec, mapping, index,  in truncate_inode_pages_range()
355 			if (page->mapping != mapping) {  in truncate_inode_pages_range()
363 		delete_from_page_cache_batch(mapping, &locked_pvec);  in truncate_inode_pages_range()
366 		truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);  in truncate_inode_pages_range()
372 		struct page *page = find_lock_page(mapping, start - 1);  in truncate_inode_pages_range()
382 			cleancache_invalidate_page(mapping, page);  in truncate_inode_pages_range()
391 		struct page *page = find_lock_page(mapping, end);  in truncate_inode_pages_range()
395 			cleancache_invalidate_page(mapping, page);  in truncate_inode_pages_range()
413 		if (!pagevec_lookup_entries(&pvec, mapping, index,  in truncate_inode_pages_range()
446 			truncate_inode_page(mapping, page);  in truncate_inode_pages_range()
449 		truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);  in truncate_inode_pages_range()
455 	cleancache_invalidate_inode(mapping);  in truncate_inode_pages_range()
471 void truncate_inode_pages(struct address_space *mapping, loff_t lstart)  in truncate_inode_pages()  argument
473 	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);  in truncate_inode_pages()
486 void truncate_inode_pages_final(struct address_space *mapping)  in truncate_inode_pages_final()  argument
498 	mapping_set_exiting(mapping);  in truncate_inode_pages_final()
505 	nrpages = mapping->nrpages;  in truncate_inode_pages_final()
507 	nrexceptional = mapping->nrexceptional;  in truncate_inode_pages_final()
516 		xa_lock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
517 		xa_unlock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
524 	truncate_inode_pages(mapping, 0);  in truncate_inode_pages_final()
528 static unsigned long __invalidate_mapping_pages(struct address_space *mapping,  in __invalidate_mapping_pages()  argument
539 	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,  in __invalidate_mapping_pages()
551 				invalidate_exceptional_entry(mapping, index,  in __invalidate_mapping_pages()
629 unsigned long invalidate_mapping_pages(struct address_space *mapping,  in invalidate_mapping_pages()  argument
632 	return __invalidate_mapping_pages(mapping, start, end, NULL);  in invalidate_mapping_pages()
641 void invalidate_mapping_pagevec(struct address_space *mapping,  in invalidate_mapping_pagevec()  argument
644 	__invalidate_mapping_pages(mapping, start, end, nr_pagevec);  in invalidate_mapping_pagevec()
655 invalidate_complete_page2(struct address_space *mapping, struct page *page)  in invalidate_complete_page2()  argument
659 	if (page->mapping != mapping)  in invalidate_complete_page2()
665 	xa_lock_irqsave(&mapping->i_pages, flags);  in invalidate_complete_page2()
671 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in invalidate_complete_page2()
673 	if (mapping->a_ops->freepage)  in invalidate_complete_page2()
674 		mapping->a_ops->freepage(page);  in invalidate_complete_page2()
679 	xa_unlock_irqrestore(&mapping->i_pages, flags);  in invalidate_complete_page2()
683 static int do_launder_page(struct address_space *mapping, struct page *page)  in do_launder_page()  argument
687 	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)  in do_launder_page()
689 	return mapping->a_ops->launder_page(page);  in do_launder_page()
703 int invalidate_inode_pages2_range(struct address_space *mapping,  in invalidate_inode_pages2_range()  argument
714 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in invalidate_inode_pages2_range()
719 	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,  in invalidate_inode_pages2_range()
731 				if (!invalidate_exceptional_entry2(mapping,  in invalidate_inode_pages2_range()
742 				unmap_mapping_pages(mapping, index,  in invalidate_inode_pages2_range()
749 			if (page->mapping != mapping) {  in invalidate_inode_pages2_range()
759 			ret2 = do_launder_page(mapping, page);  in invalidate_inode_pages2_range()
761 				if (!invalidate_complete_page2(mapping, page))  in invalidate_inode_pages2_range()
780 	if (dax_mapping(mapping)) {  in invalidate_inode_pages2_range()
781 		unmap_mapping_pages(mapping, start, end - start + 1, false);  in invalidate_inode_pages2_range()
784 	cleancache_invalidate_inode(mapping);  in invalidate_inode_pages2_range()
798 int invalidate_inode_pages2(struct address_space *mapping)  in invalidate_inode_pages2()  argument
800 	return invalidate_inode_pages2_range(mapping, 0, -1);  in invalidate_inode_pages2()
821 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache()  local
833 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
834 	truncate_inode_pages(mapping, newsize);  in truncate_pagecache()
835 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
929 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache_range()  local
946 		unmap_mapping_range(mapping, unmap_start,  in truncate_pagecache_range()
948 	truncate_inode_pages_range(mapping, lstart, lend);  in truncate_pagecache_range()