28 static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,  in clear_shadow_entry()  argument
34 	spin_lock_irq(&mapping->tree_lock);  in clear_shadow_entry()
40 	if (!__radix_tree_lookup(&mapping->page_tree, index, &node, &slot))  in clear_shadow_entry()
44 	__radix_tree_replace(&mapping->page_tree, node, slot, NULL,  in clear_shadow_entry()
45 			     workingset_update_node, mapping);  in clear_shadow_entry()
46 	mapping->nrexceptional--;  in clear_shadow_entry()
48 	spin_unlock_irq(&mapping->tree_lock);  in clear_shadow_entry()
54 static void truncate_exceptional_entry(struct address_space *mapping,  in truncate_exceptional_entry()  argument
58 	if (shmem_mapping(mapping))  in truncate_exceptional_entry()
61 	if (dax_mapping(mapping)) {  in truncate_exceptional_entry()
62 		dax_delete_mapping_entry(mapping, index);  in truncate_exceptional_entry()
65 	clear_shadow_entry(mapping, index, entry);  in truncate_exceptional_entry()
72 static int invalidate_exceptional_entry(struct address_space *mapping,  in invalidate_exceptional_entry()  argument
76 	if (shmem_mapping(mapping) || dax_mapping(mapping))  in invalidate_exceptional_entry()
78 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry()
86 static int invalidate_exceptional_entry2(struct address_space *mapping,  in invalidate_exceptional_entry2()  argument
90 	if (shmem_mapping(mapping))  in invalidate_exceptional_entry2()
92 	if (dax_mapping(mapping))  in invalidate_exceptional_entry2()
93 		return dax_invalidate_mapping_entry_sync(mapping, index);  in invalidate_exceptional_entry2()
94 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry2()
118 	invalidatepage = page->mapping->a_ops->invalidatepage;  in do_invalidatepage()
138 truncate_complete_page(struct address_space *mapping, struct page *page)  in truncate_complete_page()  argument
140 	if (page->mapping != mapping)  in truncate_complete_page()
166 invalidate_complete_page(struct address_space *mapping, struct page *page)  in invalidate_complete_page()  argument
170 	if (page->mapping != mapping)  in invalidate_complete_page()
176 	ret = remove_mapping(mapping, page);  in invalidate_complete_page()
181 int truncate_inode_page(struct address_space *mapping, struct page *page)  in truncate_inode_page()  argument
188 		unmap_mapping_range(mapping,  in truncate_inode_page()
192 	return truncate_complete_page(mapping, page);  in truncate_inode_page()
198 int generic_error_remove_page(struct address_space *mapping, struct page *page)  in generic_error_remove_page()  argument
200 	if (!mapping)  in generic_error_remove_page()
206 	if (!S_ISREG(mapping->host->i_mode))  in generic_error_remove_page()
208 	return truncate_inode_page(mapping, page);  in generic_error_remove_page()
220 	struct address_space *mapping = page_mapping(page);  in invalidate_inode_page()  local
221 	if (!mapping)  in invalidate_inode_page()
227 	return invalidate_complete_page(mapping, page);  in invalidate_inode_page()
254 void truncate_inode_pages_range(struct address_space *mapping,  in truncate_inode_pages_range()  argument
266 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in truncate_inode_pages_range()
292 	while (index < end && pagevec_lookup_entries(&pvec, mapping, index,  in truncate_inode_pages_range()
304 				truncate_exceptional_entry(mapping, index,  in truncate_inode_pages_range()
316 			truncate_inode_page(mapping, page);  in truncate_inode_pages_range()
326 		struct page *page = find_lock_page(mapping, start - 1);  in truncate_inode_pages_range()
336 			cleancache_invalidate_page(mapping, page);  in truncate_inode_pages_range()
345 		struct page *page = find_lock_page(mapping, end);  in truncate_inode_pages_range()
349 			cleancache_invalidate_page(mapping, page);  in truncate_inode_pages_range()
367 		if (!pagevec_lookup_entries(&pvec, mapping, index,  in truncate_inode_pages_range()
394 				truncate_exceptional_entry(mapping, index,  in truncate_inode_pages_range()
402 			truncate_inode_page(mapping, page);  in truncate_inode_pages_range()
411 	cleancache_invalidate_inode(mapping);  in truncate_inode_pages_range()
427 void truncate_inode_pages(struct address_space *mapping, loff_t lstart)  in truncate_inode_pages()  argument
429 	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);  in truncate_inode_pages()
442 void truncate_inode_pages_final(struct address_space *mapping)  in truncate_inode_pages_final()  argument
454 	mapping_set_exiting(mapping);  in truncate_inode_pages_final()
461 	nrpages = mapping->nrpages;  in truncate_inode_pages_final()
463 	nrexceptional = mapping->nrexceptional;  in truncate_inode_pages_final()
472 		spin_lock_irq(&mapping->tree_lock);  in truncate_inode_pages_final()
473 		spin_unlock_irq(&mapping->tree_lock);  in truncate_inode_pages_final()
480 	truncate_inode_pages(mapping, 0);  in truncate_inode_pages_final()
497 unsigned long invalidate_mapping_pages(struct address_space *mapping,  in invalidate_mapping_pages()  argument
508 	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,  in invalidate_mapping_pages()
520 				invalidate_exceptional_entry(mapping, index,  in invalidate_mapping_pages()
575 invalidate_complete_page2(struct address_space *mapping, struct page *page)  in invalidate_complete_page2()  argument
579 	if (page->mapping != mapping)  in invalidate_complete_page2()
585 	spin_lock_irqsave(&mapping->tree_lock, flags);  in invalidate_complete_page2()
591 	spin_unlock_irqrestore(&mapping->tree_lock, flags);  in invalidate_complete_page2()
593 	if (mapping->a_ops->freepage)  in invalidate_complete_page2()
594 		mapping->a_ops->freepage(page);  in invalidate_complete_page2()
599 	spin_unlock_irqrestore(&mapping->tree_lock, flags);  in invalidate_complete_page2()
603 static int do_launder_page(struct address_space *mapping, struct page *page)  in do_launder_page()  argument
607 	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)  in do_launder_page()
609 	return mapping->a_ops->launder_page(page);  in do_launder_page()
623 int invalidate_inode_pages2_range(struct address_space *mapping,  in invalidate_inode_pages2_range()  argument
634 	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)  in invalidate_inode_pages2_range()
639 	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,  in invalidate_inode_pages2_range()
651 				if (!invalidate_exceptional_entry2(mapping,  in invalidate_inode_pages2_range()
659 			if (page->mapping != mapping) {  in invalidate_inode_pages2_range()
669 					unmap_mapping_range(mapping,  in invalidate_inode_pages2_range()
679 					unmap_mapping_range(mapping,  in invalidate_inode_pages2_range()
685 			ret2 = do_launder_page(mapping, page);  in invalidate_inode_pages2_range()
687 				if (!invalidate_complete_page2(mapping, page))  in invalidate_inode_pages2_range()
706 	if (dax_mapping(mapping)) {  in invalidate_inode_pages2_range()
707 		unmap_mapping_range(mapping, (loff_t)start << PAGE_SHIFT,  in invalidate_inode_pages2_range()
711 	cleancache_invalidate_inode(mapping);  in invalidate_inode_pages2_range()
725 int invalidate_inode_pages2(struct address_space *mapping)  in invalidate_inode_pages2()  argument
727 	return invalidate_inode_pages2_range(mapping, 0, -1);  in invalidate_inode_pages2()
748 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache()  local
760 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
761 	truncate_inode_pages(mapping, newsize);  in truncate_pagecache()
762 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
856 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache_range()  local
873 		unmap_mapping_range(mapping, unmap_start,  in truncate_pagecache_range()
875 	truncate_inode_pages_range(mapping, lstart, lend);  in truncate_pagecache_range()