3  * Memory Migration functionality - linux/mm/migrate.c
16 #include <linux/migrate.h>
56 #include <trace/events/migrate.h>
70 	 * drained them. Those pages will fail to migrate like other  in migrate_prep()
639 	/* Migrate the page's xpm state */  in migrate_page_states()
721  * Common logic to directly migrate a single LRU page suitable for
1014 			 * cannot try to migrate this page.  in move_to_new_page()
1183  * Obtain the lock on page, remove all ptes and migrate the page
1229 		 * Compaction can migrate also non-LRU pages which are  in unmap_and_move()
1284  * This means that when we try to migrate hugepage whose subpages are
1413  * migrate_pages - migrate the pages specified in a list, to the free pages
1717 		 * pages to migrate.  Since we are going to  in move_pages_and_store_status()
1731  * Migrate an array of page address onto an array of nodes and fill
2087  * Attempt to migrate a misplaced page to the specified destination
2100 	 * Don't migrate file pages that are mapped in multiple processes  in migrate_misplaced_page()
2108 	 * Also do not migrate dirty pages as not all filesystems can move  in migrate_misplaced_page()
2283 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_hole()  local
2289 			migrate->src[migrate->npages] = 0;  in migrate_vma_collect_hole()
2290 			migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
2291 			migrate->npages++;  in migrate_vma_collect_hole()
2297 		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;  in migrate_vma_collect_hole()
2298 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
2299 		migrate->npages++;  in migrate_vma_collect_hole()
2300 		migrate->cpages++;  in migrate_vma_collect_hole()
2310 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_skip()  local
2314 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_skip()
2315 		migrate->src[migrate->npages++] = 0;  in migrate_vma_collect_skip()
2326 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_pmd()  local
2390 				migrate->cpages++;  in migrate_vma_collect_pmd()
2406 			if (!(migrate->flags &  in migrate_vma_collect_pmd()
2408 			    page->pgmap->owner != migrate->pgmap_owner)  in migrate_vma_collect_pmd()
2416 			if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))  in migrate_vma_collect_pmd()
2421 				migrate->cpages++;  in migrate_vma_collect_pmd()
2424 			page = vm_normal_page(migrate->vma, addr, pte);  in migrate_vma_collect_pmd()
2445 		migrate->cpages++;  in migrate_vma_collect_pmd()
2488 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_pmd()
2489 		migrate->src[migrate->npages++] = mpfn;  in migrate_vma_collect_pmd()
2509  * @migrate: migrate struct containing all migration information
2515 static void migrate_vma_collect(struct migrate_vma *migrate)  in migrate_vma_collect()  argument
2524 	mmu_notifier_range_init_migrate(&range, 0, migrate->vma,  in migrate_vma_collect()
2525 		migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
2526 		migrate->pgmap_owner);  in migrate_vma_collect()
2529 	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
2530 			&migrate_vma_walk_ops, migrate);  in migrate_vma_collect()
2533 	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);  in migrate_vma_collect()
2569 		 * avoid 2 racing thread trying to migrate back to CPU to enter  in migrate_vma_check_page()
2591  * @migrate: migrate struct containing all migration information
2598 static void migrate_vma_prepare(struct migrate_vma *migrate)  in migrate_vma_prepare()  argument
2600 	const unsigned long npages = migrate->npages;  in migrate_vma_prepare()
2601 	const unsigned long start = migrate->start;  in migrate_vma_prepare()
2607 	for (i = 0; (i < npages) && migrate->cpages; i++) {  in migrate_vma_prepare()
2608 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2614 		if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {  in migrate_vma_prepare()
2624 				migrate->src[i] = 0;  in migrate_vma_prepare()
2625 				migrate->cpages--;  in migrate_vma_prepare()
2630 			migrate->src[i] |= MIGRATE_PFN_LOCKED;  in migrate_vma_prepare()
2643 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2644 					migrate->cpages--;  in migrate_vma_prepare()
2647 					migrate->src[i] = 0;  in migrate_vma_prepare()
2649 					migrate->cpages--;  in migrate_vma_prepare()
2659 		if (!migrate_vma_check_page(page, migrate->fault_page)) {  in migrate_vma_prepare()
2661 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_prepare()
2662 				migrate->cpages--;  in migrate_vma_prepare()
2670 				migrate->src[i] = 0;  in migrate_vma_prepare()
2672 				migrate->cpages--;  in migrate_vma_prepare()
2683 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_prepare()
2685 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_prepare()
2688 		remove_migration_pte(page, migrate->vma, addr, page);  in migrate_vma_prepare()
2690 		migrate->src[i] = 0;  in migrate_vma_prepare()
2699  * @migrate: migrate struct containing all migration information
2703  * cannot migrate them.
2708 static void migrate_vma_unmap(struct migrate_vma *migrate)  in migrate_vma_unmap()  argument
2711 	const unsigned long npages = migrate->npages;  in migrate_vma_unmap()
2712 	const unsigned long start = migrate->start;  in migrate_vma_unmap()
2716 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2718 		if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2727 		if (migrate_vma_check_page(page, migrate->fault_page))  in migrate_vma_unmap()
2731 		migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_unmap()
2732 		migrate->cpages--;  in migrate_vma_unmap()
2737 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_unmap()
2739 		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_unmap()
2744 		migrate->src[i] = 0;  in migrate_vma_unmap()
2756  * migrate_vma_setup() - prepare to migrate a range of memory
2762  * Prepare to migrate a range of memory virtual address range by collecting all
2777  * Note that the caller does not have to migrate all the pages that are marked
2779  * device memory to system memory.  If the caller cannot migrate a device page
2796  * then migrate_vma_pages() to migrate struct page information from the source
2797  * struct page to the destination struct page.  If it fails to migrate the
2869 static void migrate_vma_insert_page(struct migrate_vma *migrate,  in migrate_vma_insert_page()  argument
2875 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_insert_page()
3002  * migrate_vma_pages() - migrate meta-data from src page to dst page
3003  * @migrate: migrate struct containing all migration information
3009 void migrate_vma_pages(struct migrate_vma *migrate)  in migrate_vma_pages()  argument
3011 	const unsigned long npages = migrate->npages;  in migrate_vma_pages()
3012 	const unsigned long start = migrate->start;  in migrate_vma_pages()
3018 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_pages()
3019 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_pages()
3024 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
3029 			if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in migrate_vma_pages()
3037 							migrate->vma->vm_mm,  in migrate_vma_pages()
3038 							addr, migrate->end);  in migrate_vma_pages()
3041 			migrate_vma_insert_page(migrate, addr, newpage,  in migrate_vma_pages()
3042 						&migrate->src[i],  in migrate_vma_pages()
3043 						&migrate->dst[i]);  in migrate_vma_pages()
3056 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
3064 				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
3069 		if (migrate->fault_page == page)  in migrate_vma_pages()
3076 			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;  in migrate_vma_pages()
3091  * @migrate: migrate struct containing all migration information
3100 void migrate_vma_finalize(struct migrate_vma *migrate)  in migrate_vma_finalize()  argument
3102 	const unsigned long npages = migrate->npages;  in migrate_vma_finalize()
3106 		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);  in migrate_vma_finalize()
3107 		struct page *page = migrate_pfn_to_page(migrate->src[i]);  in migrate_vma_finalize()
3117 		if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {  in migrate_vma_finalize()