28 static int devmap_managed_enable_get(struct dev_pagemap *pgmap)  in devmap_managed_enable_get()  argument
30 	if (!pgmap->ops || !pgmap->ops->page_free) {  in devmap_managed_enable_get()
40 static int devmap_managed_enable_get(struct dev_pagemap *pgmap)  in devmap_managed_enable_get()  argument
56 static unsigned long pfn_first(struct dev_pagemap *pgmap)  in pfn_first()  argument
58 	return PHYS_PFN(pgmap->res.start) +  in pfn_first()
59 		vmem_altmap_offset(pgmap_altmap(pgmap));  in pfn_first()
62 static unsigned long pfn_end(struct dev_pagemap *pgmap)  in pfn_end()  argument
64 	const struct resource *res = &pgmap->res;  in pfn_end()
79 static void dev_pagemap_kill(struct dev_pagemap *pgmap)  in dev_pagemap_kill()  argument
81 	if (pgmap->ops && pgmap->ops->kill)  in dev_pagemap_kill()
82 		pgmap->ops->kill(pgmap);  in dev_pagemap_kill()
84 		percpu_ref_kill(pgmap->ref);  in dev_pagemap_kill()
87 static void dev_pagemap_cleanup(struct dev_pagemap *pgmap)  in dev_pagemap_cleanup()  argument
89 	if (pgmap->ops && pgmap->ops->cleanup) {  in dev_pagemap_cleanup()
90 		pgmap->ops->cleanup(pgmap);  in dev_pagemap_cleanup()
92 		wait_for_completion(&pgmap->done);  in dev_pagemap_cleanup()
93 		percpu_ref_exit(pgmap->ref);  in dev_pagemap_cleanup()
99 	if (pgmap->ref == &pgmap->internal_ref)  in dev_pagemap_cleanup()
100 		pgmap->ref = NULL;  in dev_pagemap_cleanup()
103 void memunmap_pages(struct dev_pagemap *pgmap)  in memunmap_pages()  argument
105 	struct resource *res = &pgmap->res;  in memunmap_pages()
110 	dev_pagemap_kill(pgmap);  in memunmap_pages()
111 	for_each_device_pfn(pfn, pgmap)  in memunmap_pages()
113 	dev_pagemap_cleanup(pgmap);  in memunmap_pages()
116 	first_page = pfn_to_page(pfn_first(pgmap));  in memunmap_pages()
122 	if (pgmap->type == MEMORY_DEVICE_PRIVATE) {  in memunmap_pages()
127 				pgmap_altmap(pgmap));  in memunmap_pages()
134 	WARN_ONCE(pgmap->altmap.alloc, "failed to free all reserved pages\n");  in memunmap_pages()
146 	struct dev_pagemap *pgmap =  in dev_pagemap_percpu_release()  local
149 	complete(&pgmap->done);  in dev_pagemap_percpu_release()
157 void *memremap_pages(struct dev_pagemap *pgmap, int nid)  in memremap_pages()  argument
159 	struct resource *res = &pgmap->res;  in memremap_pages()
165 		.altmap = pgmap_altmap(pgmap),  in memremap_pages()
171 	switch (pgmap->type) {  in memremap_pages()
177 		if (!pgmap->ops || !pgmap->ops->migrate_to_ram) {  in memremap_pages()
194 		WARN(1, "Invalid pgmap type %d\n", pgmap->type);  in memremap_pages()
198 	if (!pgmap->ref) {  in memremap_pages()
199 		if (pgmap->ops && (pgmap->ops->kill || pgmap->ops->cleanup))  in memremap_pages()
202 		init_completion(&pgmap->done);  in memremap_pages()
203 		error = percpu_ref_init(&pgmap->internal_ref,  in memremap_pages()
207 		pgmap->ref = &pgmap->internal_ref;  in memremap_pages()
209 		if (!pgmap->ops || !pgmap->ops->kill || !pgmap->ops->cleanup) {  in memremap_pages()
216 		error = devmap_managed_enable_get(pgmap);  in memremap_pages()
248 				PHYS_PFN(res->end), pgmap, GFP_KERNEL));  in memremap_pages()
273 	if (pgmap->type == MEMORY_DEVICE_PRIVATE) {  in memremap_pages()
305 				PHYS_PFN(resource_size(res)), pgmap);  in memremap_pages()
306 	percpu_ref_get_many(pgmap->ref, pfn_end(pgmap) - pfn_first(pgmap));  in memremap_pages()
316 	dev_pagemap_kill(pgmap);  in memremap_pages()
317 	dev_pagemap_cleanup(pgmap);  in memremap_pages()
343 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)  in devm_memremap_pages()  argument
348 	ret = memremap_pages(pgmap, dev_to_node(dev));  in devm_memremap_pages()
353 			pgmap);  in devm_memremap_pages()
360 void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap)  in devm_memunmap_pages()  argument
362 	devm_release_action(dev, devm_memremap_pages_release, pgmap);  in devm_memunmap_pages()
388 		struct dev_pagemap *pgmap)  in get_dev_pagemap()  argument
395 	if (pgmap) {  in get_dev_pagemap()
396 		if (phys >= pgmap->res.start && phys <= pgmap->res.end)  in get_dev_pagemap()
397 			return pgmap;  in get_dev_pagemap()
398 		put_dev_pagemap(pgmap);  in get_dev_pagemap()
403 	pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));  in get_dev_pagemap()
404 	if (pgmap && !percpu_ref_tryget_live(pgmap->ref))  in get_dev_pagemap()
405 		pgmap = NULL;  in get_dev_pagemap()
408 	return pgmap;  in get_dev_pagemap()
452 		page->pgmap->ops->page_free(page);  in __put_devmap_managed_page()