79 static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,  in ttm_pool_alloc_page()  argument
95 	if (!pool->use_dma_alloc) {  in ttm_pool_alloc_page()
96 		p = alloc_pages_node(pool->nid, gfp_flags, order);  in ttm_pool_alloc_page()
109 	vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE,  in ttm_pool_alloc_page()
132 static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching,  in ttm_pool_free_page()  argument
147 	if (!pool || !pool->use_dma_alloc) {  in ttm_pool_free_page()
157 	dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr,  in ttm_pool_free_page()
185 static int ttm_pool_map(struct ttm_pool *pool, unsigned int order,  in ttm_pool_map()  argument
191 	if (pool->use_dma_alloc) {  in ttm_pool_map()
198 		addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL);  in ttm_pool_map()
199 		if (dma_mapping_error(pool->dev, addr))  in ttm_pool_map()
212 static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr,  in ttm_pool_unmap()  argument
216 	if (pool->use_dma_alloc)  in ttm_pool_unmap()
219 	dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT,  in ttm_pool_unmap()
258 static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool,  in ttm_pool_type_init()  argument
261 	pt->pool = pool;  in ttm_pool_type_init()
282 		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);  in ttm_pool_type_fini()
286 static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,  in ttm_pool_select_type()  argument
290 	if (pool->use_dma_alloc)  in ttm_pool_select_type()
291 		return &pool->caching[caching].orders[order];  in ttm_pool_select_type()
296 		if (pool->nid != NUMA_NO_NODE)  in ttm_pool_select_type()
297 			return &pool->caching[caching].orders[order];  in ttm_pool_select_type()
299 		if (pool->use_dma32)  in ttm_pool_select_type()
304 		if (pool->nid != NUMA_NO_NODE)  in ttm_pool_select_type()
305 			return &pool->caching[caching].orders[order];  in ttm_pool_select_type()
307 		if (pool->use_dma32)  in ttm_pool_select_type()
333 		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);  in ttm_pool_shrink()
343 static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p)  in ttm_pool_page_order()  argument
345 	if (pool->use_dma_alloc) {  in ttm_pool_page_order()
355 static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,  in ttm_pool_page_allocated()  argument
364 		r = ttm_pool_map(pool, order, p, dma_addr);  in ttm_pool_page_allocated()
389 static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt,  in ttm_pool_free_range()  argument
400 		order = ttm_pool_page_order(pool, *pages);  in ttm_pool_free_range()
403 			ttm_pool_unmap(pool, tt->dma_address[i], nr);  in ttm_pool_free_range()
405 		pt = ttm_pool_select_type(pool, caching, order);  in ttm_pool_free_range()
409 			ttm_pool_free_page(pool, caching, order, *pages);  in ttm_pool_free_range()
425 int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,  in ttm_pool_alloc()  argument
440 	WARN_ON(dma_addr && !pool->dev);  in ttm_pool_alloc()
448 	if (pool->use_dma32)  in ttm_pool_alloc()
459 		pt = ttm_pool_select_type(pool, tt->caching, order);  in ttm_pool_alloc()
469 				r = ttm_pool_page_allocated(pool, order, p,  in ttm_pool_alloc()
486 		       (p = ttm_pool_alloc_page(pool, gfp_flags, order))) {  in ttm_pool_alloc()
495 			r = ttm_pool_page_allocated(pool, order, p, &dma_addr,  in ttm_pool_alloc()
520 	ttm_pool_free_page(pool, page_caching, order, p);  in ttm_pool_alloc()
525 	ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide);  in ttm_pool_alloc()
526 	ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, num_pages);  in ttm_pool_alloc()
540 void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)  in ttm_pool_free()  argument
542 	ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages);  in ttm_pool_free()
560 void ttm_pool_init(struct ttm_pool *pool, struct device *dev,  in ttm_pool_init()  argument
567 	pool->dev = dev;  in ttm_pool_init()
568 	pool->nid = nid;  in ttm_pool_init()
569 	pool->use_dma_alloc = use_dma_alloc;  in ttm_pool_init()
570 	pool->use_dma32 = use_dma32;  in ttm_pool_init()
577 			pt = ttm_pool_select_type(pool, i, j);  in ttm_pool_init()
578 			if (pt != &pool->caching[i].orders[j])  in ttm_pool_init()
581 			ttm_pool_type_init(pt, pool, i, j);  in ttm_pool_init()
595 void ttm_pool_fini(struct ttm_pool *pool)  in ttm_pool_fini()  argument
603 			pt = ttm_pool_select_type(pool, i, j);  in ttm_pool_fini()
604 			if (pt != &pool->caching[i].orders[j])  in ttm_pool_fini()
715 int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m)  in ttm_pool_debugfs()  argument
719 	if (!pool->use_dma_alloc) {  in ttm_pool_debugfs()
740 		ttm_pool_debugfs_orders(pool->caching[i].orders, m);  in ttm_pool_debugfs()