288 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,  in ttm_pool_update_free_locked()  argument
291 	pool->npages -= freed_pages;  in ttm_pool_update_free_locked()
292 	pool->nfrees += freed_pages;  in ttm_pool_update_free_locked()
305 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,  in ttm_page_pool_free()  argument
329 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_page_pool_free()
331 	list_for_each_entry_reverse(p, &pool->list, lru) {  in ttm_page_pool_free()
339 			__list_del(p->lru.prev, &pool->list);  in ttm_page_pool_free()
341 			ttm_pool_update_free_locked(pool, freed_pages);  in ttm_page_pool_free()
346 			spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_page_pool_free()
374 		__list_del(&p->lru, &pool->list);  in ttm_page_pool_free()
376 		ttm_pool_update_free_locked(pool, freed_pages);  in ttm_page_pool_free()
380 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_page_pool_free()
404 	struct ttm_page_pool *pool;  in ttm_pool_shrink_scan()  local
416 		pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];  in ttm_pool_shrink_scan()
418 		shrink_pages = ttm_page_pool_free(pool, nr_free, true);  in ttm_pool_shrink_scan()
576 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,  in ttm_page_pool_fill_locked()  argument
588 	if (pool->fill_lock)  in ttm_page_pool_fill_locked()
591 	pool->fill_lock = true;  in ttm_page_pool_fill_locked()
596 		&& count > pool->npages) {  in ttm_page_pool_fill_locked()
604 		spin_unlock_irqrestore(&pool->lock, *irq_flags);  in ttm_page_pool_fill_locked()
607 		r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,  in ttm_page_pool_fill_locked()
609 		spin_lock_irqsave(&pool->lock, *irq_flags);  in ttm_page_pool_fill_locked()
612 			list_splice(&new_pages, &pool->list);  in ttm_page_pool_fill_locked()
613 			++pool->nrefills;  in ttm_page_pool_fill_locked()
614 			pool->npages += alloc_size;  in ttm_page_pool_fill_locked()
616 			pr_err("Failed to fill pool (%p)\n", pool);  in ttm_page_pool_fill_locked()
621 			list_splice(&new_pages, &pool->list);  in ttm_page_pool_fill_locked()
622 			pool->npages += cpages;  in ttm_page_pool_fill_locked()
626 	pool->fill_lock = false;  in ttm_page_pool_fill_locked()
634 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,  in ttm_page_pool_get_pages()  argument
644 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_page_pool_get_pages()
645 	ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);  in ttm_page_pool_get_pages()
647 	if (count >= pool->npages) {  in ttm_page_pool_get_pages()
649 		list_splice_init(&pool->list, pages);  in ttm_page_pool_get_pages()
650 		count -= pool->npages;  in ttm_page_pool_get_pages()
651 		pool->npages = 0;  in ttm_page_pool_get_pages()
656 	if (count <= pool->npages/2) {  in ttm_page_pool_get_pages()
658 		list_for_each(p, &pool->list) {  in ttm_page_pool_get_pages()
663 		i = pool->npages + 1;  in ttm_page_pool_get_pages()
664 		list_for_each_prev(p, &pool->list) {  in ttm_page_pool_get_pages()
670 	list_cut_position(pages, &pool->list, p);  in ttm_page_pool_get_pages()
671 	pool->npages -= count;  in ttm_page_pool_get_pages()
674 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_page_pool_get_pages()
683 	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);  in ttm_put_pages()  local
686 	if (pool == NULL) {  in ttm_put_pages()
699 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_put_pages()
704 			list_add_tail(&pages[i]->lru, &pool->list);  in ttm_put_pages()
706 			pool->npages++;  in ttm_put_pages()
711 	if (pool->npages > _manager->options.max_size) {  in ttm_put_pages()
712 		npages = pool->npages - _manager->options.max_size;  in ttm_put_pages()
718 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_put_pages()
720 		ttm_page_pool_free(pool, npages, false);  in ttm_put_pages()
730 	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);  in ttm_get_pages()  local
742 	if (pool == NULL) {  in ttm_get_pages()
762 	gfp_flags |= pool->gfp_flags;  in ttm_get_pages()
766 	npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);  in ttm_get_pages()
804 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,  in ttm_page_pool_init_locked()  argument
807 	spin_lock_init(&pool->lock);  in ttm_page_pool_init_locked()
808 	pool->fill_lock = false;  in ttm_page_pool_init_locked()
809 	INIT_LIST_HEAD(&pool->list);  in ttm_page_pool_init_locked()
810 	pool->npages = pool->nfrees = 0;  in ttm_page_pool_init_locked()
811 	pool->gfp_flags = flags;  in ttm_page_pool_init_locked()
812 	pool->name = name;  in ttm_page_pool_init_locked()