170 static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo,  in i915_ttm_tt_create()  argument
174 		ttm_manager_type(bo->bdev, bo->resource->mem_type);  in i915_ttm_tt_create()
175 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_tt_create()
187 	ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags,  in i915_ttm_tt_create()
222 static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo,  in i915_ttm_eviction_valuable()  argument
225 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_eviction_valuable()
231 static void i915_ttm_evict_flags(struct ttm_buffer_object *bo,  in i915_ttm_evict_flags()  argument
237 static int i915_ttm_move_notify(struct ttm_buffer_object *bo)  in i915_ttm_move_notify()  argument
239 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_move_notify()
274 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);  in i915_ttm_adjust_domains_after_move()  local
276 	if (cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) {  in i915_ttm_adjust_domains_after_move()
287 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);  in i915_ttm_adjust_gem_after_move()  local
297 	if (intel_region_to_ttm_type(obj->mm.region) != bo->resource->mem_type) {  in i915_ttm_adjust_gem_after_move()
301 			if (intel_region_to_ttm_type(mr) == bo->resource->mem_type &&  in i915_ttm_adjust_gem_after_move()
312 	obj->mem_flags |= cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM :  in i915_ttm_adjust_gem_after_move()
315 	cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource,  in i915_ttm_adjust_gem_after_move()
316 					   bo->ttm);  in i915_ttm_adjust_gem_after_move()
322 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);  in i915_ttm_purge()  local
334 	ret = ttm_bo_validate(bo, &place, &ctx);  in i915_ttm_purge()
344 static void i915_ttm_swap_notify(struct ttm_buffer_object *bo)  in i915_ttm_swap_notify()  argument
346 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_swap_notify()
347 	int ret = i915_ttm_move_notify(bo);  in i915_ttm_swap_notify()
355 static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo)  in i915_ttm_delete_mem_notify()  argument
357 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_delete_mem_notify()
416 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);  in i915_ttm_resource_get_st()  local
419 		return i915_ttm_tt_get_st(bo->ttm);  in i915_ttm_resource_get_st()
429 static int i915_ttm_accel_move(struct ttm_buffer_object *bo,  in i915_ttm_accel_move()  argument
433 	struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915),  in i915_ttm_accel_move()
436 		ttm_manager_type(bo->bdev, bo->resource->mem_type);  in i915_ttm_accel_move()
437 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_accel_move()
440 	struct ttm_tt *ttm = bo->ttm;  in i915_ttm_accel_move()
449 		if (bo->type == ttm_bo_type_kernel)  in i915_ttm_accel_move()
470 		src_level = i915_ttm_cache_level(i915, bo->resource, ttm);  in i915_ttm_accel_move()
474 						 gpu_binds_iomem(bo->resource),  in i915_ttm_accel_move()
488 static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,  in i915_ttm_move()  argument
493 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_move()
495 		ttm_manager_type(bo->bdev, dst_mem->mem_type);  in i915_ttm_move()
505 	dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type);  in i915_ttm_move()
506 	src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type);  in i915_ttm_move()
510 	ret = ttm_bo_wait_ctx(bo, ctx);  in i915_ttm_move()
514 	ret = i915_ttm_move_notify(bo);  in i915_ttm_move()
520 		ttm_resource_free(bo, &dst_mem);  in i915_ttm_move()
525 	if (bo->ttm && (dst_man->use_tt ||  in i915_ttm_move()
526 			(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED))) {  in i915_ttm_move()
527 		ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);  in i915_ttm_move()
536 	ret = i915_ttm_accel_move(bo, dst_mem, dst_st);  in i915_ttm_move()
540 			ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm) :  in i915_ttm_move()
544 		src_iter = !cpu_maps_iomem(bo->resource) ?  in i915_ttm_move()
545 			ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) :  in i915_ttm_move()
550 		ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter);  in i915_ttm_move()
553 	ttm_bo_move_sync_cleanup(bo, dst_mem);  in i915_ttm_move()
578 static unsigned long i915_ttm_io_mem_pfn(struct ttm_buffer_object *bo,  in i915_ttm_io_mem_pfn()  argument
581 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_io_mem_pfn()
586 	GEM_WARN_ON(bo->ttm);  in i915_ttm_io_mem_pfn()
619 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);  in __i915_ttm_get_pages()  local
630 	ret = ttm_bo_validate(bo, placement, &ctx);  in __i915_ttm_get_pages()
646 		ret = ttm_bo_validate(bo, placement, &ctx);  in __i915_ttm_get_pages()
652 	if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) {  in __i915_ttm_get_pages()
653 		ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx);  in __i915_ttm_get_pages()
663 		st = bo->ttm ? i915_ttm_tt_get_st(bo->ttm) : obj->ttm.cached_io_st;  in __i915_ttm_get_pages()
746 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);  in i915_ttm_adjust_lru()  local
752 	if (!kref_read(&bo->kref))  in i915_ttm_adjust_lru()
758 	spin_lock(&bo->bdev->lru_lock);  in i915_ttm_adjust_lru()
760 		bo->priority = I915_TTM_PRIO_PURGE;  in i915_ttm_adjust_lru()
762 		bo->priority = I915_TTM_PRIO_NO_PAGES;  in i915_ttm_adjust_lru()
764 		bo->priority = I915_TTM_PRIO_HAS_PAGES;  in i915_ttm_adjust_lru()
767 	ttm_bo_move_to_lru_tail(bo, bo->resource, NULL);  in i915_ttm_adjust_lru()
768 	spin_unlock(&bo->bdev->lru_lock);  in i915_ttm_adjust_lru()
869 void i915_ttm_bo_destroy(struct ttm_buffer_object *bo)  in i915_ttm_bo_destroy()  argument
871 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);  in i915_ttm_bo_destroy()