115 #define as_pd(x) container_of((x), typeof(struct i915_page_directory), pt)
189 	free_px(vm, &pd->pt, lvl);  in __gen8_ppgtt_cleanup()
222 		struct i915_page_table *pt = pd->entry[idx];  in __gen8_ppgtt_clear()  local
224 		if (atomic_fetch_inc(&pt->used) >> gen8_pd_shift(1) &&  in __gen8_ppgtt_clear()
229 			__gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl);  in __gen8_ppgtt_clear()
235 			start = __gen8_ppgtt_clear(vm, as_pd(pt),  in __gen8_ppgtt_clear()
245 			    atomic_read(&pt->used));  in __gen8_ppgtt_clear()
246 			GEM_BUG_ON(!count || count >= atomic_read(&pt->used));  in __gen8_ppgtt_clear()
248 			vaddr = px_vaddr(pt);  in __gen8_ppgtt_clear()
253 			atomic_sub(count, &pt->used);  in __gen8_ppgtt_clear()
257 		if (release_pd_entry(pd, idx, pt, scratch))  in __gen8_ppgtt_clear()
258 			free_px(vm, pt, lvl);  in __gen8_ppgtt_clear()
297 		struct i915_page_table *pt = pd->entry[idx];  in __gen8_ppgtt_alloc()  local
299 		if (!pt) {  in __gen8_ppgtt_alloc()
305 			pt = stash->pt[!!lvl];  in __gen8_ppgtt_alloc()
306 			__i915_gem_object_pin_pages(pt->base);  in __gen8_ppgtt_alloc()
307 			i915_gem_object_make_unshrinkable(pt->base);  in __gen8_ppgtt_alloc()
309 			fill_px(pt, vm->scratch[lvl]->encode);  in __gen8_ppgtt_alloc()
313 				stash->pt[!!lvl] = pt->stash;  in __gen8_ppgtt_alloc()
314 				atomic_set(&pt->used, 0);  in __gen8_ppgtt_alloc()
315 				set_pd_entry(pd, idx, pt);  in __gen8_ppgtt_alloc()
317 				pt = pd->entry[idx];  in __gen8_ppgtt_alloc()
322 			atomic_inc(&pt->used);  in __gen8_ppgtt_alloc()
326 					   as_pd(pt), start, end, lvl);  in __gen8_ppgtt_alloc()
329 			atomic_dec(&pt->used);  in __gen8_ppgtt_alloc()
330 			GEM_BUG_ON(!atomic_read(&pt->used));  in __gen8_ppgtt_alloc()
337 			    atomic_read(&pt->used));  in __gen8_ppgtt_alloc()
339 			atomic_add(count, &pt->used);  in __gen8_ppgtt_alloc()
341 			GEM_BUG_ON(atomic_read(&pt->used) > NALLOC * I915_PDES);  in __gen8_ppgtt_alloc()
368 					    struct i915_page_table *pt,  in __gen8_ppgtt_foreach()  argument
378 		struct i915_page_table *pt = pd->entry[idx];  in __gen8_ppgtt_foreach()  local
380 		atomic_inc(&pt->used);  in __gen8_ppgtt_foreach()
384 			__gen8_ppgtt_foreach(vm, as_pd(pt), start, end, lvl,  in __gen8_ppgtt_foreach()
387 			fn(vm, pt, data);  in __gen8_ppgtt_foreach()
392 		atomic_dec(&pt->used);  in __gen8_ppgtt_foreach()
400 					  struct i915_page_table *pt,  in gen8_ppgtt_foreach()  argument
492 			struct i915_page_table *pt =  in gen8_ppgtt_insert_huge()  local
505 			vaddr = px_vaddr(pt);  in gen8_ppgtt_insert_huge()
705 		err = map_pt_dma(vm, pde->pt.base);  in gen8_preallocate_top_level_pdp()
733 	pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen8_alloc_top_pd()
734 	if (IS_ERR(pd->pt.base)) {  in gen8_alloc_top_pd()
735 		err = PTR_ERR(pd->pt.base);  in gen8_alloc_top_pd()
736 		pd->pt.base = NULL;  in gen8_alloc_top_pd()
740 	err = map_pt_dma(vm, pd->pt.base);  in gen8_alloc_top_pd()