436 			       struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_start()  argument
438 	cursor->pfn = start;  in amdgpu_vm_pt_start()
439 	cursor->parent = NULL;  in amdgpu_vm_pt_start()
440 	cursor->entry = &vm->root;  in amdgpu_vm_pt_start()
441 	cursor->level = adev->vm_manager.root_level;  in amdgpu_vm_pt_start()
455 				    struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_descendant()  argument
459 	if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry ||  in amdgpu_vm_pt_descendant()
460 	    !cursor->entry->bo)  in amdgpu_vm_pt_descendant()
463 	mask = amdgpu_vm_entries_mask(adev, cursor->level);  in amdgpu_vm_pt_descendant()
464 	shift = amdgpu_vm_level_shift(adev, cursor->level);  in amdgpu_vm_pt_descendant()
466 	++cursor->level;  in amdgpu_vm_pt_descendant()
467 	idx = (cursor->pfn >> shift) & mask;  in amdgpu_vm_pt_descendant()
468 	cursor->parent = cursor->entry;  in amdgpu_vm_pt_descendant()
469 	cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx];  in amdgpu_vm_pt_descendant()
484 				 struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_sibling()  argument
489 	if (!cursor->parent)  in amdgpu_vm_pt_sibling()
493 	shift = amdgpu_vm_level_shift(adev, cursor->level - 1);  in amdgpu_vm_pt_sibling()
494 	num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);  in amdgpu_vm_pt_sibling()
496 	if (cursor->entry == &to_amdgpu_bo_vm(cursor->parent->bo)->entries[num_entries - 1])  in amdgpu_vm_pt_sibling()
499 	cursor->pfn += 1ULL << shift;  in amdgpu_vm_pt_sibling()
500 	cursor->pfn &= ~((1ULL << shift) - 1);  in amdgpu_vm_pt_sibling()
501 	++cursor->entry;  in amdgpu_vm_pt_sibling()
514 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_ancestor()  argument
516 	if (!cursor->parent)  in amdgpu_vm_pt_ancestor()
519 	--cursor->level;  in amdgpu_vm_pt_ancestor()
520 	cursor->entry = cursor->parent;  in amdgpu_vm_pt_ancestor()
521 	cursor->parent = amdgpu_vm_pt_parent(cursor->parent);  in amdgpu_vm_pt_ancestor()
534 			      struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_next()  argument
537 	if (amdgpu_vm_pt_descendant(adev, cursor))  in amdgpu_vm_pt_next()
541 	while (!amdgpu_vm_pt_sibling(adev, cursor)) {  in amdgpu_vm_pt_next()
543 		if (!amdgpu_vm_pt_ancestor(cursor)) {  in amdgpu_vm_pt_next()
544 			cursor->pfn = ~0ll;  in amdgpu_vm_pt_next()
563 				   struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_first_dfs()  argument
566 		*cursor = *start;  in amdgpu_vm_pt_first_dfs()
568 		amdgpu_vm_pt_start(adev, vm, 0, cursor);  in amdgpu_vm_pt_first_dfs()
569 	while (amdgpu_vm_pt_descendant(adev, cursor));  in amdgpu_vm_pt_first_dfs()
596 				  struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_next_dfs()  argument
598 	if (!cursor->entry)  in amdgpu_vm_pt_next_dfs()
601 	if (!cursor->parent)  in amdgpu_vm_pt_next_dfs()
602 		cursor->entry = NULL;  in amdgpu_vm_pt_next_dfs()
603 	else if (amdgpu_vm_pt_sibling(adev, cursor))  in amdgpu_vm_pt_next_dfs()
604 		while (amdgpu_vm_pt_descendant(adev, cursor));  in amdgpu_vm_pt_next_dfs()
606 		amdgpu_vm_pt_ancestor(cursor);  in amdgpu_vm_pt_next_dfs()
612 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)		\  argument
613 	for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),		\
614 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
616 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
1007 			       struct amdgpu_vm_pt_cursor *cursor,  in amdgpu_vm_alloc_pts()  argument
1010 	struct amdgpu_vm_bo_base *entry = cursor->entry;  in amdgpu_vm_alloc_pts()
1018 	r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);  in amdgpu_vm_alloc_pts()
1026 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);  in amdgpu_vm_alloc_pts()
1071 	struct amdgpu_vm_pt_cursor cursor;  in amdgpu_vm_free_pts()  local
1076 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)  in amdgpu_vm_free_pts()
1368 	struct amdgpu_vm_pt_cursor cursor;  in amdgpu_vm_invalidate_pds()  local
1371 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)  in amdgpu_vm_invalidate_pds()
1537 	struct amdgpu_vm_pt_cursor cursor;  in amdgpu_vm_update_ptes()  local
1546 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);  in amdgpu_vm_update_ptes()
1547 	while (cursor.pfn < end) {  in amdgpu_vm_update_ptes()
1557 						&cursor, params->immediate);  in amdgpu_vm_update_ptes()
1562 		shift = amdgpu_vm_level_shift(adev, cursor.level);  in amdgpu_vm_update_ptes()
1563 		parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);  in amdgpu_vm_update_ptes()
1566 			if (amdgpu_vm_pt_descendant(adev, &cursor))  in amdgpu_vm_update_ptes()
1571 			if (cursor.level != AMDGPU_VM_PTB) {  in amdgpu_vm_update_ptes()
1572 				if (!amdgpu_vm_pt_descendant(adev, &cursor))  in amdgpu_vm_update_ptes()
1581 			if (amdgpu_vm_pt_descendant(adev, &cursor))  in amdgpu_vm_update_ptes()
1587 			if (!amdgpu_vm_pt_ancestor(&cursor))  in amdgpu_vm_update_ptes()
1592 		pt = cursor.entry->bo;  in amdgpu_vm_update_ptes()
1601 			if (!amdgpu_vm_pt_ancestor(&cursor))  in amdgpu_vm_update_ptes()
1604 			pt = cursor.entry->bo;  in amdgpu_vm_update_ptes()
1612 		mask = amdgpu_vm_entries_mask(adev, cursor.level);  in amdgpu_vm_update_ptes()
1613 		pe_start = ((cursor.pfn >> shift) & mask) * 8;  in amdgpu_vm_update_ptes()
1615 		entry_end += cursor.pfn & ~(entry_end - 1);  in amdgpu_vm_update_ptes()
1634 					       cursor.level, pe_start, dst,  in amdgpu_vm_update_ptes()
1650 		if (amdgpu_vm_pt_descendant(adev, &cursor)) {  in amdgpu_vm_update_ptes()
1657 			while (cursor.pfn < frag_start) {  in amdgpu_vm_update_ptes()
1659 				if (cursor.entry->bo) {  in amdgpu_vm_update_ptes()
1661 					amdgpu_vm_free_pts(adev, params->vm, &cursor);  in amdgpu_vm_update_ptes()
1663 				amdgpu_vm_pt_next(adev, &cursor);  in amdgpu_vm_update_ptes()
1668 			amdgpu_vm_pt_next(adev, &cursor);  in amdgpu_vm_update_ptes()
1710 	struct amdgpu_res_cursor cursor;  in amdgpu_vm_bo_update_mapping()  local
1751 			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);  in amdgpu_vm_bo_update_mapping()
1752 	while (cursor.remaining) {  in amdgpu_vm_bo_update_mapping()
1755 		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;  in amdgpu_vm_bo_update_mapping()
1760 				uint64_t pfn = cursor.start >> PAGE_SHIFT;  in amdgpu_vm_bo_update_mapping()
1780 				addr = cursor.start;  in amdgpu_vm_bo_update_mapping()
1783 				addr = pages_addr[cursor.start >> PAGE_SHIFT];  in amdgpu_vm_bo_update_mapping()
1789 				cursor.start;  in amdgpu_vm_bo_update_mapping()
1799 		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);  in amdgpu_vm_bo_update_mapping()