Lines Matching refs:mem
84 struct kgd_mem *mem) in check_if_add_bo_to_vm() argument
88 list_for_each_entry(entry, &mem->bo_va_list, bo_list) in check_if_add_bo_to_vm()
102 uint64_t mem; in amdgpu_amdkfd_gpuvm_init_mem_limits() local
105 mem = si.totalram - si.totalhigh; in amdgpu_amdkfd_gpuvm_init_mem_limits()
106 mem *= si.mem_unit; in amdgpu_amdkfd_gpuvm_init_mem_limits()
109 kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2); in amdgpu_amdkfd_gpuvm_init_mem_limits()
110 kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); in amdgpu_amdkfd_gpuvm_init_mem_limits()
371 static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, in add_bo_to_vm() argument
377 struct amdgpu_bo *bo = mem->bo; in add_bo_to_vm()
378 uint64_t va = mem->va; in add_bo_to_vm()
379 struct list_head *list_bo_va = &mem->bo_va_list; in add_bo_to_vm()
380 unsigned long bo_size = bo->tbo.mem.size; in add_bo_to_vm()
408 mem->mapping_flags); in add_bo_to_vm()
443 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, in add_kgd_mem_to_kfd_bo_list() argument
447 struct ttm_validate_buffer *entry = &mem->validate_list; in add_kgd_mem_to_kfd_bo_list()
448 struct amdgpu_bo *bo = mem->bo; in add_kgd_mem_to_kfd_bo_list()
461 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, in remove_kgd_mem_from_kfd_bo_list() argument
466 bo_list_entry = &mem->validate_list; in remove_kgd_mem_from_kfd_bo_list()
484 static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm, in init_user_pages() argument
487 struct amdkfd_process_info *process_info = mem->process_info; in init_user_pages()
488 struct amdgpu_bo *bo = mem->bo; in init_user_pages()
518 amdgpu_bo_placement_from_domain(bo, mem->domain); in init_user_pages()
561 static int reserve_bo_and_vm(struct kgd_mem *mem, in reserve_bo_and_vm() argument
565 struct amdgpu_bo *bo = mem->bo; in reserve_bo_and_vm()
572 ctx->sync = &mem->sync; in reserve_bo_and_vm()
611 static int reserve_bo_and_cond_vms(struct kgd_mem *mem, in reserve_bo_and_cond_vms() argument
615 struct amdgpu_bo *bo = mem->bo; in reserve_bo_and_cond_vms()
623 ctx->sync = &mem->sync; in reserve_bo_and_cond_vms()
628 list_for_each_entry(entry, &mem->bo_va_list, bo_list) { in reserve_bo_and_cond_vms()
650 list_for_each_entry(entry, &mem->bo_va_list, bo_list) { in reserve_bo_and_cond_vms()
1072 void *vm, struct kgd_mem **mem, in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu() argument
1122 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1123 if (!*mem) { in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1127 INIT_LIST_HEAD(&(*mem)->bo_va_list); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1128 mutex_init(&(*mem)->lock); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1129 (*mem)->aql_queue = !!(flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1135 if ((*mem)->aql_queue) in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1156 (*mem)->mapping_flags = mapping_flags; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1158 amdgpu_sync_create(&(*mem)->sync); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1186 bo->kfd_bo = *mem; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1187 (*mem)->bo = bo; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1191 (*mem)->va = va; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1192 (*mem)->domain = domain; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1193 (*mem)->mapped_to_gpu_memory = 0; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1194 (*mem)->process_info = avm->process_info; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1195 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1198 ret = init_user_pages(*mem, current->mm, user_addr); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1209 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1216 mutex_destroy(&(*mem)->lock); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1217 kfree(*mem); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1227 struct kgd_dev *kgd, struct kgd_mem *mem) in amdgpu_amdkfd_gpuvm_free_memory_of_gpu() argument
1229 struct amdkfd_process_info *process_info = mem->process_info; in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1230 unsigned long bo_size = mem->bo->tbo.mem.size; in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1236 mutex_lock(&mem->lock); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1238 if (mem->mapped_to_gpu_memory > 0) { in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1240 mem->va, bo_size); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1241 mutex_unlock(&mem->lock); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1245 mutex_unlock(&mem->lock); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1251 amdgpu_mn_unregister(mem->bo); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1254 bo_list_entry = &mem->validate_list; in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1259 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1267 amdgpu_amdkfd_remove_eviction_fence(mem->bo, in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1269 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1270 mem->va + bo_size * (1 + mem->aql_queue)); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1273 list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list) in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1280 amdgpu_sync_free(&mem->sync); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1285 if (mem->bo->tbo.sg) { in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1286 sg_free_table(mem->bo->tbo.sg); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1287 kfree(mem->bo->tbo.sg); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1291 amdgpu_bo_unref(&mem->bo); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1292 mutex_destroy(&mem->lock); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1293 kfree(mem); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1299 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm) in amdgpu_amdkfd_gpuvm_map_memory_to_gpu() argument
1313 bo = mem->bo; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1323 mutex_lock(&mem->process_info->lock); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1331 is_invalid_userptr = atomic_read(&mem->invalid); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1335 mutex_lock(&mem->lock); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1337 domain = mem->domain; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1338 bo_size = bo->tbo.mem.size; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1341 mem->va, in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1342 mem->va + bo_size * (1 + mem->aql_queue), in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1345 ret = reserve_bo_and_vm(mem, vm, &ctx); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1355 bo->tbo.mem.mem_type == TTM_PL_SYSTEM) in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1358 if (check_if_add_bo_to_vm(avm, mem)) { in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1359 ret = add_bo_to_vm(adev, mem, avm, false, in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1363 if (mem->aql_queue) { in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1364 ret = add_bo_to_vm(adev, mem, avm, in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1375 if (mem->mapped_to_gpu_memory == 0 && in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1388 list_for_each_entry(entry, &mem->bo_va_list, bo_list) { in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1408 mem->mapped_to_gpu_memory++; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1410 mem->mapped_to_gpu_memory); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1431 mutex_unlock(&mem->process_info->lock); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1432 mutex_unlock(&mem->lock); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1437 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm) in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu() argument
1442 unsigned long bo_size = mem->bo->tbo.mem.size; in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1447 mutex_lock(&mem->lock); in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1449 ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx); in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1463 mem->va, in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1464 mem->va + bo_size * (1 + mem->aql_queue), in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1467 list_for_each_entry(entry, &mem->bo_va_list, bo_list) { in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1479 mem->va); in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1483 mem->mapped_to_gpu_memory--; in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1485 mem->mapped_to_gpu_memory); in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1492 if (mem->mapped_to_gpu_memory == 0 && in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1493 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count) in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1494 amdgpu_amdkfd_remove_eviction_fence(mem->bo, in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1500 mutex_unlock(&mem->lock); in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1505 struct kgd_dev *kgd, struct kgd_mem *mem, bool intr) in amdgpu_amdkfd_gpuvm_sync_memory() argument
1512 mutex_lock(&mem->lock); in amdgpu_amdkfd_gpuvm_sync_memory()
1513 amdgpu_sync_clone(&mem->sync, &sync); in amdgpu_amdkfd_gpuvm_sync_memory()
1514 mutex_unlock(&mem->lock); in amdgpu_amdkfd_gpuvm_sync_memory()
1522 struct kgd_mem *mem, void **kptr, uint64_t *size) in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() argument
1525 struct amdgpu_bo *bo = mem->bo; in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1535 mutex_lock(&mem->process_info->lock); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1556 bo, mem->process_info->eviction_fence); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1557 list_del_init(&mem->validate_list.head); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1564 mutex_unlock(&mem->process_info->lock); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1572 mutex_unlock(&mem->process_info->lock); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1578 struct kfd_vm_fault_info *mem) in amdgpu_amdkfd_gpuvm_get_vm_fault_info() argument
1584 *mem = *adev->gmc.vm_fault_info; in amdgpu_amdkfd_gpuvm_get_vm_fault_info()
1594 struct kgd_mem **mem, uint64_t *size, in amdgpu_amdkfd_gpuvm_import_dmabuf() argument
1617 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); in amdgpu_amdkfd_gpuvm_import_dmabuf()
1618 if (!*mem) in amdgpu_amdkfd_gpuvm_import_dmabuf()
1627 INIT_LIST_HEAD(&(*mem)->bo_va_list); in amdgpu_amdkfd_gpuvm_import_dmabuf()
1628 mutex_init(&(*mem)->lock); in amdgpu_amdkfd_gpuvm_import_dmabuf()
1629 (*mem)->mapping_flags = in amdgpu_amdkfd_gpuvm_import_dmabuf()
1633 (*mem)->bo = amdgpu_bo_ref(bo); in amdgpu_amdkfd_gpuvm_import_dmabuf()
1634 (*mem)->va = va; in amdgpu_amdkfd_gpuvm_import_dmabuf()
1635 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? in amdgpu_amdkfd_gpuvm_import_dmabuf()
1637 (*mem)->mapped_to_gpu_memory = 0; in amdgpu_amdkfd_gpuvm_import_dmabuf()
1638 (*mem)->process_info = avm->process_info; in amdgpu_amdkfd_gpuvm_import_dmabuf()
1639 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); in amdgpu_amdkfd_gpuvm_import_dmabuf()
1640 amdgpu_sync_create(&(*mem)->sync); in amdgpu_amdkfd_gpuvm_import_dmabuf()
1656 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, in amdgpu_amdkfd_evict_userptr() argument
1659 struct amdkfd_process_info *process_info = mem->process_info; in amdgpu_amdkfd_evict_userptr()
1663 invalid = atomic_inc_return(&mem->invalid); in amdgpu_amdkfd_evict_userptr()
1686 struct kgd_mem *mem, *tmp_mem; in update_invalid_user_pages() local
1694 list_for_each_entry_safe(mem, tmp_mem, in update_invalid_user_pages()
1697 if (!atomic_read(&mem->invalid)) in update_invalid_user_pages()
1700 bo = mem->bo; in update_invalid_user_pages()
1713 list_move_tail(&mem->validate_list.head, in update_invalid_user_pages()
1721 list_for_each_entry(mem, &process_info->userptr_inval_list, in update_invalid_user_pages()
1723 invalid = atomic_read(&mem->invalid); in update_invalid_user_pages()
1730 bo = mem->bo; in update_invalid_user_pages()
1747 if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid) in update_invalid_user_pages()
1768 struct kgd_mem *mem, *tmp_mem; in validate_invalid_user_pages() local
1792 list_for_each_entry(mem, &process_info->userptr_inval_list, in validate_invalid_user_pages()
1794 list_add_tail(&mem->resv_list.head, &resv_list); in validate_invalid_user_pages()
1795 mem->resv_list.bo = mem->validate_list.bo; in validate_invalid_user_pages()
1796 mem->resv_list.num_shared = mem->validate_list.num_shared; in validate_invalid_user_pages()
1813 list_for_each_entry_safe(mem, tmp_mem, in validate_invalid_user_pages()
1818 bo = mem->bo; in validate_invalid_user_pages()
1822 amdgpu_bo_placement_from_domain(bo, mem->domain); in validate_invalid_user_pages()
1830 list_move_tail(&mem->validate_list.head, in validate_invalid_user_pages()
1839 list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) { in validate_invalid_user_pages()
1849 atomic_inc(&mem->invalid); in validate_invalid_user_pages()
1964 struct kgd_mem *mem; in amdgpu_amdkfd_gpuvm_restore_process_bos() local
1990 list_for_each_entry(mem, &process_info->kfd_bo_list, in amdgpu_amdkfd_gpuvm_restore_process_bos()
1993 list_add_tail(&mem->resv_list.head, &ctx.list); in amdgpu_amdkfd_gpuvm_restore_process_bos()
1994 mem->resv_list.bo = mem->validate_list.bo; in amdgpu_amdkfd_gpuvm_restore_process_bos()
1995 mem->resv_list.num_shared = mem->validate_list.num_shared; in amdgpu_amdkfd_gpuvm_restore_process_bos()
2019 list_for_each_entry(mem, &process_info->kfd_bo_list, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2022 struct amdgpu_bo *bo = mem->bo; in amdgpu_amdkfd_gpuvm_restore_process_bos()
2023 uint32_t domain = mem->domain; in amdgpu_amdkfd_gpuvm_restore_process_bos()
2036 list_for_each_entry(bo_va_entry, &mem->bo_va_list, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2076 list_for_each_entry(mem, &process_info->kfd_bo_list, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2078 amdgpu_bo_fence(mem->bo, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2098 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) in amdgpu_amdkfd_add_gws_to_process() argument
2107 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); in amdgpu_amdkfd_add_gws_to_process()
2108 if (!*mem) in amdgpu_amdkfd_add_gws_to_process()
2111 mutex_init(&(*mem)->lock); in amdgpu_amdkfd_add_gws_to_process()
2112 (*mem)->bo = amdgpu_bo_ref(gws_bo); in amdgpu_amdkfd_add_gws_to_process()
2113 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; in amdgpu_amdkfd_add_gws_to_process()
2114 (*mem)->process_info = process_info; in amdgpu_amdkfd_add_gws_to_process()
2115 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); in amdgpu_amdkfd_add_gws_to_process()
2116 amdgpu_sync_create(&(*mem)->sync); in amdgpu_amdkfd_add_gws_to_process()
2120 mutex_lock(&(*mem)->process_info->lock); in amdgpu_amdkfd_add_gws_to_process()
2141 mutex_unlock(&(*mem)->process_info->lock); in amdgpu_amdkfd_add_gws_to_process()
2149 mutex_unlock(&(*mem)->process_info->lock); in amdgpu_amdkfd_add_gws_to_process()
2150 amdgpu_sync_free(&(*mem)->sync); in amdgpu_amdkfd_add_gws_to_process()
2151 remove_kgd_mem_from_kfd_bo_list(*mem, process_info); in amdgpu_amdkfd_add_gws_to_process()
2153 mutex_destroy(&(*mem)->lock); in amdgpu_amdkfd_add_gws_to_process()
2154 kfree(*mem); in amdgpu_amdkfd_add_gws_to_process()
2155 *mem = NULL; in amdgpu_amdkfd_add_gws_to_process()
2159 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) in amdgpu_amdkfd_remove_gws_from_process() argument
2163 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; in amdgpu_amdkfd_remove_gws_from_process()
2183 kfree(mem); in amdgpu_amdkfd_remove_gws_from_process()