86 	struct kfd_process_device *pdd;  member
100 	struct kfd_process_device *pdd;  in kfd_sdma_activity_worker()  local
113 	pdd = workarea->pdd;  in kfd_sdma_activity_worker()
114 	if (!pdd)  in kfd_sdma_activity_worker()
116 	dqm = pdd->dev->dqm;  in kfd_sdma_activity_worker()
117 	qpd = &pdd->qpd;  in kfd_sdma_activity_worker()
172 		workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;  in kfd_sdma_activity_worker()
182 	mm = get_task_mm(pdd->process->lead_thread);  in kfd_sdma_activity_worker()
209 	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;  in kfd_sdma_activity_worker()
271 	struct kfd_process_device *pdd = NULL;  in kfd_get_cu_occupancy()  local
273 	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);  in kfd_get_cu_occupancy()
274 	dev = pdd->dev;  in kfd_get_cu_occupancy()
279 	proc = pdd->process;  in kfd_get_cu_occupancy()
280 	if (pdd->qpd.queue_count == 0) {  in kfd_get_cu_occupancy()
306 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,  in kfd_procfs_show()  local
308 		return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));  in kfd_procfs_show()
310 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,  in kfd_procfs_show()  local
317 		sdma_activity_work_handler.pdd = pdd;  in kfd_procfs_show()
397 		struct kfd_process_device *pdd = container_of(attr,  in kfd_procfs_stats_show()  local
402 		evict_jiffies = atomic64_read(&pdd->evict_duration_counter);  in kfd_procfs_stats_show()
422 	struct kfd_process_device *pdd;  in kfd_sysfs_counters_show()  local
425 		pdd = container_of(attr, struct kfd_process_device,  in kfd_sysfs_counters_show()
427 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));  in kfd_sysfs_counters_show()
430 		pdd = container_of(attr, struct kfd_process_device,  in kfd_sysfs_counters_show()
432 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));  in kfd_sysfs_counters_show()
435 		pdd = container_of(attr, struct kfd_process_device,  in kfd_sysfs_counters_show()
437 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));  in kfd_sysfs_counters_show()
548 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_procfs_add_sysfs_stats()  local
551 				"stats_%u", pdd->dev->id);  in kfd_procfs_add_sysfs_stats()
552 		pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);  in kfd_procfs_add_sysfs_stats()
553 		if (!pdd->kobj_stats)  in kfd_procfs_add_sysfs_stats()
556 		ret = kobject_init_and_add(pdd->kobj_stats,  in kfd_procfs_add_sysfs_stats()
564 			kobject_put(pdd->kobj_stats);  in kfd_procfs_add_sysfs_stats()
565 			pdd->kobj_stats = NULL;  in kfd_procfs_add_sysfs_stats()
569 		kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,  in kfd_procfs_add_sysfs_stats()
572 		if (pdd->dev->kfd2kgd->get_cu_occupancy)  in kfd_procfs_add_sysfs_stats()
573 			kfd_sysfs_create_file(pdd->kobj_stats,  in kfd_procfs_add_sysfs_stats()
574 					      &pdd->attr_cu_occupancy,  in kfd_procfs_add_sysfs_stats()
596 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_procfs_add_sysfs_counters()  local
600 			"counters_%u", pdd->dev->id);  in kfd_procfs_add_sysfs_counters()
614 		pdd->kobj_counters = kobj_counters;  in kfd_procfs_add_sysfs_counters()
615 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,  in kfd_procfs_add_sysfs_counters()
617 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,  in kfd_procfs_add_sysfs_counters()
619 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,  in kfd_procfs_add_sysfs_counters()
637 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_procfs_add_sysfs_files()  local
639 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",  in kfd_procfs_add_sysfs_files()
640 			 pdd->dev->id);  in kfd_procfs_add_sysfs_files()
641 		kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,  in kfd_procfs_add_sysfs_files()
642 				      pdd->vram_filename);  in kfd_procfs_add_sysfs_files()
644 		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",  in kfd_procfs_add_sysfs_files()
645 			 pdd->dev->id);  in kfd_procfs_add_sysfs_files()
646 		kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,  in kfd_procfs_add_sysfs_files()
647 					    pdd->sdma_filename);  in kfd_procfs_add_sysfs_files()
688 			struct kfd_process_device *pdd)  in kfd_process_free_gpuvm()  argument
690 	struct kfd_dev *dev = pdd->dev;  in kfd_process_free_gpuvm()
692 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);  in kfd_process_free_gpuvm()
693 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,  in kfd_process_free_gpuvm()
703 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,  in kfd_process_alloc_gpuvm()  argument
707 	struct kfd_dev *kdev = pdd->dev;  in kfd_process_alloc_gpuvm()
713 						 pdd->drm_priv, &mem, NULL, flags);  in kfd_process_alloc_gpuvm()
718 			pdd->drm_priv, NULL);  in kfd_process_alloc_gpuvm()
733 	handle = kfd_process_device_create_obj_handle(pdd, mem);  in kfd_process_alloc_gpuvm()
752 	kfd_process_device_remove_obj_handle(pdd, handle);  in kfd_process_alloc_gpuvm()
755 	kfd_process_free_gpuvm(mem, pdd);  in kfd_process_alloc_gpuvm()
759 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv,  in kfd_process_alloc_gpuvm()
772 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)  in kfd_process_device_reserve_ib_mem()  argument
774 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_reserve_ib_mem()
786 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,  in kfd_process_device_reserve_ib_mem()
922 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)  in kfd_process_device_free_bos()  argument
924 	struct kfd_process *p = pdd->process;  in kfd_process_device_free_bos()
933 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {  in kfd_process_device_free_bos()
944 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,  in kfd_process_device_free_bos()
945 						       pdd->drm_priv, NULL);  in kfd_process_device_free_bos()
946 		kfd_process_device_remove_obj_handle(pdd, id);  in kfd_process_device_free_bos()
963 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_destroy_pdds()  local
966 				pdd->dev->id, p->pasid);  in kfd_process_destroy_pdds()
968 		if (pdd->drm_file) {  in kfd_process_destroy_pdds()
970 					pdd->dev->kgd, pdd->drm_priv);  in kfd_process_destroy_pdds()
971 			fput(pdd->drm_file);  in kfd_process_destroy_pdds()
974 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)  in kfd_process_destroy_pdds()
975 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,  in kfd_process_destroy_pdds()
978 		kfree(pdd->qpd.doorbell_bitmap);  in kfd_process_destroy_pdds()
979 		idr_destroy(&pdd->alloc_idr);  in kfd_process_destroy_pdds()
981 		kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);  in kfd_process_destroy_pdds()
987 		if (pdd->runtime_inuse) {  in kfd_process_destroy_pdds()
988 			pm_runtime_mark_last_busy(pdd->dev->ddev->dev);  in kfd_process_destroy_pdds()
989 			pm_runtime_put_autosuspend(pdd->dev->ddev->dev);  in kfd_process_destroy_pdds()
990 			pdd->runtime_inuse = false;  in kfd_process_destroy_pdds()
993 		kfree(pdd);  in kfd_process_destroy_pdds()
1001 	struct kfd_process_device *pdd;  in kfd_process_remove_sysfs()  local
1013 		pdd = p->pdds[i];  in kfd_process_remove_sysfs()
1015 		sysfs_remove_file(p->kobj, &pdd->attr_vram);  in kfd_process_remove_sysfs()
1016 		sysfs_remove_file(p->kobj, &pdd->attr_sdma);  in kfd_process_remove_sysfs()
1018 		sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);  in kfd_process_remove_sysfs()
1019 		if (pdd->dev->kfd2kgd->get_cu_occupancy)  in kfd_process_remove_sysfs()
1020 			sysfs_remove_file(pdd->kobj_stats,  in kfd_process_remove_sysfs()
1021 					  &pdd->attr_cu_occupancy);  in kfd_process_remove_sysfs()
1022 		kobject_del(pdd->kobj_stats);  in kfd_process_remove_sysfs()
1023 		kobject_put(pdd->kobj_stats);  in kfd_process_remove_sysfs()
1024 		pdd->kobj_stats = NULL;  in kfd_process_remove_sysfs()
1028 		pdd = p->pdds[i];  in kfd_process_remove_sysfs()
1030 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);  in kfd_process_remove_sysfs()
1031 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);  in kfd_process_remove_sysfs()
1032 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);  in kfd_process_remove_sysfs()
1033 		kobject_del(pdd->kobj_counters);  in kfd_process_remove_sysfs()
1034 		kobject_put(pdd->kobj_counters);  in kfd_process_remove_sysfs()
1035 		pdd->kobj_counters = NULL;  in kfd_process_remove_sysfs()
1194 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)  in kfd_process_device_init_cwsr_dgpu()  argument
1196 	struct kfd_dev *dev = pdd->dev;  in kfd_process_device_init_cwsr_dgpu()
1197 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_init_cwsr_dgpu()
1208 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,  in kfd_process_device_init_cwsr_dgpu()
1424 	struct kfd_process_device *pdd = NULL;  in kfd_create_process_device_data()  local
1428 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);  in kfd_create_process_device_data()
1429 	if (!pdd)  in kfd_create_process_device_data()
1432 	if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {  in kfd_create_process_device_data()
1437 	if (init_doorbell_bitmap(&pdd->qpd, dev)) {  in kfd_create_process_device_data()
1442 	pdd->dev = dev;  in kfd_create_process_device_data()
1443 	INIT_LIST_HEAD(&pdd->qpd.queues_list);  in kfd_create_process_device_data()
1444 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);  in kfd_create_process_device_data()
1445 	pdd->qpd.dqm = dev->dqm;  in kfd_create_process_device_data()
1446 	pdd->qpd.pqm = &p->pqm;  in kfd_create_process_device_data()
1447 	pdd->qpd.evicted = 0;  in kfd_create_process_device_data()
1448 	pdd->qpd.mapped_gws_queue = false;  in kfd_create_process_device_data()
1449 	pdd->process = p;  in kfd_create_process_device_data()
1450 	pdd->bound = PDD_UNBOUND;  in kfd_create_process_device_data()
1451 	pdd->already_dequeued = false;  in kfd_create_process_device_data()
1452 	pdd->runtime_inuse = false;  in kfd_create_process_device_data()
1453 	pdd->vram_usage = 0;  in kfd_create_process_device_data()
1454 	pdd->sdma_past_activity_counter = 0;  in kfd_create_process_device_data()
1455 	atomic64_set(&pdd->evict_duration_counter, 0);  in kfd_create_process_device_data()
1456 	p->pdds[p->n_pdds++] = pdd;  in kfd_create_process_device_data()
1459 	idr_init(&pdd->alloc_idr);  in kfd_create_process_device_data()
1461 	return pdd;  in kfd_create_process_device_data()
1464 	kfree(pdd);  in kfd_create_process_device_data()
1482 int kfd_process_device_init_vm(struct kfd_process_device *pdd,  in kfd_process_device_init_vm()  argument
1492 	if (pdd->drm_priv)  in kfd_process_device_init_vm()
1495 	p = pdd->process;  in kfd_process_device_init_vm()
1496 	dev = pdd->dev;  in kfd_process_device_init_vm()
1505 	pdd->drm_priv = drm_file->private_data;  in kfd_process_device_init_vm()
1507 	ret = kfd_process_device_reserve_ib_mem(pdd);  in kfd_process_device_init_vm()
1510 	ret = kfd_process_device_init_cwsr_dgpu(pdd);  in kfd_process_device_init_vm()
1514 	pdd->drm_file = drm_file;  in kfd_process_device_init_vm()
1520 	kfd_process_device_free_bos(pdd);  in kfd_process_device_init_vm()
1521 	pdd->drm_priv = NULL;  in kfd_process_device_init_vm()
1536 	struct kfd_process_device *pdd;  in kfd_bind_process_to_device()  local
1539 	pdd = kfd_get_process_device_data(dev, p);  in kfd_bind_process_to_device()
1540 	if (!pdd) {  in kfd_bind_process_to_device()
1545 	if (!pdd->drm_priv)  in kfd_bind_process_to_device()
1553 	if (!pdd->runtime_inuse) {  in kfd_bind_process_to_device()
1561 	err = kfd_iommu_bind_process_to_device(pdd);  in kfd_bind_process_to_device()
1569 	pdd->runtime_inuse = true;  in kfd_bind_process_to_device()
1571 	return pdd;  in kfd_bind_process_to_device()
1575 	if (!pdd->runtime_inuse) {  in kfd_bind_process_to_device()
1586 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,  in kfd_process_device_create_obj_handle()  argument
1589 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);  in kfd_process_device_create_obj_handle()
1595 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,  in kfd_process_device_translate_handle()  argument
1601 	return idr_find(&pdd->alloc_idr, handle);  in kfd_process_device_translate_handle()
1607 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,  in kfd_process_device_remove_obj_handle()  argument
1611 		idr_remove(&pdd->alloc_idr, handle);  in kfd_process_device_remove_obj_handle()
1663 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_evict_queues()  local
1665 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,  in kfd_process_evict_queues()
1666 							    &pdd->qpd);  in kfd_process_evict_queues()
1681 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_evict_queues()  local
1685 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,  in kfd_process_evict_queues()
1686 							      &pdd->qpd))  in kfd_process_evict_queues()
1702 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_restore_queues()  local
1704 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,  in kfd_process_restore_queues()
1705 							      &pdd->qpd);  in kfd_process_restore_queues()
1862 	struct kfd_process_device *pdd;  in kfd_reserved_mem_mmap()  local
1870 	pdd = kfd_get_process_device_data(dev, process);  in kfd_reserved_mem_mmap()
1871 	if (!pdd)  in kfd_reserved_mem_mmap()
1873 	qpd = &pdd->qpd;  in kfd_reserved_mem_mmap()
1890 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)  in kfd_flush_tlb()  argument
1892 	struct kfd_dev *dev = pdd->dev;  in kfd_flush_tlb()
1898 		if (pdd->qpd.vmid)  in kfd_flush_tlb()
1900 							pdd->qpd.vmid);  in kfd_flush_tlb()
1903 					pdd->process->pasid, type);  in kfd_flush_tlb()