Lines Matching refs:svm
52 svm_lookup_device_by_sid(struct intel_svm *svm, u16 sid) in svm_lookup_device_by_sid() argument
57 list_for_each_entry_rcu(t, &svm->devs, list) { in svm_lookup_device_by_sid()
69 svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) in svm_lookup_device_by_dev() argument
74 list_for_each_entry_rcu(t, &svm->devs, list) { in svm_lookup_device_by_dev()
198 static void __flush_svm_range_dev(struct intel_svm *svm, in __flush_svm_range_dev() argument
208 qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); in __flush_svm_range_dev()
211 svm->pasid, sdev->qdep, address, in __flush_svm_range_dev()
215 static void intel_flush_svm_range_dev(struct intel_svm *svm, in intel_flush_svm_range_dev() argument
226 __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); in intel_flush_svm_range_dev()
231 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, in intel_flush_svm_range() argument
237 list_for_each_entry_rcu(sdev, &svm->devs, list) in intel_flush_svm_range()
238 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); in intel_flush_svm_range()
247 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); in intel_invalidate_range() local
249 intel_flush_svm_range(svm, start, in intel_invalidate_range()
255 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); in intel_mm_release() local
271 list_for_each_entry_rcu(sdev, &svm->devs, list) in intel_mm_release()
273 svm->pasid, true); in intel_mm_release()
290 struct intel_svm *svm; in pasid_to_svm_sdev() local
299 svm = pasid_private_find(pasid); in pasid_to_svm_sdev()
300 if (IS_ERR(svm)) in pasid_to_svm_sdev()
301 return PTR_ERR(svm); in pasid_to_svm_sdev()
303 if (!svm) in pasid_to_svm_sdev()
310 if (WARN_ON(list_empty(&svm->devs))) in pasid_to_svm_sdev()
312 sdev = svm_lookup_device_by_dev(svm, dev); in pasid_to_svm_sdev()
315 *rsvm = svm; in pasid_to_svm_sdev()
328 struct intel_svm *svm = NULL; in intel_svm_bind_gpasid() local
367 ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev); in intel_svm_bind_gpasid()
378 svm->pasid); in intel_svm_bind_gpasid()
383 if (!svm) { in intel_svm_bind_gpasid()
385 svm = kzalloc(sizeof(*svm), GFP_KERNEL); in intel_svm_bind_gpasid()
386 if (!svm) { in intel_svm_bind_gpasid()
395 svm->mm = get_task_mm(current); in intel_svm_bind_gpasid()
396 svm->pasid = data->hpasid; in intel_svm_bind_gpasid()
398 svm->gpasid = data->gpasid; in intel_svm_bind_gpasid()
399 svm->flags |= SVM_FLAG_GUEST_PASID; in intel_svm_bind_gpasid()
401 pasid_private_add(data->hpasid, svm); in intel_svm_bind_gpasid()
402 INIT_LIST_HEAD_RCU(&svm->devs); in intel_svm_bind_gpasid()
403 mmput(svm->mm); in intel_svm_bind_gpasid()
449 svm->flags |= SVM_FLAG_GUEST_MODE; in intel_svm_bind_gpasid()
452 list_add_rcu(&sdev->list, &svm->devs); in intel_svm_bind_gpasid()
454 if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) { in intel_svm_bind_gpasid()
456 kfree(svm); in intel_svm_bind_gpasid()
467 struct intel_svm *svm; in intel_svm_unbind_gpasid() local
474 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); in intel_svm_unbind_gpasid()
484 svm->pasid, false); in intel_svm_unbind_gpasid()
485 intel_svm_drain_prq(dev, svm->pasid); in intel_svm_unbind_gpasid()
488 if (list_empty(&svm->devs)) { in intel_svm_unbind_gpasid()
499 kfree(svm); in intel_svm_unbind_gpasid()
545 struct intel_svm *svm; in intel_svm_bind_mm() local
548 svm = pasid_private_find(mm->pasid); in intel_svm_bind_mm()
549 if (!svm) { in intel_svm_bind_mm()
550 svm = kzalloc(sizeof(*svm), GFP_KERNEL); in intel_svm_bind_mm()
551 if (!svm) in intel_svm_bind_mm()
554 svm->pasid = mm->pasid; in intel_svm_bind_mm()
555 svm->mm = mm; in intel_svm_bind_mm()
556 svm->flags = flags; in intel_svm_bind_mm()
557 INIT_LIST_HEAD_RCU(&svm->devs); in intel_svm_bind_mm()
560 svm->notifier.ops = &intel_mmuops; in intel_svm_bind_mm()
561 ret = mmu_notifier_register(&svm->notifier, mm); in intel_svm_bind_mm()
563 kfree(svm); in intel_svm_bind_mm()
568 ret = pasid_private_add(svm->pasid, svm); in intel_svm_bind_mm()
570 if (svm->notifier.ops) in intel_svm_bind_mm()
571 mmu_notifier_unregister(&svm->notifier, mm); in intel_svm_bind_mm()
572 kfree(svm); in intel_svm_bind_mm()
578 sdev = svm_lookup_device_by_dev(svm, dev); in intel_svm_bind_mm()
595 sdev->pasid = svm->pasid; in intel_svm_bind_mm()
618 if (!(flags & SVM_FLAG_SUPERVISOR_MODE) && list_empty(&svm->devs)) in intel_svm_bind_mm()
619 load_pasid(mm, svm->pasid); in intel_svm_bind_mm()
621 list_add_rcu(&sdev->list, &svm->devs); in intel_svm_bind_mm()
628 if (list_empty(&svm->devs)) { in intel_svm_bind_mm()
629 if (svm->notifier.ops) in intel_svm_bind_mm()
630 mmu_notifier_unregister(&svm->notifier, mm); in intel_svm_bind_mm()
632 kfree(svm); in intel_svm_bind_mm()
643 struct intel_svm *svm; in intel_svm_unbind_mm() local
651 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); in intel_svm_unbind_mm()
654 mm = svm->mm; in intel_svm_unbind_mm()
668 svm->pasid, false); in intel_svm_unbind_mm()
669 intel_svm_drain_prq(dev, svm->pasid); in intel_svm_unbind_mm()
672 if (list_empty(&svm->devs)) { in intel_svm_unbind_mm()
673 if (svm->notifier.ops) { in intel_svm_unbind_mm()
674 mmu_notifier_unregister(&svm->notifier, mm); in intel_svm_unbind_mm()
678 pasid_private_remove(svm->pasid); in intel_svm_unbind_mm()
683 memset(svm, 0x6b, sizeof(*svm)); in intel_svm_unbind_mm()
684 kfree(svm); in intel_svm_unbind_mm()
935 struct intel_svm *svm = NULL; in prq_event_thread() local
957 svm = NULL; in prq_event_thread()
985 if (!svm || svm->pasid != req->pasid) { in prq_event_thread()
990 svm = pasid_private_find(req->pasid); in prq_event_thread()
991 if (IS_ERR_OR_NULL(svm) || (svm->flags & SVM_FLAG_SUPERVISOR_MODE)) in prq_event_thread()
996 sdev = svm_lookup_device_by_sid(svm, req->rid); in prq_event_thread()
1111 struct intel_svm *svm = NULL; in intel_svm_page_response() local
1148 ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev); in intel_svm_page_response()
1158 if (svm->flags & SVM_FLAG_GUEST_MODE) { in intel_svm_page_response()
1167 if (mm != svm->mm) { in intel_svm_page_response()