Lines Matching +full:ctx +full:- +full:asid
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2018-2020 Intel Corporation.
70 u64 last, u32 asid);
76 return as->id; in iotlb_to_asid()
79 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid) in asid_to_as() argument
81 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS]; in asid_to_as()
85 if (as->id == asid) in asid_to_as()
91 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid) in asid_to_iotlb() argument
93 struct vhost_vdpa_as *as = asid_to_as(v, asid); in asid_to_iotlb()
98 return &as->iotlb; in asid_to_iotlb()
101 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid) in vhost_vdpa_alloc_as() argument
103 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS]; in vhost_vdpa_alloc_as()
106 if (asid_to_as(v, asid)) in vhost_vdpa_alloc_as()
109 if (asid >= v->vdpa->nas) in vhost_vdpa_alloc_as()
116 vhost_iotlb_init(&as->iotlb, 0, 0); in vhost_vdpa_alloc_as()
117 as->id = asid; in vhost_vdpa_alloc_as()
118 hlist_add_head(&as->hash_link, head); in vhost_vdpa_alloc_as()
124 u32 asid) in vhost_vdpa_find_alloc_as() argument
126 struct vhost_vdpa_as *as = asid_to_as(v, asid); in vhost_vdpa_find_alloc_as()
131 return vhost_vdpa_alloc_as(v, asid); in vhost_vdpa_find_alloc_as()
134 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid) in vhost_vdpa_remove_as() argument
136 struct vhost_vdpa_as *as = asid_to_as(v, asid); in vhost_vdpa_remove_as()
139 return -EINVAL; in vhost_vdpa_remove_as()
141 hlist_del(&as->hash_link); in vhost_vdpa_remove_as()
142 vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid); in vhost_vdpa_remove_as()
152 struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev); in handle_vq_kick()
153 const struct vdpa_config_ops *ops = v->vdpa->config; in handle_vq_kick()
155 ops->kick_vq(v->vdpa, vq - v->vqs); in handle_vq_kick()
161 struct eventfd_ctx *call_ctx = vq->call_ctx.ctx; in vhost_vdpa_virtqueue_cb()
172 struct eventfd_ctx *config_ctx = v->config_ctx; in vhost_vdpa_config_cb()
182 struct vhost_virtqueue *vq = &v->vqs[qid]; in vhost_vdpa_setup_vq_irq()
183 const struct vdpa_config_ops *ops = v->vdpa->config; in vhost_vdpa_setup_vq_irq()
184 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_setup_vq_irq()
187 if (!ops->get_vq_irq) in vhost_vdpa_setup_vq_irq()
190 irq = ops->get_vq_irq(vdpa, qid); in vhost_vdpa_setup_vq_irq()
194 irq_bypass_unregister_producer(&vq->call_ctx.producer); in vhost_vdpa_setup_vq_irq()
195 if (!vq->call_ctx.ctx) in vhost_vdpa_setup_vq_irq()
198 vq->call_ctx.producer.token = vq->call_ctx.ctx; in vhost_vdpa_setup_vq_irq()
199 vq->call_ctx.producer.irq = irq; in vhost_vdpa_setup_vq_irq()
200 ret = irq_bypass_register_producer(&vq->call_ctx.producer); in vhost_vdpa_setup_vq_irq()
202 dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret = %d\n", in vhost_vdpa_setup_vq_irq()
203 qid, vq->call_ctx.producer.token, ret); in vhost_vdpa_setup_vq_irq()
208 struct vhost_virtqueue *vq = &v->vqs[qid]; in vhost_vdpa_unsetup_vq_irq()
210 irq_bypass_unregister_producer(&vq->call_ctx.producer); in vhost_vdpa_unsetup_vq_irq()
215 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_reset()
217 v->in_batch = 0; in vhost_vdpa_reset()
224 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_bind_mm()
225 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_bind_mm()
227 if (!vdpa->use_va || !ops->bind_mm) in vhost_vdpa_bind_mm()
230 return ops->bind_mm(vdpa, v->vdev.mm); in vhost_vdpa_bind_mm()
235 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_unbind_mm()
236 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_unbind_mm()
238 if (!vdpa->use_va || !ops->unbind_mm) in vhost_vdpa_unbind_mm()
241 ops->unbind_mm(vdpa); in vhost_vdpa_unbind_mm()
246 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_device_id()
247 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_device_id()
250 device_id = ops->get_device_id(vdpa); in vhost_vdpa_get_device_id()
253 return -EFAULT; in vhost_vdpa_get_device_id()
260 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_status()
261 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_status()
264 status = ops->get_status(vdpa); in vhost_vdpa_get_status()
267 return -EFAULT; in vhost_vdpa_get_status()
274 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_status()
275 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_status()
277 u32 nvqs = v->nvqs; in vhost_vdpa_set_status()
282 return -EFAULT; in vhost_vdpa_set_status()
284 status_old = ops->get_status(vdpa); in vhost_vdpa_set_status()
291 return -EINVAL; in vhost_vdpa_set_status()
314 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_config_validate()
315 size_t size = vdpa->config->get_config_size(vdpa); in vhost_vdpa_config_validate()
317 if (c->len == 0 || c->off > size) in vhost_vdpa_config_validate()
318 return -EINVAL; in vhost_vdpa_config_validate()
320 if (c->len > size - c->off) in vhost_vdpa_config_validate()
321 return -E2BIG; in vhost_vdpa_config_validate()
329 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_config()
335 return -EFAULT; in vhost_vdpa_get_config()
337 return -EINVAL; in vhost_vdpa_get_config()
340 return -ENOMEM; in vhost_vdpa_get_config()
344 if (copy_to_user(c->buf, buf, config.len)) { in vhost_vdpa_get_config()
346 return -EFAULT; in vhost_vdpa_get_config()
356 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_config()
362 return -EFAULT; in vhost_vdpa_set_config()
364 return -EINVAL; in vhost_vdpa_set_config()
366 buf = vmemdup_user(c->buf, config.len); in vhost_vdpa_set_config()
378 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_can_suspend()
379 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_can_suspend()
381 return ops->suspend; in vhost_vdpa_can_suspend()
386 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_can_resume()
387 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_can_resume()
389 return ops->resume; in vhost_vdpa_can_resume()
394 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_features()
395 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_features()
398 features = ops->get_device_features(vdpa); in vhost_vdpa_get_features()
401 return -EFAULT; in vhost_vdpa_get_features()
408 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_backend_features()
409 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_backend_features()
411 if (!ops->get_backend_features) in vhost_vdpa_get_backend_features()
414 return ops->get_backend_features(vdpa); in vhost_vdpa_get_backend_features()
419 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_features()
420 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_features()
421 struct vhost_dev *d = &v->vdev; in vhost_vdpa_set_features()
430 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) in vhost_vdpa_set_features()
431 return -EBUSY; in vhost_vdpa_set_features()
434 return -EFAULT; in vhost_vdpa_set_features()
437 return -EINVAL; in vhost_vdpa_set_features()
440 actual_features = ops->get_driver_features(vdpa); in vhost_vdpa_set_features()
441 for (i = 0; i < d->nvqs; ++i) { in vhost_vdpa_set_features()
442 struct vhost_virtqueue *vq = d->vqs[i]; in vhost_vdpa_set_features()
444 mutex_lock(&vq->mutex); in vhost_vdpa_set_features()
445 vq->acked_features = actual_features; in vhost_vdpa_set_features()
446 mutex_unlock(&vq->mutex); in vhost_vdpa_set_features()
454 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_vring_num()
455 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_vring_num()
458 num = ops->get_vq_num_max(vdpa); in vhost_vdpa_get_vring_num()
461 return -EFAULT; in vhost_vdpa_get_vring_num()
468 if (v->config_ctx) { in vhost_vdpa_config_put()
469 eventfd_ctx_put(v->config_ctx); in vhost_vdpa_config_put()
470 v->config_ctx = NULL; in vhost_vdpa_config_put()
478 struct eventfd_ctx *ctx; in vhost_vdpa_set_config_call() local
483 return -EFAULT; in vhost_vdpa_set_config_call()
485 ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); in vhost_vdpa_set_config_call()
486 swap(ctx, v->config_ctx); in vhost_vdpa_set_config_call()
488 if (!IS_ERR_OR_NULL(ctx)) in vhost_vdpa_set_config_call()
489 eventfd_ctx_put(ctx); in vhost_vdpa_set_config_call()
491 if (IS_ERR(v->config_ctx)) { in vhost_vdpa_set_config_call()
492 long ret = PTR_ERR(v->config_ctx); in vhost_vdpa_set_config_call()
494 v->config_ctx = NULL; in vhost_vdpa_set_config_call()
498 v->vdpa->config->set_config_cb(v->vdpa, &cb); in vhost_vdpa_set_config_call()
506 .first = v->range.first, in vhost_vdpa_get_iova_range()
507 .last = v->range.last, in vhost_vdpa_get_iova_range()
511 return -EFAULT; in vhost_vdpa_get_iova_range()
517 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_config_size()
518 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_config_size()
521 size = ops->get_config_size(vdpa); in vhost_vdpa_get_config_size()
524 return -EFAULT; in vhost_vdpa_get_config_size()
531 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_vqs_count()
533 if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs))) in vhost_vdpa_get_vqs_count()
534 return -EFAULT; in vhost_vdpa_get_vqs_count()
546 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_suspend()
547 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_suspend()
549 if (!ops->suspend) in vhost_vdpa_suspend()
550 return -EOPNOTSUPP; in vhost_vdpa_suspend()
552 return ops->suspend(vdpa); in vhost_vdpa_suspend()
561 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_resume()
562 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_resume()
564 if (!ops->resume) in vhost_vdpa_resume()
565 return -EOPNOTSUPP; in vhost_vdpa_resume()
567 return ops->resume(vdpa); in vhost_vdpa_resume()
573 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_vring_ioctl()
574 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_vring_ioctl()
586 if (idx >= v->nvqs) in vhost_vdpa_vring_ioctl()
587 return -ENOBUFS; in vhost_vdpa_vring_ioctl()
589 idx = array_index_nospec(idx, v->nvqs); in vhost_vdpa_vring_ioctl()
590 vq = &v->vqs[idx]; in vhost_vdpa_vring_ioctl()
595 return -EFAULT; in vhost_vdpa_vring_ioctl()
596 ops->set_vq_ready(vdpa, idx, s.num); in vhost_vdpa_vring_ioctl()
599 if (!ops->get_vq_group) in vhost_vdpa_vring_ioctl()
600 return -EOPNOTSUPP; in vhost_vdpa_vring_ioctl()
602 s.num = ops->get_vq_group(vdpa, idx); in vhost_vdpa_vring_ioctl()
603 if (s.num >= vdpa->ngroups) in vhost_vdpa_vring_ioctl()
604 return -EIO; in vhost_vdpa_vring_ioctl()
606 return -EFAULT; in vhost_vdpa_vring_ioctl()
610 return -EFAULT; in vhost_vdpa_vring_ioctl()
611 if (s.num >= vdpa->nas) in vhost_vdpa_vring_ioctl()
612 return -EINVAL; in vhost_vdpa_vring_ioctl()
613 if (!ops->set_group_asid) in vhost_vdpa_vring_ioctl()
614 return -EOPNOTSUPP; in vhost_vdpa_vring_ioctl()
615 return ops->set_group_asid(vdpa, idx, s.num); in vhost_vdpa_vring_ioctl()
617 r = ops->get_vq_state(v->vdpa, idx, &vq_state); in vhost_vdpa_vring_ioctl()
622 vq->last_avail_idx = vq_state.packed.last_avail_idx | in vhost_vdpa_vring_ioctl()
624 vq->last_used_idx = vq_state.packed.last_used_idx | in vhost_vdpa_vring_ioctl()
627 vq->last_avail_idx = vq_state.split.avail_index; in vhost_vdpa_vring_ioctl()
632 r = vhost_vring_ioctl(&v->vdev, cmd, argp); in vhost_vdpa_vring_ioctl()
638 if (ops->set_vq_address(vdpa, idx, in vhost_vdpa_vring_ioctl()
639 (u64)(uintptr_t)vq->desc, in vhost_vdpa_vring_ioctl()
640 (u64)(uintptr_t)vq->avail, in vhost_vdpa_vring_ioctl()
641 (u64)(uintptr_t)vq->used)) in vhost_vdpa_vring_ioctl()
642 r = -EINVAL; in vhost_vdpa_vring_ioctl()
647 vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff; in vhost_vdpa_vring_ioctl()
648 vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000); in vhost_vdpa_vring_ioctl()
649 vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff; in vhost_vdpa_vring_ioctl()
650 vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000); in vhost_vdpa_vring_ioctl()
652 vq_state.split.avail_index = vq->last_avail_idx; in vhost_vdpa_vring_ioctl()
654 r = ops->set_vq_state(vdpa, idx, &vq_state); in vhost_vdpa_vring_ioctl()
658 if (vq->call_ctx.ctx) { in vhost_vdpa_vring_ioctl()
661 cb.trigger = vq->call_ctx.ctx; in vhost_vdpa_vring_ioctl()
667 ops->set_vq_cb(vdpa, idx, &cb); in vhost_vdpa_vring_ioctl()
672 ops->set_vq_num(vdpa, idx, vq->num); in vhost_vdpa_vring_ioctl()
682 struct vhost_vdpa *v = filep->private_data; in vhost_vdpa_unlocked_ioctl()
683 struct vhost_dev *d = &v->vdev; in vhost_vdpa_unlocked_ioctl()
691 return -EFAULT; in vhost_vdpa_unlocked_ioctl()
696 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
699 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
702 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
703 vhost_set_backend_features(&v->vdev, features); in vhost_vdpa_unlocked_ioctl()
707 mutex_lock(&d->mutex); in vhost_vdpa_unlocked_ioctl()
735 if (copy_to_user(argp, &v->vdpa->ngroups, in vhost_vdpa_unlocked_ioctl()
736 sizeof(v->vdpa->ngroups))) in vhost_vdpa_unlocked_ioctl()
737 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
740 if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas))) in vhost_vdpa_unlocked_ioctl()
741 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
745 r = -ENOIOCTLCMD; in vhost_vdpa_unlocked_ioctl()
758 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
776 r = vhost_dev_ioctl(&v->vdev, cmd, argp); in vhost_vdpa_unlocked_ioctl()
777 if (r == -ENOIOCTLCMD) in vhost_vdpa_unlocked_ioctl()
793 mutex_unlock(&d->mutex); in vhost_vdpa_unlocked_ioctl()
797 struct vhost_iotlb_map *map, u32 asid) in vhost_vdpa_general_unmap() argument
799 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_general_unmap()
800 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_general_unmap()
801 if (ops->dma_map) { in vhost_vdpa_general_unmap()
802 ops->dma_unmap(vdpa, asid, map->start, map->size); in vhost_vdpa_general_unmap()
803 } else if (ops->set_map == NULL) { in vhost_vdpa_general_unmap()
804 iommu_unmap(v->domain, map->start, map->size); in vhost_vdpa_general_unmap()
809 u64 start, u64 last, u32 asid) in vhost_vdpa_pa_unmap() argument
811 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_pa_unmap()
817 pinned = PFN_DOWN(map->size); in vhost_vdpa_pa_unmap()
818 for (pfn = PFN_DOWN(map->addr); in vhost_vdpa_pa_unmap()
819 pinned > 0; pfn++, pinned--) { in vhost_vdpa_pa_unmap()
821 if (map->perm & VHOST_ACCESS_WO) in vhost_vdpa_pa_unmap()
825 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); in vhost_vdpa_pa_unmap()
826 vhost_vdpa_general_unmap(v, map, asid); in vhost_vdpa_pa_unmap()
832 u64 start, u64 last, u32 asid) in vhost_vdpa_va_unmap() argument
838 map_file = (struct vdpa_map_file *)map->opaque; in vhost_vdpa_va_unmap()
839 fput(map_file->file); in vhost_vdpa_va_unmap()
841 vhost_vdpa_general_unmap(v, map, asid); in vhost_vdpa_va_unmap()
848 u64 last, u32 asid) in vhost_vdpa_iotlb_unmap() argument
850 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_iotlb_unmap()
852 if (vdpa->use_va) in vhost_vdpa_iotlb_unmap()
853 return vhost_vdpa_va_unmap(v, iotlb, start, last, asid); in vhost_vdpa_iotlb_unmap()
855 return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid); in vhost_vdpa_iotlb_unmap()
883 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_map()
884 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_map()
885 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_map()
886 u32 asid = iotlb_to_asid(iotlb); in vhost_vdpa_map() local
889 r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1, in vhost_vdpa_map()
894 if (ops->dma_map) { in vhost_vdpa_map()
895 r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque); in vhost_vdpa_map()
896 } else if (ops->set_map) { in vhost_vdpa_map()
897 if (!v->in_batch) in vhost_vdpa_map()
898 r = ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_map()
900 r = iommu_map(v->domain, iova, pa, size, in vhost_vdpa_map()
904 vhost_iotlb_del_range(iotlb, iova, iova + size - 1); in vhost_vdpa_map()
908 if (!vdpa->use_va) in vhost_vdpa_map()
909 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm); in vhost_vdpa_map()
918 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_unmap()
919 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_unmap()
920 u32 asid = iotlb_to_asid(iotlb); in vhost_vdpa_unmap() local
922 vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid); in vhost_vdpa_unmap()
924 if (ops->set_map) { in vhost_vdpa_unmap()
925 if (!v->in_batch) in vhost_vdpa_unmap()
926 ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_unmap()
935 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_va_map()
941 mmap_read_lock(dev->mm); in vhost_vdpa_va_map()
944 vma = find_vma(dev->mm, uaddr); in vhost_vdpa_va_map()
946 ret = -EINVAL; in vhost_vdpa_va_map()
949 map_size = min(size, vma->vm_end - uaddr); in vhost_vdpa_va_map()
950 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) && in vhost_vdpa_va_map()
951 !(vma->vm_flags & (VM_IO | VM_PFNMAP)))) in vhost_vdpa_va_map()
956 ret = -ENOMEM; in vhost_vdpa_va_map()
959 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start; in vhost_vdpa_va_map()
960 map_file->offset = offset; in vhost_vdpa_va_map()
961 map_file->file = get_file(vma->vm_file); in vhost_vdpa_va_map()
965 fput(map_file->file); in vhost_vdpa_va_map()
970 size -= map_size; in vhost_vdpa_va_map()
975 vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova); in vhost_vdpa_va_map()
977 mmap_read_unlock(dev->mm); in vhost_vdpa_va_map()
986 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_pa_map()
999 return -ENOMEM; in vhost_vdpa_pa_map()
1006 ret = -EINVAL; in vhost_vdpa_pa_map()
1010 mmap_read_lock(dev->mm); in vhost_vdpa_pa_map()
1013 if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) { in vhost_vdpa_pa_map()
1014 ret = -ENOMEM; in vhost_vdpa_pa_map()
1031 ret = -ENOMEM; in vhost_vdpa_pa_map()
1046 csize = PFN_PHYS(last_pfn - map_pfn + 1); in vhost_vdpa_pa_map()
1060 pinned - i); in vhost_vdpa_pa_map()
1073 npages -= pinned; in vhost_vdpa_pa_map()
1077 ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1), in vhost_vdpa_pa_map()
1100 mmap_read_unlock(dev->mm); in vhost_vdpa_pa_map()
1111 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_process_iotlb_update()
1113 if (msg->iova < v->range.first || !msg->size || in vhost_vdpa_process_iotlb_update()
1114 msg->iova > U64_MAX - msg->size + 1 || in vhost_vdpa_process_iotlb_update()
1115 msg->iova + msg->size - 1 > v->range.last) in vhost_vdpa_process_iotlb_update()
1116 return -EINVAL; in vhost_vdpa_process_iotlb_update()
1118 if (vhost_iotlb_itree_first(iotlb, msg->iova, in vhost_vdpa_process_iotlb_update()
1119 msg->iova + msg->size - 1)) in vhost_vdpa_process_iotlb_update()
1120 return -EEXIST; in vhost_vdpa_process_iotlb_update()
1122 if (vdpa->use_va) in vhost_vdpa_process_iotlb_update()
1123 return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size, in vhost_vdpa_process_iotlb_update()
1124 msg->uaddr, msg->perm); in vhost_vdpa_process_iotlb_update()
1126 return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr, in vhost_vdpa_process_iotlb_update()
1127 msg->perm); in vhost_vdpa_process_iotlb_update()
1130 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid, in vhost_vdpa_process_iotlb_msg() argument
1134 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_process_iotlb_msg()
1135 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_process_iotlb_msg()
1140 mutex_lock(&dev->mutex); in vhost_vdpa_process_iotlb_msg()
1146 if (msg->type == VHOST_IOTLB_UPDATE || in vhost_vdpa_process_iotlb_msg()
1147 msg->type == VHOST_IOTLB_BATCH_BEGIN) { in vhost_vdpa_process_iotlb_msg()
1148 as = vhost_vdpa_find_alloc_as(v, asid); in vhost_vdpa_process_iotlb_msg()
1150 dev_err(&v->dev, "can't find and alloc asid %d\n", in vhost_vdpa_process_iotlb_msg()
1151 asid); in vhost_vdpa_process_iotlb_msg()
1152 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1155 iotlb = &as->iotlb; in vhost_vdpa_process_iotlb_msg()
1157 iotlb = asid_to_iotlb(v, asid); in vhost_vdpa_process_iotlb_msg()
1159 if ((v->in_batch && v->batch_asid != asid) || !iotlb) { in vhost_vdpa_process_iotlb_msg()
1160 if (v->in_batch && v->batch_asid != asid) { in vhost_vdpa_process_iotlb_msg()
1161 dev_info(&v->dev, "batch id %d asid %d\n", in vhost_vdpa_process_iotlb_msg()
1162 v->batch_asid, asid); in vhost_vdpa_process_iotlb_msg()
1165 dev_err(&v->dev, "no iotlb for asid %d\n", asid); in vhost_vdpa_process_iotlb_msg()
1166 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1170 switch (msg->type) { in vhost_vdpa_process_iotlb_msg()
1175 vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size); in vhost_vdpa_process_iotlb_msg()
1178 v->batch_asid = asid; in vhost_vdpa_process_iotlb_msg()
1179 v->in_batch = true; in vhost_vdpa_process_iotlb_msg()
1182 if (v->in_batch && ops->set_map) in vhost_vdpa_process_iotlb_msg()
1183 ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_process_iotlb_msg()
1184 v->in_batch = false; in vhost_vdpa_process_iotlb_msg()
1187 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1191 mutex_unlock(&dev->mutex); in vhost_vdpa_process_iotlb_msg()
1199 struct file *file = iocb->ki_filp; in vhost_vdpa_chr_write_iter()
1200 struct vhost_vdpa *v = file->private_data; in vhost_vdpa_chr_write_iter()
1201 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_chr_write_iter()
1208 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_alloc_domain()
1209 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_alloc_domain()
1215 if (ops->set_map || ops->dma_map) in vhost_vdpa_alloc_domain()
1218 bus = dma_dev->bus; in vhost_vdpa_alloc_domain()
1220 return -EFAULT; in vhost_vdpa_alloc_domain()
1223 dev_warn_once(&v->dev, in vhost_vdpa_alloc_domain()
1225 return -ENOTSUPP; in vhost_vdpa_alloc_domain()
1228 v->domain = iommu_domain_alloc(bus); in vhost_vdpa_alloc_domain()
1229 if (!v->domain) in vhost_vdpa_alloc_domain()
1230 return -EIO; in vhost_vdpa_alloc_domain()
1232 ret = iommu_attach_device(v->domain, dma_dev); in vhost_vdpa_alloc_domain()
1239 iommu_domain_free(v->domain); in vhost_vdpa_alloc_domain()
1240 v->domain = NULL; in vhost_vdpa_alloc_domain()
1246 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_free_domain()
1249 if (v->domain) { in vhost_vdpa_free_domain()
1250 iommu_detach_device(v->domain, dma_dev); in vhost_vdpa_free_domain()
1251 iommu_domain_free(v->domain); in vhost_vdpa_free_domain()
1254 v->domain = NULL; in vhost_vdpa_free_domain()
1259 struct vdpa_iova_range *range = &v->range; in vhost_vdpa_set_iova_range()
1260 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_iova_range()
1261 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_iova_range()
1263 if (ops->get_iova_range) { in vhost_vdpa_set_iova_range()
1264 *range = ops->get_iova_range(vdpa); in vhost_vdpa_set_iova_range()
1265 } else if (v->domain && v->domain->geometry.force_aperture) { in vhost_vdpa_set_iova_range()
1266 range->first = v->domain->geometry.aperture_start; in vhost_vdpa_set_iova_range()
1267 range->last = v->domain->geometry.aperture_end; in vhost_vdpa_set_iova_range()
1269 range->first = 0; in vhost_vdpa_set_iova_range()
1270 range->last = ULLONG_MAX; in vhost_vdpa_set_iova_range()
1277 u32 asid; in vhost_vdpa_cleanup() local
1279 for (asid = 0; asid < v->vdpa->nas; asid++) { in vhost_vdpa_cleanup()
1280 as = asid_to_as(v, asid); in vhost_vdpa_cleanup()
1282 vhost_vdpa_remove_as(v, asid); in vhost_vdpa_cleanup()
1286 vhost_dev_cleanup(&v->vdev); in vhost_vdpa_cleanup()
1287 kfree(v->vdev.vqs); in vhost_vdpa_cleanup()
1298 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); in vhost_vdpa_open()
1300 opened = atomic_cmpxchg(&v->opened, 0, 1); in vhost_vdpa_open()
1302 return -EBUSY; in vhost_vdpa_open()
1304 nvqs = v->nvqs; in vhost_vdpa_open()
1311 r = -ENOMEM; in vhost_vdpa_open()
1315 dev = &v->vdev; in vhost_vdpa_open()
1317 vqs[i] = &v->vqs[i]; in vhost_vdpa_open()
1318 vqs[i]->handle_kick = handle_vq_kick; in vhost_vdpa_open()
1329 filep->private_data = v; in vhost_vdpa_open()
1336 atomic_dec(&v->opened); in vhost_vdpa_open()
1344 for (i = 0; i < v->nvqs; i++) in vhost_vdpa_clean_irq()
1350 struct vhost_vdpa *v = filep->private_data; in vhost_vdpa_release()
1351 struct vhost_dev *d = &v->vdev; in vhost_vdpa_release()
1353 mutex_lock(&d->mutex); in vhost_vdpa_release()
1354 filep->private_data = NULL; in vhost_vdpa_release()
1357 vhost_dev_stop(&v->vdev); in vhost_vdpa_release()
1361 mutex_unlock(&d->mutex); in vhost_vdpa_release()
1363 atomic_dec(&v->opened); in vhost_vdpa_release()
1364 complete(&v->completion); in vhost_vdpa_release()
1372 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; in vhost_vdpa_fault()
1373 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_fault()
1374 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_fault()
1376 struct vm_area_struct *vma = vmf->vma; in vhost_vdpa_fault()
1377 u16 index = vma->vm_pgoff; in vhost_vdpa_fault()
1379 notify = ops->get_vq_notification(vdpa, index); in vhost_vdpa_fault()
1381 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); in vhost_vdpa_fault()
1382 if (remap_pfn_range(vma, vmf->address & PAGE_MASK, in vhost_vdpa_fault()
1384 vma->vm_page_prot)) in vhost_vdpa_fault()
1396 struct vhost_vdpa *v = vma->vm_file->private_data; in vhost_vdpa_mmap()
1397 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_mmap()
1398 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_mmap()
1400 unsigned long index = vma->vm_pgoff; in vhost_vdpa_mmap()
1402 if (vma->vm_end - vma->vm_start != PAGE_SIZE) in vhost_vdpa_mmap()
1403 return -EINVAL; in vhost_vdpa_mmap()
1404 if ((vma->vm_flags & VM_SHARED) == 0) in vhost_vdpa_mmap()
1405 return -EINVAL; in vhost_vdpa_mmap()
1406 if (vma->vm_flags & VM_READ) in vhost_vdpa_mmap()
1407 return -EINVAL; in vhost_vdpa_mmap()
1409 return -EINVAL; in vhost_vdpa_mmap()
1410 if (!ops->get_vq_notification) in vhost_vdpa_mmap()
1411 return -ENOTSUPP; in vhost_vdpa_mmap()
1417 notify = ops->get_vq_notification(vdpa, index); in vhost_vdpa_mmap()
1418 if (notify.addr & (PAGE_SIZE - 1)) in vhost_vdpa_mmap()
1419 return -EINVAL; in vhost_vdpa_mmap()
1420 if (vma->vm_end - vma->vm_start != notify.size) in vhost_vdpa_mmap()
1421 return -ENOTSUPP; in vhost_vdpa_mmap()
1424 vma->vm_ops = &vhost_vdpa_vm_ops; in vhost_vdpa_mmap()
1446 ida_simple_remove(&vhost_vdpa_ida, v->minor); in vhost_vdpa_release_dev()
1447 kfree(v->vqs); in vhost_vdpa_release_dev()
1453 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_probe()
1461 if (!ops->set_map && !ops->dma_map && in vhost_vdpa_probe()
1462 (vdpa->ngroups > 1 || vdpa->nas > 1)) in vhost_vdpa_probe()
1463 return -EOPNOTSUPP; in vhost_vdpa_probe()
1467 return -ENOMEM; in vhost_vdpa_probe()
1476 atomic_set(&v->opened, 0); in vhost_vdpa_probe()
1477 v->minor = minor; in vhost_vdpa_probe()
1478 v->vdpa = vdpa; in vhost_vdpa_probe()
1479 v->nvqs = vdpa->nvqs; in vhost_vdpa_probe()
1480 v->virtio_id = ops->get_device_id(vdpa); in vhost_vdpa_probe()
1482 device_initialize(&v->dev); in vhost_vdpa_probe()
1483 v->dev.release = vhost_vdpa_release_dev; in vhost_vdpa_probe()
1484 v->dev.parent = &vdpa->dev; in vhost_vdpa_probe()
1485 v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor); in vhost_vdpa_probe()
1486 v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue), in vhost_vdpa_probe()
1488 if (!v->vqs) { in vhost_vdpa_probe()
1489 r = -ENOMEM; in vhost_vdpa_probe()
1493 r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); in vhost_vdpa_probe()
1497 cdev_init(&v->cdev, &vhost_vdpa_fops); in vhost_vdpa_probe()
1498 v->cdev.owner = THIS_MODULE; in vhost_vdpa_probe()
1500 r = cdev_device_add(&v->cdev, &v->dev); in vhost_vdpa_probe()
1504 init_completion(&v->completion); in vhost_vdpa_probe()
1508 INIT_HLIST_HEAD(&v->as[i]); in vhost_vdpa_probe()
1513 put_device(&v->dev); in vhost_vdpa_probe()
1522 cdev_device_del(&v->cdev, &v->dev); in vhost_vdpa_remove()
1525 opened = atomic_cmpxchg(&v->opened, 0, 1); in vhost_vdpa_remove()
1528 wait_for_completion(&v->completion); in vhost_vdpa_remove()
1531 put_device(&v->dev); in vhost_vdpa_remove()
1547 "vhost-vdpa"); in vhost_vdpa_init()
1574 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");